Support new plural keys format.

All the old plural phrases were changed to work with the new format.

Telegram/SourceFiles/codegen/lang/generator.cpp

@@ -31,8 +31,6 @@ namespace codegen {
 namespace lang {
 namespace {
 
-constexpr auto kMaxPluralVariants = 6;
-
 char hexChar(uchar ch) {
 	if (ch < 10) {
 		return '0' + ch;
@@ -120,7 +118,6 @@ bool Generator::writeHeader() {
 	header_->include("lang/lang_tag.h").newline().pushNamespace("Lang").stream() << "\
 \n\
 constexpr auto kTagsCount = " << langpack_.tags.size() << ";\n\
-constexpr auto kTagsPluralVariants = " << kMaxPluralVariants << ";\n\
 \n";
 
 	header_->popNamespace().newline();
@@ -141,21 +138,26 @@ enum LangKey {\n";
 \n\
 QString lang(LangKey key);\n\
 \n";
+
 	for (auto &entry : langpack_.entries) {
-		if (!entry.tags.empty()) {
-			auto &key = entry.key;
-			auto params = QStringList();
-			auto applyTags = QStringList();
-			for (auto &tagData : entry.tags) {
-				auto &tag = tagData.tag;
-				auto isPlural = isTagPlural(key, tag);
-				params.push_back("lngtag_" + tag + ", " + (isPlural ? "float64 " : "const QString &") + tag + "__val");
-				applyTags.push_back("\tresult = Lang::Tag(result, lt_" + tag + ", " + (isPlural ? ("Lang::Plural(" + key + "__" + tag + "0, lt_" + tag + ", " + tag + "__val)") : (tag + "__val")) + ");");
+		auto isPlural = !entry.keyBase.isEmpty();
+		auto &key = entry.key;
+		auto params = QStringList();
+		auto applyTags = QStringList();
+		for (auto &tagData : entry.tags) {
+			auto &tag = tagData.tag;
+			auto isPluralTag = isPlural && (tag == kPluralTag);
+			params.push_back("lngtag_" + tag + ", " + (isPluralTag ? "float64 " : "const QString &") + tag + "__val");
+			if (!isPluralTag) {
+				applyTags.push_back("\tresult = Lang::Tag(result, lt_" + tag + ", " + tag + "__val);\n");
 			}
+		}
+		if (!entry.tags.empty() && (!isPlural || key == ComputePluralKey(entry.keyBase, 0))) {
+			auto initialString = isPlural ? ("Lang::Plural(" + key + ", lt_" + kPluralTag + ", " + kPluralTag + "__val)") : ("lang(" + getFullKey(entry) + ")");
 			header_->stream() << "\
-inline QString " << entry.key << "(" << params.join(QString(", ")) << ") {\n\
-	auto result = lang(" << entry.key << "__tagged);\n\
-" << applyTags.join('\n') << ";\n\
+inline QString " << (isPlural ? entry.keyBase : key) << "(" << params.join(QString(", ")) << ") {\n\
+	auto result = " << initialString << ";\n\
+" << applyTags.join(QString()) << "\
 	return result;\n\
 }\n\
 \n";
@@ -167,7 +169,6 @@ inline QString " << entry.key << "(" << params.join(QString(", ")) << ") {\n\
 const char *GetKeyName(LangKey key);\n\
 ushort GetTagIndex(QLatin1String tag);\n\
 LangKey GetKeyIndex(QLatin1String key);\n\
-LangKey GetSubkeyIndex(LangKey key, ushort tag, ushort index);\n\
 bool IsTagReplaced(LangKey key, ushort tag);\n\
 QString GetOriginalValue(LangKey key);\n\
 \n";
@@ -258,15 +259,19 @@ LangKey GetKeyIndex(QLatin1String key) {\n\
 	auto taggedKeys = std::map<QString, QString>();
 	auto keysSet = std::set<QString, std::greater<QString>>();
 	for (auto &entry : langpack_.entries) {
-		if (entry.key.mid(0, entry.key.size() - 1).endsWith("__count")) {
-			continue;
+		if (!entry.keyBase.isEmpty()) {
+			for (auto i = 0; i != kPluralPartCount; ++i) {
+				auto keyName = entry.keyBase + '#' + kPluralParts[i];
+				taggedKeys.emplace(keyName, ComputePluralKey(entry.keyBase, i));
+				keysSet.insert(keyName);
+			}
+		} else {
+			auto full = getFullKey(entry);
+			if (full != entry.key) {
+				taggedKeys.emplace(entry.key, full);
+			}
+			keysSet.insert(entry.key);
 		}
-
-		auto full = getFullKey(entry);
-		if (full != entry.key) {
-			taggedKeys.emplace(entry.key, full);
-		}
-		keysSet.insert(entry.key);
 	}
 
 	writeSetSearch(keysSet, [&taggedKeys](const QString &key) {
@@ -277,44 +282,28 @@ LangKey GetKeyIndex(QLatin1String key) {\n\
 	source_->stream() << "\
 }\n\
 \n\
-LangKey GetSubkeyIndex(LangKey key, ushort tag, ushort index) {\n\
-	if (index >= kTagsPluralVariants) return kLangKeysCount;\n\
-\n\
-	switch (key) {\n";
-
-	for (auto &entry : langpack_.entries) {
-		auto cases = QString();
-		for (auto &tag : entry.tags) {
-			if (isTagPlural(entry.key, tag.tag)) {
-				cases += "\t\t\tcase lt_" + tag.tag + ": return LangKey(" + entry.key + "__" + tag.tag + "0 + index);\n";
-			}
-		}
-		if (cases.isEmpty()) {
-			continue;
-		}
-		source_->stream() << "\
-	case " << entry.key << "__tagged: {\n\
-		switch (tag) {\n\
-" << cases << "\
-		}\n\
-	} break;\n";
-	}
-
-	source_->stream() << "\
-	}\n\
-\n\
-	return kLangKeysCount;\n\
-}\n\
-\n\
 bool IsTagReplaced(LangKey key, ushort tag) {\n\
 	switch (key) {\n";
 
+	auto lastWrittenPluralEntry = QString();
 	for (auto &entry : langpack_.entries) {
 		if (entry.tags.empty()) {
 			continue;
 		}
+		if (!entry.keyBase.isEmpty()) {
+			if (entry.keyBase == lastWrittenPluralEntry) {
+				continue;
+			}
+			lastWrittenPluralEntry = entry.keyBase;
+			for (auto i = 0; i != kPluralPartCount; ++i) {
+				source_->stream() << "\
+	case " << ComputePluralKey(entry.keyBase, i) << ":" << ((i + 1 == kPluralPartCount) ? " {" : "") << "\n";
+			}
+		} else {
+			source_->stream() << "\
+	case " << getFullKey(entry) << ": {\n";
+		}
 		source_->stream() << "\
-	case " << entry.key << "__tagged: {\n\
 		switch (tag) {\n";
 		for (auto &tag : entry.tags) {
 			source_->stream() << "\
@@ -480,21 +469,11 @@ void Generator::writeSetSearch(const std::set<QString, std::greater<QString>> &s
 }
 
 QString Generator::getFullKey(const LangPack::Entry &entry) {
-	if (entry.tags.empty()) {
+	if (!entry.keyBase.isEmpty() || entry.tags.empty()) {
 		return entry.key;
 	}
 	return entry.key + "__tagged";
 }
 
-bool Generator::isTagPlural(const QString &key, const QString &tag) const {
-	auto searchForKey = key + "__" + tag + "0";
-	for (auto &entry : langpack_.entries) {
-		if (entry.key == searchForKey) {
-			return true;
-		}
-	}
-	return false;
-}
-
 } // namespace lang
 } // namespace codegen

Telegram/SourceFiles/codegen/lang/generator.h

@@ -43,7 +43,6 @@ public:
 
 private:
 	QString getFullKey(const LangPack::Entry &entry);
-	bool isTagPlural(const QString &key, const QString &tag) const;
 
 	template <typename ComputeResult>
 	void writeSetSearch(const std::set<QString, std::greater<QString>> &set, ComputeResult computeResult, const QString &invalidResult);

Telegram/SourceFiles/codegen/lang/parsed_file.cpp

@@ -46,7 +46,7 @@ bool ValidateAnsiString(const QString &value) {
 }
 
 bool ValidateKey(const QString &key) {
-	static const auto validator = QRegularExpression("^[a-z0-9_.-]+$", QRegularExpression::CaseInsensitiveOption);
+	static const auto validator = QRegularExpression("^[a-z0-9_.-]+(#(one|other))?$", QRegularExpression::CaseInsensitiveOption);
 	if (!validator.match(key).hasMatch()) {
 		return false;
 	}
@@ -78,6 +78,21 @@ QString PrepareCommandString(int index) {
 
 } // namespace
 
+const std::array<QString, kPluralPartCount> kPluralParts = {
+	"zero",
+	"one",
+	"two",
+	"few",
+	"many",
+	"other",
+};
+
+const QString kPluralTag = "count";
+
+QString ComputePluralKey(const QString &base, int index) {
+	return base + "__plural" + QString::number(index);
+}
+
 ParsedFile::ParsedFile(const Options &options)
 : filePath_(options.inputPath)
 , file_(filePath_)
@@ -104,7 +119,7 @@ bool ParsedFile::read() {
 					logErrorUnexpectedToken() << "'=' for '" << keyToken.value.toStdString() << "' key";
 				}
 			} else {
-				logErrorUnexpectedToken() << "string key name (/^[a-z0-9_.-]+$/i)";
+				logErrorUnexpectedToken() << "string key name (/^[a-z0-9_.-]+(#(one|other))?$/i)";
 			}
 		}
 		if (file_.atEnd()) {
@@ -113,9 +128,46 @@ bool ParsedFile::read() {
 		logErrorUnexpectedToken() << "ansi string key name";
 	} while (!failed());
 
+	fillPluralTags();
+
 	return !failed();
 }
 
+void ParsedFile::fillPluralTags() {
+	auto count = result_.entries.size();
+	for (auto i = 0; i != count;) {
+		auto &baseEntry = result_.entries[i];
+		if (baseEntry.keyBase.isEmpty()) {
+			++i;
+			continue;
+		}
+		logAssert(i + kPluralPartCount < count);
+
+		// Accumulate all tags from all plural variants.
+		auto tags = std::vector<LangPack::Tag>();
+		for (auto j = i; j != i + kPluralPartCount; ++j) {
+			if (tags.empty()) {
+				tags = result_.entries[j].tags;
+			} else {
+				for (auto &tag : result_.entries[j].tags) {
+					if (std::find(tags.begin(), tags.end(), tag) == tags.end()) {
+						tags.push_back(tag);
+					}
+				}
+			}
+		}
+		logAssert(!tags.empty());
+		logAssert(tags.front().tag == kPluralTag);
+
+		// Set this tags list to all plural variants.
+		for (auto j = i; j != i + kPluralPartCount; ++j) {
+			result_.entries[j].tags = tags;
+		}
+
+		i += kPluralPartCount;
+	}
+}
+
 BasicToken ParsedFile::assertNextToken(BasicToken::Type type) {
 	auto result = file_.getToken(type);
 	if (!result) {
@@ -204,24 +256,85 @@ QString ParsedFile::extractTagData(const QString &tagText, LangPack *to) {
 	return PrepareCommandString(tagIndex);
 }
 
-void ParsedFile::addEntity(const QString &key, const QString &value) {
-	for (auto &entry : result_.entries) {
-		if (entry.key == key) {
-			logError(kErrorBadString) << "duplicate found for key '" << key.toStdString() << "'";
+void ParsedFile::addEntity(QString key, const QString &value) {
+	auto pluralPartOffset = key.indexOf('#');
+	auto pluralIndex = -1;
+	if (pluralPartOffset >= 0) {
+		auto pluralPart = key.mid(pluralPartOffset + 1);
+		pluralIndex = std::find(kPluralParts.begin(), kPluralParts.end(), pluralPart) - kPluralParts.begin();
+		if (pluralIndex < 0 || pluralIndex >= kPluralParts.size()) {
+			logError(kErrorBadString) << "bad plural part for key '" << key.toStdString() << "': '" << pluralPart.toStdString() << "'";
 			return;
 		}
+		key = key.mid(0, pluralPartOffset);
+	}
+	auto checkKey = [this](const QString &key) {
+		for (auto &entry : result_.entries) {
+			if (entry.key == key) {
+				if (entry.keyBase.isEmpty() || !entry.tags.empty()) {
+					// Empty tags in plural entry means it was not encountered yet.
+					logError(kErrorBadString) << "duplicate found for key '" << key.toStdString() << "'";
+					return false;
+				}
+			}
+		}
+		return true;
+	};
+	if (!checkKey(key)) {
+		return;
 	}
 	auto tagsData = LangPack();
 	auto entry = LangPack::Entry();
 	entry.key = key;
 	entry.value = extractTagsData(value, &tagsData);
 	entry.tags = tagsData.tags;
-	result_.entries.push_back(entry);
-	for (auto &pluralEntry : tagsData.entries) {
-		auto taggedEntry = LangPack::Entry();
-		taggedEntry.key = key + "__" + pluralEntry.key;
-		taggedEntry.value = pluralEntry.value;
-		result_.entries.push_back(taggedEntry);
+	if (pluralIndex >= 0) {
+		logAssert(tagsData.entries.empty());
+
+		entry.keyBase = entry.key;
+		entry.key = ComputePluralKey(entry.keyBase, pluralIndex);
+		if (!checkKey(entry.key)) {
+			return;
+		}
+		auto baseIndex = -1;
+		auto alreadyCount = result_.entries.size();
+		for (auto i = 0; i != alreadyCount; ++i) {
+			if (result_.entries[i].keyBase == entry.keyBase) {
+				// This is not the first appearance of this plural key.
+				baseIndex = i;
+				break;
+			}
+		}
+		if (baseIndex < 0) {
+			baseIndex = result_.entries.size();
+			for (auto i = 0; i != kPluralPartCount; ++i) {
+				auto addingEntry = LangPack::Entry();
+				addingEntry.keyBase = entry.keyBase;
+				addingEntry.key = ComputePluralKey(entry.keyBase, i);
+				result_.entries.push_back(addingEntry);
+			}
+		}
+		auto entryIndex = baseIndex + pluralIndex;
+		logAssert(entryIndex < result_.entries.size());
+		auto &realEntry = result_.entries[entryIndex];
+		logAssert(realEntry.key == entry.key);
+		realEntry.value = entry.value;
+
+		// Add all new tags to the existing ones.
+		realEntry.tags = std::vector<LangPack::Tag>(1, LangPack::Tag { kPluralTag });
+		for (auto &tag : entry.tags) {
+			if (std::find(realEntry.tags.begin(), realEntry.tags.end(), tag) == realEntry.tags.end()) {
+				realEntry.tags.push_back(tag);
+			}
+		}
+	} else {
+		result_.entries.push_back(entry);
+		for (auto &pluralEntry : tagsData.entries) {
+			auto taggedEntry = LangPack::Entry();
+			taggedEntry.key = key + "__" + pluralEntry.key;
+			taggedEntry.value = pluralEntry.value;
+			result_.entries.push_back(taggedEntry);
+		}
 	}
 }
 

Telegram/SourceFiles/codegen/lang/parsed_file.h

@@ -22,6 +22,7 @@ Copyright (c) 2014-2017 John Preston, https://desktop.telegram.org
 
 #include <memory>
 #include <string>
+#include <array>
 #include <functional>
 #include <QImage>
 #include "codegen/common/basic_tokenized_file.h"
@@ -30,6 +31,11 @@ Copyright (c) 2014-2017 John Preston, https://desktop.telegram.org
 namespace codegen {
 namespace lang {
 
+constexpr auto kPluralPartCount = 6;
+extern const std::array<QString, kPluralPartCount> kPluralParts;
+extern const QString kPluralTag;
+QString ComputePluralKey(const QString &base, int index);
+
 struct LangPack {
 	struct Tag {
 		QString tag;
@@ -37,6 +43,7 @@ struct LangPack {
 	struct Entry {
 		QString key;
 		QString value;
+		QString keyBase; // Empty for not plural entries.
 		std::vector<Tag> tags;
 	};
 	std::vector<Entry> entries;
@@ -44,6 +51,14 @@ struct LangPack {
 
 };
 
+inline bool operator==(const LangPack::Tag &a, const LangPack::Tag &b) {
+	return a.tag == b.tag;
+}
+
+inline bool operator!=(const LangPack::Tag &a, const LangPack::Tag &b) {
+	return !(a == b);
+}
+
 // Parses an input file to the internal struct.
 class ParsedFile {
 public:
@@ -83,10 +98,12 @@ private:
 	using BasicToken = common::BasicTokenizedFile::Token;
 	BasicToken assertNextToken(BasicToken::Type type);
 
-	void addEntity(const QString &key, const QString &value);
+	void addEntity(QString key, const QString &value);
 	QString extractTagsData(const QString &value, LangPack *to);
 	QString extractTagData(const QString &tag, LangPack *to);
 
+	void fillPluralTags();
+
 	QString filePath_;
 	common::BasicTokenizedFile file_;
 	Options options_;