[Core] Language system

Telegram Desktop uses static language system with code generation, which
requires almost full app rebuild on any string modification, no matter
how small it is.

Since Kotatogram is options-driven, static language system will slow the
development. This language system solves the problem by using JSON and
runtime string search instead.

Telegram/CMakeLists.txt

@@ -987,6 +987,8 @@ PRIVATE
     intro/intro_step.h
     intro/intro_widget.cpp
     intro/intro_widget.h
+    kotato/kotato_lang.cpp
+    kotato/kotato_lang.h
     lang/lang_cloud_manager.cpp
     lang/lang_cloud_manager.h
     lang/lang_instance.cpp

Telegram/Resources/langs/rewrites/en.json

@@ -0,0 +1,3 @@
+{
+	"dummy_last_string": ""
+}

Telegram/Resources/qrc/telegram/telegram.qrc

@@ -59,4 +59,7 @@
     <file alias="default_shortcuts-custom.json">../../default_shortcuts-custom.json</file>
     <file alias="io.github.kotatogram.desktop">../../../../lib/xdg/io.github.kotatogram.desktop</file>
   </qresource>
+  <qresource prefix="/ktg_lang">
+    <file alias="en.json">../../langs/rewrites/en.json</file>
+  </qresource>
 </RCC>

Telegram/SourceFiles/core/application.cpp

@@ -7,6 +7,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 */
 #include "core/application.h"
 
+#include "kotato/kotato_lang.h"
 #include "data/data_abstract_structure.h"
 #include "data/data_photo.h"
 #include "data/data_document.h"
@@ -254,6 +255,7 @@ void Application::run() {
 
 	startLocalStorage();
 	ValidateScale();
+	Kotato::Lang::Load(Lang::GetInstance().baseId(), Lang::GetInstance().id());
 
 	refreshGlobalProxy(); // Depends on app settings being read.
 

Telegram/SourceFiles/kotato/kotato_lang.cpp

@@ -0,0 +1,294 @@
+/*
+This file is part of Kotatogram Desktop,
+the unofficial app based on Telegram Desktop.
+
+For license and copyright information please follow this link:
+https://github.com/kotatogram/kotatogram-desktop/blob/dev/LEGAL
+*/
+#include "kotato/kotato_lang.h"
+
+#include "base/parse_helper.h"
+#include "lang/lang_tag.h"
+
+#include <QtCore/QJsonDocument>
+#include <QtCore/QJsonObject>
+#include <QtCore/QJsonArray>
+#include <QtCore/QDir>
+
+namespace Kotato {
+namespace Lang {
+namespace {
+
+const auto kDefaultLanguage = qsl("en");
+const std::vector<QString> kPostfixes = {
+	"#zero",
+	"#one",
+	"#two",
+	"#few",
+	"#many",
+	"#other"
+};
+
+QString BaseLangCode;
+QString LangCode;
+
+QMap<QString, QString> DefaultValues;
+QMap<QString, QString> CurrentValues;
+
+rpl::event_stream<> LangChanges;
+
+QString LangDir() {
+	return cWorkingDir() + "tdata/ktg_lang/";
+}
+
+void ParseLanguageData(
+	const QString &langCode,
+	bool isDefault) {
+	const auto filename = isDefault
+		? qsl(":/ktg_lang/%1.json").arg(langCode)
+		: LangDir() + (qsl("%1.json").arg(langCode));
+
+	QFile file(filename);
+	if (!file.exists()) {
+		return;
+	}
+	if (!file.open(QIODevice::ReadOnly)) {
+		LOG(("Kotato::Lang Info: file %1 could not be read.").arg(filename));
+		return;
+	}
+	auto error = QJsonParseError{ 0, QJsonParseError::NoError };
+	const auto document = QJsonDocument::fromJson(
+		base::parse::stripComments(file.readAll()),
+		&error);
+	file.close();
+
+	if (error.error != QJsonParseError::NoError) {
+		LOG(("Kotato::Lang Info: file %1 has failed to parse. Error: %2"
+			).arg(filename
+			).arg(error.errorString()));
+		return;
+	} else if (!document.isObject()) {
+		LOG(("Kotato::Lang Info: file %1 has failed to parse. Error: object expected"
+			).arg(filename));
+		return;
+	}
+
+	const auto applyValue = [&](const QString &name, const QString &translation) {
+		if (langCode == kDefaultLanguage) {
+			DefaultValues.insert(name, translation);
+		} else {
+			CurrentValues.insert(name, translation);
+		}
+	};
+
+	const auto langKeys = document.object();
+	const auto keyList = langKeys.keys();
+
+	for (auto i = keyList.constBegin(), e = keyList.constEnd(); i != e; ++i) {
+		const auto key = *i;
+		if (key.startsWith("dummy_")) {
+			continue;
+		}
+
+		const auto value = langKeys.constFind(key);
+
+		if ((*value).isString()) {
+
+			applyValue(key, (*value).toString());
+
+		} else if ((*value).isObject()) {
+
+			const auto keyPlurals = (*value).toObject();
+			const auto pluralList = keyPlurals.keys();
+
+			for (auto pli = pluralList.constBegin(), ple = pluralList.constEnd(); pli != ple; ++pli) {
+				const auto plural = *pli;
+				const auto pluralValue = keyPlurals.constFind(plural);
+
+				if (!(*pluralValue).isString()) {
+					LOG(("Kotato::Lang Info: wrong value for key %1 in %2 in file %3, string expected")
+						.arg(plural).arg(key).arg(filename));
+					continue;
+				}
+
+				const auto name = QString(key + "#" + plural);
+				const auto translation = (*pluralValue).toString();
+
+				applyValue(name, translation);
+			}
+		} else {
+			LOG(("Kotato::Lang Info: wrong value for key %1 in file %2, string or object expected")
+				.arg(key).arg(filename));
+		}
+	}
+}
+
+void UnpackDefault() {
+	const auto langDir = LangDir();
+	if (!QDir().exists(langDir)) QDir().mkpath(langDir);
+
+	const auto langs = QDir(":/ktg_lang").entryList(QStringList() << "*.json", QDir::Files);
+	auto neededLangs = QStringList() << kDefaultLanguage << LangCode << BaseLangCode;
+	neededLangs.removeDuplicates();
+
+	for (auto language : langs) {
+		language.chop(5);
+		if (!neededLangs.contains(language)) {
+			continue;
+		}
+
+		const auto path = langDir + language + ".default.json";
+		auto input = QFile(qsl(":/ktg_lang/%1.json").arg(language));
+		auto output = QFile(path);
+		if (input.open(QIODevice::ReadOnly)) {
+			auto inputData = input.readAll();
+			if (output.open(QIODevice::WriteOnly)) {
+				output.write(inputData);
+				output.close();
+			}
+			input.close();
+		}
+	}
+}
+
+} // namespace
+
+void Load(const QString &baseLangCode, const QString &langCode) {
+	BaseLangCode = baseLangCode;
+	if (BaseLangCode.endsWith("-raw")) {
+		BaseLangCode.chop(4);
+	}
+
+	LangCode = langCode.isEmpty() ? baseLangCode : langCode;
+	if (LangCode.endsWith("-raw")) {
+		LangCode.chop(4);
+	}
+
+	DefaultValues.clear();
+	CurrentValues.clear();
+
+	if (BaseLangCode != kDefaultLanguage) {
+		ParseLanguageData(kDefaultLanguage, true);
+		ParseLanguageData(kDefaultLanguage, false);
+	}
+
+	ParseLanguageData(BaseLangCode, true);
+	ParseLanguageData(BaseLangCode, false);
+
+	if (LangCode != BaseLangCode) {
+		ParseLanguageData(LangCode, true);
+		ParseLanguageData(LangCode, false);
+	}
+
+	UnpackDefault();
+	LangChanges.fire({});
+}
+
+QString Translate(const QString &key, Var var1, Var var2, Var var3, Var var4) {
+	auto phrase = (CurrentValues.contains(key) && !CurrentValues.value(key).isEmpty())
+		? CurrentValues.value(key)
+		: DefaultValues.value(key);
+
+	for (const auto &v : { var1, var2, var3, var4 }) {
+		if (!v.key.isEmpty()) {
+			auto skipNext = false;
+			const auto key = qsl("{%1}").arg(v.key);
+			const auto neededLength = phrase.length() - key.length();
+			for (auto i = 0; i <= neededLength; i++) {
+				if (skipNext) {
+					skipNext = false;
+					continue;
+				}
+				
+				if (phrase.at(i) == QChar('\\')) {
+					skipNext = true;
+				} else if (phrase.at(i) == QChar('{') && phrase.mid(i, key.length()) == key) {
+					phrase.replace(i, key.length(), v.value);
+					break;
+				}
+			}
+		}
+	}
+
+	return phrase;
+}
+
+QString Translate(const QString &key, float64 value, Var var1, Var var2, Var var3, Var var4) {
+	const auto shift = ::Lang::PluralShift(value);
+	return Translate(key + kPostfixes.at(shift), var1, var2, var3);
+}
+
+TextWithEntities TranslateWithEntities(const QString &key, EntVar var1, EntVar var2, EntVar var3, EntVar var4) {
+	TextWithEntities phrase = {
+		(CurrentValues.contains(key) && !CurrentValues.value(key).isEmpty())
+			? CurrentValues.value(key)
+			: DefaultValues.value(key)
+	};
+
+	for (const auto &v : { var1, var2, var3, var4 }) {
+		if (!v.key.isEmpty()) {
+			auto skipNext = false;
+			const auto key = qsl("{%1}").arg(v.key);
+			const auto neededLength = phrase.text.length() - key.length();
+			for (auto i = 0; i <= neededLength; i++) {
+				if (skipNext) {
+					skipNext = false;
+					continue;
+				}
+				
+				if (phrase.text.at(i) == QChar('\\')) {
+					skipNext = true;
+				} else if (phrase.text.at(i) == QChar('{') && phrase.text.mid(i, key.length()) == key) {
+					phrase.text.replace(i, key.length(), v.value.text);
+					const auto endOld = i + key.length();
+					const auto endNew = i + v.value.text.length();
+
+					// Shift existing entities
+					if (endNew > endOld) {
+						const auto diff = endNew - endOld;
+						for (auto &entity : phrase.entities) {
+							if (entity.offset() > endOld) {
+								entity.shiftRight(diff);
+							} else if (entity.offset() <= i && entity.offset() + entity.length() >= endOld) {
+								entity.extendToRight(diff);
+							}
+						}
+					} else if (endNew < endOld) {
+						const auto diff = endOld - endNew;
+						for (auto &entity : phrase.entities) {
+							if (entity.offset() > endNew) {
+								entity.shiftLeft(diff);
+							} else if (entity.offset() <= i && entity.offset() + entity.length() >= endNew) {
+								entity.shrinkFromRight(diff);
+							}
+						}
+					}
+
+					// Add new entities
+					for (auto entity : v.value.entities) {
+						phrase.entities.append(EntityInText(
+							entity.type(),
+							entity.offset() + i,
+							entity.length(),
+							entity.data()));
+					}
+					break;
+				}
+			}
+		}
+	}
+
+	return phrase;
+}
+
+TextWithEntities TranslateWithEntities(const QString &key, float64 value, EntVar var1, EntVar var2, EntVar var3, EntVar var4) {
+	const auto shift = ::Lang::PluralShift(value);
+	return TranslateWithEntities(key + kPostfixes.at(shift), var1, var2, var3, var4);
+}
+
+rpl::producer<> Events() {
+	return LangChanges.events();
+}
+
+} // namespace Lang
+} // namespace Kotato

Telegram/SourceFiles/kotato/kotato_lang.h

@@ -0,0 +1,166 @@
+/*
+This file is part of Kotatogram Desktop,
+the unofficial app based on Telegram Desktop.
+
+For license and copyright information please follow this link:
+https://github.com/kotatogram/kotatogram-desktop/blob/dev/LEGAL
+*/
+#pragma once
+
+namespace Kotato {
+namespace Lang {
+
+struct Var {
+	Var() {};
+	Var(const QString &k, const QString &v) {
+		key = k;
+		value = v;
+	}
+
+	QString key;
+	QString value;
+};
+
+struct EntVar {
+	EntVar() {};
+	EntVar(const QString &k, TextWithEntities v) {
+		key = k;
+		value = v;
+	}
+
+	QString key;
+	TextWithEntities value;
+};
+
+void Load(const QString &baseLangCode, const QString &langCode);
+
+QString Translate(
+	const QString &key,
+	Var var1 = Var(),
+	Var var2 = Var(),
+	Var var3 = Var(),
+	Var var4 = Var());
+QString Translate(
+	const QString &key,
+	float64 value,
+	Var var1 = Var(),
+	Var var2 = Var(),
+	Var var3 = Var(),
+	Var var4 = Var());
+
+TextWithEntities TranslateWithEntities(
+	const QString &key,
+	EntVar var1 = EntVar(),
+	EntVar var2 = EntVar(),
+	EntVar var3 = EntVar(),
+	EntVar var4 = EntVar());
+TextWithEntities TranslateWithEntities(
+	const QString &key,
+	float64 value,
+	EntVar var1 = EntVar(),
+	EntVar var2 = EntVar(),
+	EntVar var3 = EntVar(),
+	EntVar var4 = EntVar());
+
+rpl::producer<> Events();
+
+} // namespace Lang
+} // namespace Kotato
+
+// Shorthands
+
+inline QString ktr(
+	const QString &key,
+	::Kotato::Lang::Var var1 = ::Kotato::Lang::Var(),
+	::Kotato::Lang::Var var2 = ::Kotato::Lang::Var(),
+	::Kotato::Lang::Var var3 = ::Kotato::Lang::Var(),
+	::Kotato::Lang::Var var4 = ::Kotato::Lang::Var()) {
+	return ::Kotato::Lang::Translate(key, var1, var2, var3, var4);
+}
+
+inline QString ktr(
+	const QString &key,
+	float64 value,
+	::Kotato::Lang::Var var1 = ::Kotato::Lang::Var(),
+	::Kotato::Lang::Var var2 = ::Kotato::Lang::Var(),
+	::Kotato::Lang::Var var3 = ::Kotato::Lang::Var(),
+	::Kotato::Lang::Var var4 = ::Kotato::Lang::Var()) {
+	return ::Kotato::Lang::Translate(key, value, var1, var2, var3, var4);
+}
+
+inline TextWithEntities ktre(
+	const QString &key,
+	::Kotato::Lang::EntVar var1 = ::Kotato::Lang::EntVar(),
+	::Kotato::Lang::EntVar var2 = ::Kotato::Lang::EntVar(),
+	::Kotato::Lang::EntVar var3 = ::Kotato::Lang::EntVar(),
+	::Kotato::Lang::EntVar var4 = ::Kotato::Lang::EntVar()) {
+	return ::Kotato::Lang::TranslateWithEntities(key, var1, var2, var3, var4);
+}
+
+inline TextWithEntities ktre(
+	const QString &key,
+	float64 value,
+	::Kotato::Lang::EntVar var1 = ::Kotato::Lang::EntVar(),
+	::Kotato::Lang::EntVar var2 = ::Kotato::Lang::EntVar(),
+	::Kotato::Lang::EntVar var3 = ::Kotato::Lang::EntVar(),
+	::Kotato::Lang::EntVar var4 = ::Kotato::Lang::EntVar()) {
+	return ::Kotato::Lang::TranslateWithEntities(key, value, var1, var2, var3, var4);
+}
+
+inline rpl::producer<QString> rktr(
+	const QString &key,
+	::Kotato::Lang::Var var1 = ::Kotato::Lang::Var(),
+	::Kotato::Lang::Var var2 = ::Kotato::Lang::Var(),
+	::Kotato::Lang::Var var3 = ::Kotato::Lang::Var(),
+	::Kotato::Lang::Var var4 = ::Kotato::Lang::Var()) {
+	return rpl::single(
+			::Kotato::Lang::Translate(key, var1, var2, var3, var4)
+		) | rpl::then(
+			::Kotato::Lang::Events() | rpl::map(
+				[=]{ return ::Kotato::Lang::Translate(key, var1, var2, var3, var4); })
+		);
+}
+
+inline rpl::producer<QString> rktr(
+	const QString &key,
+	float64 value,
+	::Kotato::Lang::Var var1 = ::Kotato::Lang::Var(),
+	::Kotato::Lang::Var var2 = ::Kotato::Lang::Var(),
+	::Kotato::Lang::Var var3 = ::Kotato::Lang::Var(),
+	::Kotato::Lang::Var var4 = ::Kotato::Lang::Var()) {
+	return rpl::single(
+			::Kotato::Lang::Translate(key, value, var1, var2, var3, var4)
+		) | rpl::then(
+			::Kotato::Lang::Events() | rpl::map(
+				[=]{ return ::Kotato::Lang::Translate(key, value, var1, var2, var3, var4); })
+		);
+}
+
+inline rpl::producer<TextWithEntities> rktre(
+	const QString &key,
+	::Kotato::Lang::EntVar var1 = ::Kotato::Lang::EntVar(),
+	::Kotato::Lang::EntVar var2 = ::Kotato::Lang::EntVar(),
+	::Kotato::Lang::EntVar var3 = ::Kotato::Lang::EntVar(),
+	::Kotato::Lang::EntVar var4 = ::Kotato::Lang::EntVar()) {
+	return rpl::single(
+			::Kotato::Lang::TranslateWithEntities(key, var1, var2, var3, var4)
+		) | rpl::then(
+			::Kotato::Lang::Events() | rpl::map(
+				[=]{ return ::Kotato::Lang::TranslateWithEntities(key, var1, var2, var3, var4); })
+		);
+}
+
+inline rpl::producer<TextWithEntities> rktre(
+	const QString &key,
+	float64 value,
+	::Kotato::Lang::EntVar var1 = ::Kotato::Lang::EntVar(),
+	::Kotato::Lang::EntVar var2 = ::Kotato::Lang::EntVar(),
+	::Kotato::Lang::EntVar var3 = ::Kotato::Lang::EntVar(),
+	::Kotato::Lang::EntVar var4 = ::Kotato::Lang::EntVar()) {
+	return rpl::single(
+			::Kotato::Lang::TranslateWithEntities(key, value, var1, var2, var3, var4)
+		) | rpl::then(
+			::Kotato::Lang::Events() | rpl::map(
+				[=]{ return ::Kotato::Lang::TranslateWithEntities(key, value, var1, var2, var3, var4); })
+		);
+}

Telegram/SourceFiles/lang/lang_cloud_manager.cpp

@@ -7,6 +7,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 */
 #include "lang/lang_cloud_manager.h"
 
+#include "kotato/kotato_lang.h"
 #include "lang/lang_instance.h"
 #include "lang/lang_file_parser.h"
 #include "lang/lang_text_entity.h"
@@ -447,6 +448,7 @@ void CloudManager::sendSwitchingToLanguageRequest() {
 		const auto finalize = [=] {
 			if (canApplyWithoutRestart(language.id)) {
 				performSwitchAndAddToRecent(language);
+				Kotato::Lang::Load(Lang::GetInstance().baseId(), Lang::GetInstance().id());
 			} else {
 				performSwitchAndRestart(language);
 			}
@@ -482,6 +484,7 @@ void CloudManager::switchToLanguage(const Language &data) {
 		performSwitchToCustom();
 	} else if (canApplyWithoutRestart(data.id)) {
 		performSwitchAndAddToRecent(data);
+		Kotato::Lang::Load(Lang::GetInstance().baseId(), Lang::GetInstance().id());
 	} else {
 		QVector<MTPstring> keys;
 		keys.reserve(3);
@@ -535,6 +538,7 @@ void CloudManager::performSwitchToCustom() {
 			}
 			if (canApplyWithoutRestart(u"#custom"_q)) {
 				_langpack.switchToCustomFile(filePath);
+				Kotato::Lang::Load(Lang::GetInstance().baseId(), Lang::GetInstance().id());
 			} else {
 				const auto values = loader.found();
 				const auto getValue = [&](ushort key) {

Telegram/SourceFiles/lang/lang_tag.cpp

@@ -987,6 +987,35 @@ PluralResult Plural(
 	return { shift, FormatDouble(value) };
 }
 
+ushort PluralShift(float64 value, bool isShortened) {
+	// To correctly select a shift for PluralType::Short
+	// we must first round the number.
+	const auto shortened = (isShortened)
+		? FormatCountToShort(qRound(value))
+		: ShortenedCount();
+
+	// Simplified.
+	const auto n = std::abs(shortened.number ? float64(shortened.number) : value);
+	const auto i = int(std::floor(n));
+	const auto integer = (int(std::ceil(n)) == i);
+	const auto formatted = integer ? QString() : FormatDouble(n);
+	const auto dot = formatted.indexOf('.');
+	const auto fraction = (dot >= 0) ? formatted.mid(dot + 1) : QString();
+	const auto v = fraction.size();
+	const auto w = v;
+	const auto f = NonZeroPartToInt(fraction);
+	const auto t = f;
+
+	const auto useNonDefaultPlural = (ChoosePlural != ChoosePluralDefault);
+	return (useNonDefaultPlural ? ChoosePlural : ChoosePluralDefault)(
+		(integer ? i : -1),
+		i,
+		v,
+		w,
+		f,
+		t);
+}
+
 void UpdatePluralRules(const QString &languageId) {
 	static auto kMap = GeneratePluralRulesMap();
 	auto parent = uint64(0);

Telegram/SourceFiles/lang/lang_tag.h

@@ -34,6 +34,7 @@ PluralResult Plural(
 	ushort keyBase,
 	float64 value,
 	lngtag_count type);
+ushort PluralShift(float64 value, bool isShortened = false);
 void UpdatePluralRules(const QString &languageId);
 
 template <typename ResultString>