codegen_style project started, basic (common) tokenizer done.

Telegram/SourceFiles/codegen/common/basic_tokenized_file.cpp

@@ -0,0 +1,271 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#include "codegen/common/basic_tokenized_file.h"
+
+#include "codegen/common/logging.h"
+#include "codegen/common/clean_file_reader.h"
+#include "codegen/common/checked_utf8_string.h"
+
+using Token = codegen::common::BasicTokenizedFile::Token;
+using Type = Token::Type;
+
+namespace codegen {
+namespace common {
+namespace {
+
+constexpr int kErrorUnterminatedStringLiteral = 201;
+constexpr int kErrorIncorrectUtf8String       = 202;
+constexpr int kErrorIncorrectToken            = 203;
+constexpr int kErrorUnexpectedToken           = 204;
+
+bool isDigitChar(char ch) {
+	return (ch >= '0') && (ch <= '9');
+}
+
+bool isNameChar(char ch) {
+	return isDigitChar(ch) || ((ch >= 'a') && (ch <= 'z')) || ((ch >= 'A') && (ch <= 'Z')) || (ch == '_');
+}
+
+bool isWhitespaceChar(char ch) {
+	return (ch == '\n' || ch == ' ' || ch == '\t');
+}
+
+Token invalidToken() {
+	return { Type::Invalid, QString(), ConstUtf8String(nullptr, 0) };
+}
+
+} // namespace
+
+BasicTokenizedFile::BasicTokenizedFile(const QString &filepath) : reader_(filepath) {
+}
+
+bool BasicTokenizedFile::putBack() {
+	if (currentToken_ > 0) {
+		--currentToken_;
+		return true;
+	}
+	return false;
+}
+
+Token BasicTokenizedFile::getAnyToken() {
+	if (currentToken_ >= tokens_.size()) {
+		if (readToken() == Type::Invalid) {
+			return invalidToken();
+		}
+	}
+	return tokens_.at(currentToken_++);
+}
+
+Token BasicTokenizedFile::getToken(Type typeCondition) {
+	if (auto token = getAnyToken()) {
+		if (token.type == typeCondition) {
+			return token;
+		}
+		putBack();
+	}
+	return invalidToken();
+}
+
+Type BasicTokenizedFile::readToken() {
+	auto result = readOneToken(StartWithWhitespace::Allow);
+
+	// Try to read double token.
+	if (result == Type::Int) {
+		if (readOneToken(StartWithWhitespace::Deny) == Type::Dot) {
+			// We got int and dot, so it is double already.
+			result = uniteLastTokens(Type::Double);
+
+			// Try to read one more int (after dot).
+			if (readOneToken(StartWithWhitespace::Deny) == Type::Int) {
+				result = uniteLastTokens(Type::Double);
+			}
+		}
+	} else if (result == Type::Dot) {
+		if (readOneToken(StartWithWhitespace::Deny) == Type::Int) {
+			//We got dot and int, so it is double.
+			result = uniteLastTokens(Type::Double);
+		}
+	}
+	return result;
+}
+
+Type BasicTokenizedFile::readOneToken(StartWithWhitespace condition) {
+	skipWhitespaces();
+	if (tokenStartWhitespace_ && condition == StartWithWhitespace::Deny) {
+		return Type::Invalid;
+	}
+	if (reader_.atEnd()) {
+		return Type::Invalid;
+	}
+
+	auto ch = reader_.currentChar();
+	if (ch == '"') {
+		return readString();
+	} else if (isNameChar(ch)) {
+		return readNameOrNumber();
+	}
+	return readSingleLetter();
+}
+
+Type BasicTokenizedFile::saveToken(Type type, const QString &value) {
+	ConstUtf8String original = { tokenStart_, reader_.currentPtr() };
+	tokens_.push_back({ type, value, original, tokenStartWhitespace_ });
+	return type;
+}
+
+Type BasicTokenizedFile::uniteLastTokens(Type type) {
+	auto size = tokens_.size();
+	if (size < 2) {
+		return Type::Invalid;
+	}
+
+	auto &token(tokens_[size - 2]);
+	auto originalFrom = token.original.data();
+	auto originalTill = tokens_.back().original.end();
+	token.type = type;
+	token.original = { originalFrom, originalTill };
+	token.value += tokens_.back().value;
+	tokens_.pop_back();
+	return type;
+}
+
+Type BasicTokenizedFile::readNameOrNumber() {
+	bool onlyDigits = true;
+	while (!reader_.atEnd()) {
+		if (!isDigitChar(reader_.currentChar())) {
+			onlyDigits = false;
+			break;
+		}
+		reader_.skipChar();
+	}
+	while (!reader_.atEnd()) {
+		if (!isNameChar(reader_.currentChar())) {
+			break;
+		}
+		reader_.skipChar();
+	}
+	return saveToken(onlyDigits ? Type::Int : Type::Name);
+}
+
+Type BasicTokenizedFile::readString() {
+	reader_.skipChar();
+	auto offset = reader_.currentPtr();
+
+	QByteArray value;
+	while (!reader_.atEnd()) {
+		auto ch = reader_.currentChar();
+		if (ch == '"') {
+			break;
+		}
+		if (ch == '\n') {
+			reader_.logError(kErrorUnterminatedStringLiteral, lineNumber_) << "unterminated string literal.";
+			failed_ = true;
+			return Type::Invalid;
+		}
+		if (ch == '\\') {
+			reader_.skipChar();
+			ch = reader_.currentChar();
+			if (reader_.atEnd() || ch == '\n') {
+				reader_.logError(kErrorUnterminatedStringLiteral, lineNumber_) << "unterminated string literal.";
+				failed_ = true;
+				return Type::Invalid;
+			}
+			if (reader_.currentPtr() > offset + 1) {
+				value.append(offset, reader_.currentPtr() - offset - 1);
+			}
+			offset = reader_.currentPtr() + 1;
+			if (ch == 'n') {
+				value.append('\n');
+			} else if (ch == 't') {
+				value.append('\t');
+			} else if (ch == '"') {
+				value.append('"');
+			} else if (ch == '\\') {
+				value.append('\\');
+			}
+		} else {
+			value.append(ch);
+		}
+		reader_.skipChar();
+	}
+	if (reader_.atEnd()) {
+		reader_.logError(kErrorUnterminatedStringLiteral, lineNumber_) << "unterminated string literal.";
+		failed_ = true;
+		return Type::Invalid;
+	}
+	CheckedUtf8String checked(value);
+	if (!checked.isValid()) {
+		reader_.logError(kErrorIncorrectUtf8String, lineNumber_) << "incorrect UTF-8 string literal.";
+		failed_ = true;
+		return Type::Invalid;
+	}
+	reader_.skipChar();
+	return saveToken(Type::String, checked.toString());
+}
+
+Type BasicTokenizedFile::readSingleLetter() {
+	auto type = singleLetterTokens_.value(reader_.currentChar(), Type::Invalid);
+	if (type == Type::Invalid) {
+		reader_.logError(kErrorIncorrectToken, lineNumber_) << "incorrect token '" << reader_.currentChar() << "'.";
+		return Type::Invalid;
+	}
+
+	reader_.skipChar();
+	return saveToken(type);
+}
+
+void BasicTokenizedFile::skipWhitespaces() {
+	if (reader_.atEnd()) return;
+
+	auto ch = reader_.currentChar();
+	tokenStartWhitespace_ = isWhitespaceChar(ch);
+	if (tokenStartWhitespace_) {
+		do {
+			if (ch == '\n') {
+				++lineNumber_;
+			}
+			reader_.skipChar();
+			ch = reader_.currentChar();
+		} while (!reader_.atEnd() && isWhitespaceChar(ch));
+	}
+	tokenStart_ = reader_.currentPtr();
+}
+
+LogStream BasicTokenizedFile::logError(int code) const {
+	return reader_.logError(code, lineNumber_);
+}
+
+LogStream BasicTokenizedFile::logErrorUnexpectedToken(const std::string &expected) const {
+	std::string expectedTail;
+	if (!expected.empty()) {
+		expectedTail = ", expected " + expected;
+	}
+	if (currentToken_ < tokens_.size()) {
+		auto token = tokens_.at(currentToken_).original.toStdString();
+		return logError(kErrorUnexpectedToken) << "unexpected token '" << token << '\'' << expectedTail << '.';
+	}
+	return logError(kErrorUnexpectedToken) << "unexpected token" << expectedTail << '.';
+}
+
+BasicTokenizedFile::~BasicTokenizedFile() = default;
+
+} // namespace common
+} // namespace codegen

Telegram/SourceFiles/codegen/common/basic_tokenized_file.h

@@ -0,0 +1,151 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#pragma once
+
+#include <memory>
+#include <QtCore/QString>
+#include <QtCore/QMap>
+#include <QtCore/QList>
+
+#include "codegen/common/const_utf8_string.h"
+#include "codegen/common/clean_file_reader.h"
+
+namespace codegen {
+namespace common {
+
+class LogStream;
+
+// Interface for reading a cleaned from comments file by basic tokens.
+class BasicTokenizedFile {
+public:
+	BasicTokenizedFile(const QString &filepath);
+	BasicTokenizedFile(const BasicTokenizedFile &other) = delete;
+	BasicTokenizedFile &operator=(const BasicTokenizedFile &other) = delete;
+
+	struct Token {
+		// String - utf8 string converted to QString.
+		enum class Type {
+			Invalid = 0,
+			Int,
+			Double,
+			String,
+			LeftParenthesis,
+			RightParenthesis,
+			LeftBrace,
+			RightBrace,
+			LeftBracket,
+			RightBracket,
+			Colon,
+			Semicolon,
+			Comma,
+			Dot,
+			Number,
+			Plus,
+			Minus,
+			Equals,
+			Name, // [0-9a-zA-Z_]+ with at least one letter.
+		};
+		Type type;
+		QString value;
+		ConstUtf8String original;
+		bool hasLeftWhitespace;
+
+		explicit operator bool() const {
+			return (type != Type::Invalid);
+		}
+	};
+
+	bool read() {
+		return reader_.read();
+	}
+	bool atEnd() const {
+		return reader_.atEnd();
+	}
+
+	Token getAnyToken();
+	Token getToken(Token::Type typeCondition);
+	bool putBack();
+	bool failed() const {
+		return failed_;
+	}
+
+	// Log error to std::cerr with 'code' at the current position in file.
+	LogStream logError(int code) const;
+	LogStream logErrorUnexpectedToken(const std::string &expected = std::string()) const;
+
+	~BasicTokenizedFile();
+
+private:
+	using Type = Token::Type;
+
+	void skipWhitespaces();
+
+	// Reads a token, including complex tokens, like double numbers.
+	Type readToken();
+
+	// Read exactly one token, applying condition on the whitespaces.
+	enum class StartWithWhitespace {
+		Allow,
+		Deny,
+	};
+	Type readOneToken(StartWithWhitespace condition);
+
+	// helpers
+	Type readNameOrNumber();
+	Type readString();
+	Type readSingleLetter();
+
+	Type saveToken(Type type, const QString &value = QString());
+	Type uniteLastTokens(Type type);
+
+	CleanFileReader reader_;
+	QList<Token> tokens_;
+	int currentToken_ = 0;
+	int lineNumber_ = 1;
+	bool failed_ = false;
+
+	// Where the last (currently read) token has started.
+	const char *tokenStart_ = nullptr;
+
+	// Did the last (currently read) token start with a whitespace.
+	bool tokenStartWhitespace_ = false;
+
+	const QMap<char, Type> singleLetterTokens_ = {
+		{ '(', Type::LeftParenthesis },
+		{ ')', Type::RightParenthesis },
+		{ '{', Type::LeftBrace },
+		{ '}', Type::RightBrace },
+		{ '[', Type::LeftBracket },
+		{ ']', Type::RightBracket },
+		{ ':', Type::Colon },
+		{ ';', Type::Semicolon },
+		{ ',', Type::Comma },
+		{ '.', Type::Dot },
+		{ '#', Type::Number },
+		{ '+', Type::Plus },
+		{ '-', Type::Minus },
+		{ '=', Type::Equals },
+	};
+
+};
+
+} // namespace common
+} // namespace codegen

Telegram/SourceFiles/codegen/common/checked_utf8_string.cpp

@@ -0,0 +1,54 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#include "codegen/common/checked_utf8_string.h"
+
+#include <iostream>
+#include <QtCore/QTextCodec>
+
+#include "codegen/common/const_utf8_string.h"
+
+namespace codegen {
+namespace common {
+
+CheckedUtf8String::CheckedUtf8String(const char *string, int size) {
+	if (size < 0) {
+		size = strlen(string);
+	}
+	if (!size) { // Valid empty string
+		return;
+	}
+
+	QTextCodec::ConverterState state;
+	QTextCodec *codec = QTextCodec::codecForName("UTF-8");
+	string_ = codec->toUnicode(string, size, &state);
+	if (state.invalidChars > 0) {
+		valid_ = false;
+	}
+}
+
+CheckedUtf8String::CheckedUtf8String(const QByteArray &string) : CheckedUtf8String(string.constData(), string.size()) {
+}
+
+CheckedUtf8String::CheckedUtf8String(const ConstUtf8String &string) : CheckedUtf8String(string.data(), string.size()) {
+}
+
+} // namespace common
+} // namespace codegen

Telegram/SourceFiles/codegen/common/checked_utf8_string.h

@@ -0,0 +1,57 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#pragma once
+
+#include <QtCore/QString>
+
+class QByteArray;
+
+namespace codegen {
+namespace common {
+
+class ConstUtf8String;
+
+// Parses a char sequence to a QString using UTF-8 codec.
+// You can check for invalid UTF-8 sequence by isValid() method.
+class CheckedUtf8String {
+public:
+	CheckedUtf8String(const CheckedUtf8String &other) = default;
+	CheckedUtf8String &operator=(const CheckedUtf8String &other) = default;
+
+	CheckedUtf8String(const char *string, int size = -1);
+	CheckedUtf8String(const QByteArray &string);
+	CheckedUtf8String(const ConstUtf8String &string);
+
+	bool isValid() const {
+		return valid_;
+	}
+	const QString &toString() const {
+		return string_;
+	}
+
+private:
+	QString string_;
+	bool valid_ = true;
+
+};
+
+} // namespace common
+} // namespace codegen

Telegram/SourceFiles/codegen/common/clean_file.cpp

@@ -0,0 +1,159 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#include "codegen/common/clean_file.h"
+
+#include <iostream>
+#include <QtCore/QDir>
+
+#include "codegen/common/logging.h"
+
+namespace codegen {
+namespace common {
+namespace {
+
+constexpr int kErrorFileNotFound        = 101;
+constexpr int kErrorFileTooLarge        = 102;
+constexpr int kErrorFileNotOpened       = 103;
+constexpr int kErrorUnexpectedEndOfFile = 104;
+
+bool readFile(const QString &filepath, QByteArray *outResult) {
+	QFile f(filepath);
+	if (!f.exists()) {
+		logError(kErrorFileNotFound, filepath) << ": error: file does not exist.";
+		return false;
+	}
+	if (f.size() > CleanFile::MaxSize) {
+		logError(kErrorFileTooLarge, filepath) << "' is too large, size=" << f.size() << " > maxsize=" << CleanFile::MaxSize;
+		return false;
+	}
+	if (!f.open(QIODevice::ReadOnly)) {
+		logError(kErrorFileNotOpened, filepath) << "' for read.";
+		return false;
+	}
+	*outResult = f.readAll();
+	return true;
+}
+
+} // namespace
+
+
+CleanFile::CleanFile(const QString &filepath) : filepath_(filepath) {
+}
+
+bool CleanFile::read() {
+	QByteArray content;
+	if (!readFile(filepath_, &content)) {
+		return false;
+	}
+	filepath_ = QFileInfo(filepath_).absoluteFilePath();
+
+	enum class InsideComment {
+		None,
+		SingleLine,
+		MultiLine,
+	};
+	auto insideComment = InsideComment::None;
+	bool insideString = false;
+
+	const char *begin = content.cbegin(), *end = content.cend(), *offset = begin;
+	auto feedContent = [this, &offset, end](const char *ch) {
+		if (ch > offset) {
+			if (content_.isEmpty()) content_.reserve(end - offset - 2);
+			content_.append(offset, ch - offset);
+			offset = ch;
+		}
+	};
+	auto feedComment = [this, &offset, end](const char *ch) {
+		if (ch > offset) {
+//			comments_.push_back({ content_.size(), QByteArray(offset, ch - offset) });
+			if (content_.isEmpty()) content_.reserve(end - offset - 2);
+			content_.append(' ');
+			offset = ch;
+		}
+	};
+	for (const char *ch = offset; ch != end;) {
+		char currentChar = *ch;
+		char nextChar = (ch + 1 == end) ? 0 : *(ch + 1);
+
+		if (insideComment == InsideComment::None && currentChar == '"') {
+			bool escaped = ((ch > begin) && *(ch - 1) == '\\') && ((ch - 1 < begin) || *(ch - 2) != '\\');
+			if (!escaped) {
+				insideString = !insideString;
+			}
+		}
+		if (insideString) {
+			++ch;
+			continue;
+		}
+
+		if (insideComment == InsideComment::None && currentChar == '/' && nextChar == '/') {
+			feedContent(ch);
+			insideComment = InsideComment::SingleLine;
+			ch += 2;
+		} else if (insideComment == InsideComment::SingleLine && currentChar == '\r' && nextChar == '\n') {
+			feedComment(ch);
+			ch += 2;
+			insideComment = InsideComment::None;
+		} else if (insideComment == InsideComment::SingleLine && currentChar == '\n') {
+			feedComment(ch);
+			++ch;
+			insideComment = InsideComment::None;
+		} else if (insideComment == InsideComment::None && currentChar == '/' && nextChar == '*') {
+			feedContent(ch);
+			ch += 2;
+			insideComment = InsideComment::MultiLine;
+		} else if (insideComment == InsideComment::MultiLine && currentChar == '*' && nextChar == '/') {
+			ch += 2;
+			feedComment(ch);
+			insideComment = InsideComment::None;
+		} else if (insideComment == InsideComment::MultiLine && currentChar == '\r' && nextChar == '\n') {
+			feedComment(ch);
+			ch += 2;
+			feedContent(ch);
+		} else if (insideComment == InsideComment::MultiLine && currentChar == '\n') {
+			feedComment(ch);
+			++ch;
+			feedContent(ch);
+		} else {
+			++ch;
+		}
+	}
+
+	if (insideComment == InsideComment::MultiLine) {
+		common::logError(kErrorUnexpectedEndOfFile, filepath_);
+		return false;
+	}
+	if (insideComment == InsideComment::None && end > offset) {
+		if (content_.isEmpty()) {
+			content_ = content;
+		} else {
+			content_.append(offset, end - offset);
+		}
+	}
+	return true;
+}
+
+LogStream CleanFile::logError(int code, int line) const {
+	return common::logError(code, filepath_, line);
+}
+
+} // namespace common
+} // namespace codegen

Telegram/SourceFiles/codegen/common/clean_file.h

@@ -0,0 +1,66 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#pragma once
+
+#include <QtCore/QString>
+#include <QtCore/QByteArray>
+#include <QtCore/QVector>
+
+#include "codegen/common/logging.h"
+
+namespace codegen {
+namespace common {
+
+// Reads a file removing all C-style comments.
+class CleanFile {
+public:
+	CleanFile(const QString &filepath);
+	CleanFile(const CleanFile &other) = delete;
+	CleanFile &operator=(const CleanFile &other) = delete;
+
+	bool read();
+
+	const char *data() const {
+		return content_.constData();
+	}
+	const char *end() const {
+		return content_.constEnd();
+	}
+
+	static constexpr int MaxSize = 10 * 1024 * 1024;
+
+	// Log error to std::cerr with 'code' at line number 'line' in data().
+	LogStream logError(int code, int line) const;
+
+private:
+	QString filepath_;
+
+	QByteArray content_;
+	//struct Comment {
+	//	int offset;
+	//	QByteArray content;
+	//};
+	//QVector<Comment> comments_;
+
+};
+
+} // namespace common
+} // namespace codegen

Telegram/SourceFiles/codegen/common/clean_file_reader.h

@@ -0,0 +1,75 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#pragma once
+
+#include <QtCore/QString>
+
+#include "codegen/common/clean_file.h"
+
+namespace codegen {
+namespace common {
+
+// Wrapper allows you to read forward the CleanFile without overflow checks.
+class CleanFileReader {
+public:
+	CleanFileReader(const QString &filepath) : file_(filepath) {
+	}
+
+	bool read() {
+		if (!file_.read()) {
+			return false;
+		}
+		pos_ = file_.data();
+		end_ = file_.end();
+		return true;
+	}
+	bool atEnd() const {
+		return (pos_ == end_);
+	}
+	char currentChar() const {
+		return atEnd() ? 0 : *pos_;
+	}
+	bool skipChar() {
+		if (atEnd()) {
+			return false;
+		}
+		++pos_;
+		return true;
+	}
+	const char *currentPtr() const {
+		return pos_;
+	}
+
+	// Log error to std::cerr with 'code' at line number 'line' in data().
+	LogStream logError(int code, int line) const {
+		return std::forward<LogStream>(file_.logError(code, line));
+	}
+
+
+private:
+	CleanFile file_;
+	const char *pos_ = nullptr;
+	const char *end_ = nullptr;
+
+};
+
+} // namespace common
+} // namespace codegen

Telegram/SourceFiles/codegen/common/const_utf8_string.h

@@ -0,0 +1,75 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#pragma once
+
+#include <string>
+#include <QtCore/QString>
+#include <QtCore/QByteArray>
+
+namespace codegen {
+namespace common {
+
+// This is a simple wrapper around (const char*, size).
+// Not null-terminated! It does not hold any ownership.
+class ConstUtf8String {
+public:
+	ConstUtf8String(const char *string, int size = -1) : string_(string) {
+		if (size < 0) {
+			size = strlen(string);
+		}
+		size_ = size;
+	}
+	ConstUtf8String(const char *string, const char *end) : ConstUtf8String(string, end - string) {
+	}
+
+	QByteArray toByteArray() const {
+		return QByteArray(string_, size_);
+	}
+	std::string toStdString() const {
+		return std::string(string_, size_);
+	}
+	QString toStringUnchecked() const {
+		return QString::fromUtf8(string_, size_);
+	}
+	bool empty() const {
+		return size_ == 0;
+	}
+	const char *data() const {
+		return string_;
+	}
+	int size() const {
+		return size_;
+	}
+	const char *end() const {
+		return data() + size();
+	}
+	ConstUtf8String mid(int pos, int size = -1) {
+		return ConstUtf8String(string_ + pos, std::max(std::min(size, size_ - pos), 0));
+	}
+
+private:
+	const char *string_;
+	int size_;
+
+};
+
+} // namespace common
+} // namespace codegen

Telegram/SourceFiles/codegen/common/logging.cpp

@@ -0,0 +1,47 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#include "codegen/common/logging.h"
+
+#include <iostream>
+#include <QtCore/QDir>
+
+namespace codegen {
+namespace common {
+namespace {
+
+std::string relativeLocalPath(const QString &filepath) {
+	auto name = QFile::encodeName(QDir().relativeFilePath(filepath));
+	return name.constData();
+}
+
+} // namespace
+
+LogStream logError(int code, const QString &filepath, int line) {
+	std::cerr << relativeLocalPath(filepath);
+	if (line > 0) {
+		std::cerr << '(' << line << ')';
+	}
+	std::cerr << ": error " << code << ": ";
+	return LogStream(std::cerr);
+}
+
+} // namespace common
+} // namespace codegen

Telegram/SourceFiles/codegen/common/logging.h

@@ -0,0 +1,63 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#pragma once
+
+#include <QtCore/QString>
+#include <iostream>
+
+namespace codegen {
+namespace common {
+
+// Wrapper around std::ostream that adds '\n' to the end of the logging line.
+class LogStream {
+public:
+	explicit LogStream(std::ostream &stream) : stream_(stream) {
+	}
+	LogStream(LogStream &&other) : stream_(other.stream_) {
+		other.final_ = false;
+	}
+	std::ostream &stream() const {
+		return stream_;
+	}
+	~LogStream() {
+		if (final_) {
+			stream_ << '\n';
+		}
+	}
+
+private:
+	std::ostream &stream_;
+	bool final_ = true;
+
+};
+
+template <typename T>
+LogStream operator<<(LogStream &&stream, T &&value) {
+	stream.stream() << value;
+	return std::forward<LogStream>(stream);
+}
+
+// Outputs file name, line number and error code to std::err. Usage:
+// logError(kErrorFileTooLarge, filepath) << "file too large, size=" << size;
+LogStream logError(int code, const QString &filepath, int line = 0);
+
+} // namespace common
+} // namespace codegen

Telegram/SourceFiles/codegen/style/generator.cpp

@@ -0,0 +1,62 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#include "codegen/style/generator.h"
+
+#include <QtGui/QImage>
+#include "codegen/style/tokenized_file.h"
+
+using Token = codegen::style::TokenizedFile::Token;
+using Type = Token::Type;
+
+namespace codegen {
+namespace style {
+namespace {
+
+} // namespace
+
+Generator::Generator(const QString &filepath, bool rebuildDependencies)
+: file_(std::make_unique<TokenizedFile>(filepath))
+, rebuild_(rebuildDependencies) {
+
+}
+
+int Generator::process() {
+	if (!file_->read()) {
+		return -1;
+	}
+
+	while (true) {
+		auto token = file_->getToken();
+		if (token.type == Type::Using) {
+			continue;
+		}
+		if (file_->atEnd() && !file_->failed()) {
+			break;
+		}
+		return -1;
+	}
+	return 0;
+}
+
+Generator::~Generator() = default;
+
+} // namespace style
+} // namespace codegen

Telegram/SourceFiles/codegen/style/generator.h

@@ -0,0 +1,50 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#pragma once
+
+#include <memory>
+#include <QtCore/QString>
+
+namespace codegen {
+namespace style {
+
+class TokenizedFile;
+
+// Walks through a file, parses it and parses dependency files if necessary.
+class Generator {
+public:
+	Generator(const QString &filepath, bool rebuildDependencies);
+	Generator(const Generator &other) = delete;
+	Generator &operator=(const Generator &other) = delete;
+
+	// Returns 0 on success.
+	int process();
+
+	~Generator();
+
+private:
+	std::unique_ptr<TokenizedFile> file_;
+	bool rebuild_;
+
+};
+
+} // namespace style
+} // namespace codegen

Telegram/SourceFiles/codegen/style/main.cpp

@@ -1,3 +1,43 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#include <QtCore/QCoreApplication>
+#include <QtCore/QTimer>
+
+#include "codegen/style/generator.h"
+
+using namespace codegen::style;
+
 int main(int argc, char *argv[]) {
-	return 0;
+	QCoreApplication app(argc, argv);
+
+	QString filepath;
+	bool rebuildOtherFiles = false;
+	for (const auto &arg : app.arguments()) {
+		if (arg == "--rebuild") {
+			rebuildOtherFiles = true;
+		} else {
+			filepath = arg;
+		}
+	}
+
+	Generator generator(filepath, rebuildOtherFiles);
+	return generator.process();
 }

Telegram/SourceFiles/codegen/style/tokenized_file.cpp

@@ -0,0 +1,136 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#include "codegen/style/tokenized_file.h"
+
+#include <iostream>
+#include <QtCore/QDir>
+
+#include "codegen/common/basic_tokenized_file.h"
+#include "codegen/common/logging.h"
+
+using Token = codegen::style::TokenizedFile::Token;
+using Type = Token::Type;
+using BasicToken = codegen::common::BasicTokenizedFile::Token;
+using BasicType = BasicToken::Type;
+
+namespace codegen {
+namespace style {
+namespace {
+
+QString plainValue(const BasicToken &token) {
+	return token.original.toStringUnchecked();
+}
+
+Token invalidToken() {
+	return { Type::Invalid, QString() };
+}
+
+} // namespace
+
+TokenizedFile::TokenizedFile(const QString &filepath) : file_(filepath) {
+}
+
+bool TokenizedFile::putBack() {
+	if (currentToken_ > 0) {
+		--currentToken_;
+		return true;
+	}
+	return false;
+}
+
+Token TokenizedFile::getToken() {
+	if (currentToken_ >= tokens_.size()) {
+		if (readToken() == Type::Invalid) {
+			return invalidToken();
+		}
+	}
+	return tokens_.at(currentToken_++);
+}
+
+Type TokenizedFile::readToken() {
+	switch (state_) {
+	case State::Default: return readInDefault();
+	case State::StructStarted: return readInStructStarted();
+	case State::StructFieldName: return readInStructFieldName();
+	case State::Variable: return readInVariable();
+	case State::VariableParents: return readInVariableParents();
+	case State::VariableStarted: return readInVariableStarted();
+	case State::VariableChild: return readInVariableChild();
+	}
+	return Type::Invalid;
+}
+
+Type TokenizedFile::readInDefault() {
+	if (auto basicToken = file_.getToken(BasicType::Name)) {
+		if (plainValue(basicToken) == "using") {
+			if (auto usingFile = file_.getToken(BasicType::String)) {
+				if (file_.getToken(BasicType::Semicolon)) {
+					return saveToken(Type::Using, usingFile.value);
+				}
+				file_.logErrorUnexpectedToken("';'");
+			} else {
+				file_.logErrorUnexpectedToken("file path");
+			}
+			return Type::Invalid;
+		}
+		if (auto braceToken = file_.getToken(BasicType::LeftBrace)) {
+			state_ = State::StructStarted;
+			return saveToken(Type::DefineStruct, plainValue(basicToken));
+		} else if (auto colonToken = file_.getToken(BasicType::Colon)) {
+			state_ = State::Variable;
+			return saveToken(Type::DefineVariable, plainValue(basicToken));
+		}
+		file_.logErrorUnexpectedToken("using keyword, or struct definition, or variable definition");
+	}
+	return Type::Invalid;
+}
+
+Type TokenizedFile::readInStructStarted() {
+	return Type::Invalid;
+}
+
+Type TokenizedFile::readInStructFieldName() {
+	return Type::Invalid;
+}
+
+Type TokenizedFile::readInVariable() {
+	return Type::Invalid;
+}
+
+Type TokenizedFile::readInVariableParents() {
+	return Type::Invalid;
+}
+
+Type TokenizedFile::readInVariableStarted() {
+	return Type::Invalid;
+}
+
+Type TokenizedFile::readInVariableChild() {
+	return Type::Invalid;
+}
+
+Type TokenizedFile::saveToken(Type type, const QString &value) {
+	tokens_.push_back({ type, value });
+	return type;
+}
+
+} // namespace style
+} // namespace codegen

Telegram/SourceFiles/codegen/style/tokenized_file.h

@@ -0,0 +1,127 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop version of Telegram messaging app, see https://telegram.org
+
+Telegram Desktop is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+It is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+In addition, as a special exception, the copyright holders give permission
+to link the code of portions of this program with the OpenSSL library.
+
+Full license: https://github.com/telegramdesktop/tdesktop/blob/master/LICENSE
+Copyright (c) 2014-2016 John Preston, https://desktop.telegram.org
+*/
+#pragma once
+
+#include <memory>
+#include <QtCore/QString>
+#include <QtCore/QFileInfo>
+#include "codegen/common/basic_tokenized_file.h"
+
+namespace codegen {
+namespace style {
+
+// Parses a file as a list of tokens.
+class TokenizedFile {
+public:
+	TokenizedFile(const QString &filepath);
+	TokenizedFile(const TokenizedFile &other) = delete;
+	TokenizedFile &operator=(const TokenizedFile &other) = delete;
+
+	using ConstUtf8String = common::ConstUtf8String;
+	struct Token {
+		enum class Type {
+			Invalid,
+
+			Using,
+
+			DefineStruct,
+			DefineField,
+			FieldType,
+
+			DefineVariable,
+			Struct,
+			StructParent,
+
+			Int,
+			Double,
+			Pixels,
+			String,
+			Color,
+			Point,
+			Sprite,
+			Size,
+			Transition,
+			Cursor,
+			Align,
+			Margins,
+			Font,
+		};
+		Type type;
+		QString value;
+
+		explicit operator bool() const {
+			return type != Type::Invalid;
+		}
+	};
+
+	bool read() {
+		return file_.read();
+	}
+	bool atEnd() const {
+		return file_.atEnd();
+	}
+
+	Token getToken();
+	bool putBack();
+	bool failed() const {
+		return file_.failed();
+	}
+
+	// Log error to std::cerr with 'code' at the current position in file.
+	common::LogStream logError(int code) const {
+		return file_.logError(code);
+	}
+
+private:
+	using Type = Token::Type;
+	Type readToken();
+
+	// State value defines what are we waiting next.
+	enum class State {
+		Default, // [ using | struct name | variable name | end ]
+		StructStarted, // [ struct field name | struct end ]
+		StructFieldName, // [ struct field type ]
+		Variable, // [ struct name | variable value ]
+		VariableParents, // [ variable parent name | variable start ]
+		VariableStarted, // [ variable field name | variable end]
+		VariableChild, // [ variable child value ]
+	};
+
+	// Helper methods for readToken() being in specific State.
+	Type readInDefault();
+	Type readInStructStarted();
+	Type readInStructFieldName();
+	Type readInVariable();
+	Type readInVariableParents();
+	Type readInVariableStarted();
+	Type readInVariableChild();
+
+	Type saveToken(Type type, const QString &value = QString());
+
+	common::BasicTokenizedFile file_;
+	QList<Token> tokens_;
+	int currentToken_ = 0;
+	State state_ = State::Default;
+
+};
+
+} // namespace style
+} // namespace codegen