fix and update readme

src/parser.cpp

@@ -22,6 +22,13 @@ namespace /* internal */
 			: data(new char[len])
 			, end(data + len)
 		{}
+		alloced(const alloced&) = delete;
+		alloced(alloced&& other)
+			: data(other.data)
+			, end(other.end)
+		{
+			const_cast<char*&>(other.data) = nullptr;
+		}
 		~alloced()
 		{
 			delete[] data;
@@ -42,6 +49,11 @@ namespace /* internal */
 	struct token
 	{
 		token(const char* b, const char* e);
+		explicit token(TokenType t)
+			: type(t)
+		 {
+		 }
+
 		U32 size() const
 		{
 			return end - begin;
@@ -512,13 +524,13 @@ namespace /* internal */
 		return res;
 	}
 
-	vector<token> tokenize(alloced file)
+	vector<token> tokenize(const char* data_begin, const char* data_end)
 	{
 		vector<token> tokens;
 		tokens.reserve(256);
-		char* token_begin = file.data;
+		const char* token_begin = data_begin;
 
-		for(char* it = file.data; it < end_data_; ++it)
+		for(const char* it = data_begin; it < data_end; ++it)
 		{
 			switch(*it)
 			{
@@ -545,8 +557,8 @@ namespace /* internal */
 			default: break;
 			}
 		}
-		/* tokens.emplace_back(nullptr, nullptr); */
-		/* tokens.back().type = NEW_LINE; */
+		tokens.emplace_back(NEW_LINE);
+		tokens.emplace_back(NEW_LINE);
 		return tokens;
 	}
 }
@@ -561,7 +573,7 @@ Tree parseTftFile(const char* path)
 
 	MappedFile mapedfile = mapfile(file);
 	alloced preprocessed = removeComments(mapedfile);
-	vector<token> tokens = tokenize(preprocessed);
+	vector<token> tokens = tokenize(preprocessed.data, end_data_);
 
 	state_neutral(tokens);