drop preprocesing

src/parser.cpp

@@ -66,7 +66,6 @@ namespace /* internal */
 
 	using TokenIt = vector<token>::const_iterator;
 
-	char* end_data_;				/* past the end pointer */
 	unsigned line = 1;
 	pair<person_id, person_id> prev_parents(Nobody, Nobody);
 
@@ -507,36 +506,15 @@ namespace /* internal */
 		}
 	}
 
-	alloced removeComments(MappedFile file) /* todo - drop */
-	{
-		alloced res(file.len);
-
-		char* itres = res.data;
-		char* end = file.data + file.len;
-		for(char* it = file.data; it < end; ++it)
-		{
-			if(*it == '#')
-			{
-				while(++it != end && *it != '\n'); /* skip comment */
-				*itres++ = '\n';				   /* add \n */
-			}
-			else
-			{
-				*itres = *it;
-				++itres;
-			}
-		}
-		end_data_ = itres;
-		return res;
-	}
-
-	vector<token> tokenize(const char* data_begin, const char* data_end)
+	vector<token> tokenize(const MappedFile f)
 	{
 		vector<token> tokens;
 		tokens.reserve(256);
-		const char* token_begin = data_begin;
+		const char* token_begin = f.data;
+		const char* end = f.data + f.len;
 
-		for(const char* it = data_begin; it < data_end; ++it)
+		const char* it = f.data;
+		for(; it < end; ++it)
 		{
 			switch(*it)
 			{
@@ -555,15 +533,25 @@ namespace /* internal */
 			case '\n':
 				if(token_begin < it)
 					tokens.emplace_back(token_begin, it);
-				tokens.emplace_back(it, it+1);
-				token_begin = it+1;
+				tokens.emplace_back(it, it+1); /* add the , or \n as a token */
+				token_begin = it + 1;
+				break;
+			case '#':
+				if(token_begin < it)
+					tokens.emplace_back(token_begin, it);
+				while(++it < end && *it != '\n'); /* skip comment */
+				token_begin = it;
+				--it;
 				break;
 			/* case '0' ... '9': */
 
 			default: break;
 			}
 		}
-		tokens.emplace_back(NEW_LINE);
+		if(token_begin < it)
+			tokens.emplace_back(token_begin, it);
+
+		tokens.emplace_back(NEW_LINE); /* add newlines at the end for easier parsing */
 		tokens.emplace_back(NEW_LINE);
 		return tokens;
 	}
@@ -578,8 +566,7 @@ Tree parseTftFile(const char* path)
 	cur_tree = &result;
 
 	MappedFile mapedfile = mapfile(file);
-	alloced preprocessed = removeComments(mapedfile);
-	vector<token> tokens = tokenize(preprocessed.data, end_data_);
+	vector<token> tokens = tokenize(mapedfile);
 
 	state_neutral(tokens);