introduce parse tokens

build_debug.sh

@@ -1,7 +1,5 @@
 #!/bin/sh
 
-mkdir -p bin/debug
-
 language='
 	-std=c++17
 	-fno-rtti
@@ -10,12 +8,9 @@ language='
 	-fno-common'
 # force each global variable to be only defined once
 
-
 warnings='
 	-Wall
-	-Wextra
+	-Wextra'
-	-Wpedantic
-	-Wmultiple-inheritance
-	-Wvirtual-inheritance' #-fanalyzer
 
-g++ -o bin/debug/famt $language -g -O0 -march=native $warnings -pipe *.cpp -lncursesw
+cd src
+g++ -o ../tft_debug $language -g -O0 -march=native $warnings -pipe *.cpp -lncursesw

example_tree.tft

@@ -1,19 +1,19 @@
 # This is a comment
 # The dating in this tree is according to the Byzantine calendar but relative to Christ's birth
 
-Adam, M              # Adam is a male
+Adam
 b: -5509.9.06        # Birthday in format yyyy.mm.dd
 
-Eve, F
+Eve, F               # Eve is female
 b: -5509.9.06
 s: Adam              # Eve's spouse is Adam
 
-Cain, M              # Cain's birthday is unknown
+Cain                 # Cain's birthday is unknown
 p: Adam, Eve         # Cain's parents are Adam and Eve
 
-Abel, M
+Abel
 p: -                 # Abel's parents are the same as the last person's (Cain)
 
-Seth, M
+Seth
-b: -5279
+b: -5279             # Only the year os Seth's birth is known
 p: -

install.sh

@@ -26,7 +26,6 @@ warnings='
 other='-fno-fat-lto-objects'
 
 cd src
-
 g++ -o tft $language $optimizations $warnings $other -DNDEBUG -pipe -s *.cpp -lncursesw
 #-static -lncursesw -ltinfo
 

src/parser.cpp

@@ -1,27 +1,17 @@
 #include <iostream>
-/* #include <string_view> */
+#include <string_view>
 #include <string>
 #include "tree.hpp"
 #include "utils.hpp"
+#include <assert.h>
 
 using std::cerr;
-/* using std::string_view; */
+using std::string_view;
 using std::string;
 using std::vector;
 using std::pair;
 /* using std::endl; */
 
-/* macros rather then function to avoid warnings */
-#define ERR_UNEXPECTED_END()   { endwin();		\
-		cerr << "Unexpected end of file\n";		\
-		cerr.flush();							\
-		exit(1); }
-
-#define ERR_UNEXPECTED_CHAR(c) { endwin();							\
-		cerr << "Line " << line <<  ": Unexpected '" << c << "'\n";	\
-		cerr.flush();												\
-		exit(1); }
-
 namespace /* internal */
 {
 	struct alloced
@@ -31,27 +21,93 @@ namespace /* internal */
 		alloced(U32 len)
 			: data(new char[len])
 			, end(data + len)
-		{
+		{}
-		}
 		~alloced()
 		{
 			delete[] data;
 		}
 	};
 
-	char* end_data;				/* past the end pointer */
+	enum TokenType
+	{
+		ATTRIBUTE,
+		STRING,
+		/* DATE, */
+		NUMBER,
+		NEW_LINE,
+		SEPARATOR,				/* , */
+		REPEAT					/* - */
+	};
+
+	struct token
+	{
+		token(const char* b, const char* e);
+		U32 size() const
+		{
+			return end - begin;
+		}
+
+		const char* begin;
+		const char* end;
+		TokenType type;
+	};
+
+	using TokenIt = vector<token>::const_iterator;
+
+	char* end_data_;				/* past the end pointer */
 	unsigned line = 1;
 	pair<person_id, person_id> prev_parents(Nobody, Nobody);
 
 	/* state */
-	void state_neutral(char* it_data);
+	/* void state_neutral(char* it_data); */
-	void state_person(char*& it_data);
+	void state_person(TokenIt& it, TokenIt end);
 	/* void state_person_attribute() */
 
 	/* result */
 	Tree* cur_tree;
 
-	void err_third_parent(const string& extra_name)
+	[[noreturn]]void err_unexpected_end()
+	{
+		endwin();
+		cerr << "Unexpected end of file\n";
+		cerr.flush();
+		exit(1);
+	}
+
+	[[noreturn]]void err_unexpected_char(char c)
+	{
+		endwin();
+		cerr << "Line " << line <<  ": Unexpected '" << c << "'\n";
+		cerr.flush();
+		exit(1);
+	}
+
+	[[noreturn]]void err_expected(const char* what)
+	{
+		endwin();
+		cerr << "Line " << line <<  ": Expected " << what << "\n";
+		cerr.flush();
+		exit(1);
+	}
+
+
+	[[noreturn]]void err_unexpected_token(token t)
+	{
+		endwin();
+		/* cerr << "Line " << line <<  ": Unexpected token '" << c << "'\n"; */
+		cerr.flush();
+		exit(1);
+	}
+
+	[[noreturn]]void err_illegal_attribute(token attr)
+	{
+		endwin();
+		/* cerr << "Line " << line <<  ": Unexpected '" << c << "'\n"; */
+		cerr.flush();
+		exit(1);
+	}
+
+	[[noreturn]]void err_thirdParent(const string& extra_name)
 	{
 		endwin();
 		cerr << "Line " << line << ": two parents are already defined, "
@@ -60,6 +116,26 @@ namespace /* internal */
 		exit(1);
 	}
 
+	[[noreturn]]void err_personUndefined(const string& name)
+	{
+		endwin();
+		cerr << "Line " << line << ": '" << name << "' is undefined.\n"
+			"Members need to be defined before they are used as relations\n";
+		cerr.flush();
+		exit(1);
+	}
+
+	[[noreturn]]void err_indexExcedes(U32 index, U32 max)
+	{
+		endwin();
+		cerr << "Line " << line << ": '" << index << "' is too large.\n"
+			 << "At most the index after a name can be the number of people with the "
+			"same name already defined - 1, since the count starts from 0.\n"
+			"In this case the max is " << max << '\n';
+		cerr.flush();
+		exit(1);
+	}
+
 	/* void err_name_index(const string& name, U32 i) */
 	/* { */
 	/* 	endwin(); */
@@ -74,291 +150,346 @@ namespace /* internal */
 		return c >= '0' && c <= '9';
 	}
 
-	string proceedToNameEnd(char*& it_data)
+	TokenType findType(token t)
 	{
-		char* name_start = it_data;
+		TokenType type = NUMBER;
-		while(it_data < end_data &&
+		for(const char* it = t.begin; it < t.end; ++it)
-			  *it_data != ',' &&
-			  *it_data != '\n' &&
-			  !isNum(*it_data))
 		{
-			if(*it_data == ':' /* && *it_data == '-' */)
+			if(!isNum(*it))
-				ERR_UNEXPECTED_CHAR(*it_data);
+			{
-			++it_data;
+				type = STRING;
+				break;
+			}
 		}
-		return string(name_start, it_data); /* it_data points past end */
+
+		if(type == STRING)
+		{
+			if(t.size() == 1)
+			{
+				switch(t.begin[0])
+				{
+				case ',':
+					type = SEPARATOR;
+					break;
+				case '-':
+					type = REPEAT;
+					break;
+				case '\n':
+					type = NEW_LINE;
+				}
+			}
+			else if(t.size() == 2)
+			{
+				if(t.begin[1] == ':')
+					type = ATTRIBUTE;
+			}
+		}
+		return type;
 	}
 
-	U32 sameNamedIndex(char*& it_data)
+	token::token(const char* b, const char* e)
+		: begin(b)
+		, end(e)
+	{
+		type = findType(*this);
+	}
+
+
+	U32 calcNumber(token t)
+	{
+		assert(t.type == NUMBER);
+		U32 val = 0;
+		for(const char* it = t.begin; it < t.end; ++it)
+			val = val*10 + *it-'0';
+
+		return val;
+	}
+
+	void expect_newline(TokenIt& it)
+	{
+		if(it->type == NEW_LINE)
+		{
+			++line;
+			++it;
+		}
+		else
+			err_unexpected_token(*it);
+	}
+
+	/* construct name from tokens */
+	string parseName(TokenIt& it, TokenIt end)
+	{
+		if(it->type != STRING)
+			err_expected("name");
+
+		string name;
+		for(; it < end && it->type == STRING; ++it)
+		{
+			name.append(it->begin, it->end);
+			name += ' ';
+		}
+		name.pop_back();		/* remove last ' ' */
+		return name;
+	}
+
+	/* find a person's id in the tree based on his name and index
+	 * among same named */
+	person_id findPerson(const string& name, U32 same_name_index)
+	{
+		vector<person_id> same_named = cur_tree->findId(name);
+
+		if(same_named.empty())
+			err_personUndefined(name);
+		else if(same_name_index >= same_named.size())
+			err_indexExcedes(same_name_index, same_named.size()-1);
+		else
+			return same_named[same_name_index];
+	}
+
+	U32 sameNamedIndex(TokenIt& it)
 	{
 		U32 res = 0;
-		if(isNum(*it_data))
+		if(it->type == NUMBER)
 		{
-			res = *it_data - '0'; /* todo many digits */
+			res = calcNumber(*it);
-			++it_data;
+			++it;
 		}
 		return res;
 	}
 
-	Sex findSex(char*& it_data)
+	Sex parseSex(TokenIt& it, TokenIt end)
 	{
 		Sex res = M;
-		if(*it_data == ',')
+		if(it->type == SEPARATOR)
 		{
-			++it_data;
+			++it;
-			if(*it_data == 'M')
+			if(it == end)
-				res = M;
+				err_unexpected_end();
-			else if(*it_data == 'F')
-				res = F;
-			else
-				ERR_UNEXPECTED_CHAR(*it_data);
 
-			++it_data;
+			if(it->type == STRING)
-			if(*it_data != '\n')
+			{
-				ERR_UNEXPECTED_CHAR(*it_data);
+				if(it->size() == 1)
+				{
+					if(it->begin[0] == 'M')
+						res = M;
+					else if(it->begin[0] == 'F')
+						res = F;
+					else
+						err_expected("sex");
+					++it;
+				}
+				else
+					err_expected("sex");
+
+			}
+			else
+				err_unexpected_token(*it);
 		}
+
+		expect_newline(it);
+
 		return res;
 	}
 
-	I32 processDatePart(char*& it_data)
+	I32 processDatePart(const char*& it_date, const char* end_date)
 	{
 		I32 sign = 1;
 		I32 val = 0;
-		for(; it_data < end_data; ++it_data)
+		for(; it_date < end_date; ++it_date)
 		{
-			switch(*it_data)
+			switch(*it_date)
 			{
 			case '-':
 				sign = -1;
 				break;
 			case '0' ... '9':
-				val = val*10 + (*it_data - '0');
+				val = val*10 + (*it_date - '0');
 				break;
 			case '.':
 				if(val == 0)
-					ERR_UNEXPECTED_CHAR('.')
+					err_unexpected_char('.');
 				else
 					return sign * val;
 				break;
-			case '\n':
-				++line;
-				return sign * val;
 				/* case '?': todo */
 			default:
-				ERR_UNEXPECTED_CHAR(*it_data)
+				err_unexpected_char(*it_date);
 			}
 		}
-		ERR_UNEXPECTED_END()
+		return sign * val;
 	}
 
-	EventTime processDate(char*& it_data)		/* todo validate  */
+	EventTime state_date(TokenIt& it, TokenIt end)		/* todo validate  */
 	{
-		EventTime date;
+		++it;					/* skip attribute */
-		date.year = processDatePart(it_data);
+		if(it->type == STRING || it->type == NUMBER)
-		if(*it_data == '.')
 		{
-			++it_data;
+			EventTime date;
-			date.month = processDatePart(it_data);
+			const char* it_date = it->begin;
-
+			const char* end_date = it->end;
-			if(*it_data == '.')
+			date.year = processDatePart(it_date, end_date);
+			if(*it_date == '.')
 			{
-				++it_data;
+				++it_date;
-				date.day = processDatePart(it_data);
+				date.month = processDatePart(it_date, end_date);
-			}
-		}
-		return date;
-	}
 
-	pair<person_id, person_id> processParents(char*& it_data)
+				if(*it_date == '.')
-	{
-		pair<person_id, person_id> res(Nobody, Nobody);
-		U32 same_name_index = 0;
-		string name;			/* todo dont use string */
-		for(; it_data < end_data; ++it_data)
-		{
-			switch(*it_data)
-			{
-			case ',':
-				if(res.first == Nobody)
-					res.first = cur_tree->findId(name)[same_name_index]; /* todo validate index */
-				else if(res.second == Nobody)
-					res.second = cur_tree->findId(name)[same_name_index];
-				else
-					err_third_parent(name);
-
-				same_name_index = 0;
-				name.clear();
-				break;
-			case '-':
-				if(name.empty() && res.first == Nobody)
-					return prev_parents;
-				else
-					ERR_UNEXPECTED_CHAR('-')
-			case '\n':
-				++line;
-				if(res.first == Nobody)
-					res.first = cur_tree->findId(name)[same_name_index]; /* todo validate index */
-				else if(res.second == Nobody)
-					res.second = cur_tree->findId(name)[same_name_index];
-				else
-					err_third_parent(name);
-
-				return prev_parents = res;
-			case '_':
-				name.push_back(' ');
-				break;
-			case '0' ... '9':
-				same_name_index = *it_data - '0';
-				break;
-			default:
-				name.push_back(*it_data);
-			}
-		}
-		ERR_UNEXPECTED_END()
-	}
-
-	vector<person_id> parseSpouses(char*& it_data)
-	{
-		vector<person_id> res;
-		U32 same_name_index = 0;
-		string name;
-		for(; it_data < end_data; ++it_data)
-		{
-			switch(*it_data)
-			{
-			case '-':
-				if(name.empty())
-					res.push_back( cur_tree->last() );
-				else
-					ERR_UNEXPECTED_CHAR('-')
-				break;
-			case ',':
-				if(!name.empty())
 				{
-					res.push_back( cur_tree->findId(name)[same_name_index] );
+					++it_date;
-					name.clear();
+					date.day = processDatePart(it_date, end_date);
-					same_name_index = 0;
 				}
-				else
-					ERR_UNEXPECTED_CHAR(',')
-				break;
-			case '\n':
-				++line;
-				if(!name.empty())
-					res.push_back( cur_tree->findId(name)[same_name_index] );
-				return res;
-			case '_':
-				name.push_back(' ');
-				break;
-			case '0' ... '9':
-				same_name_index = *it_data - '0';
-				break;
-			default:
-				name.push_back(*it_data);
 			}
+
+			++it;
+			expect_newline(it);
+			return date;
 		}
-		if(!name.empty())
+		else
-			res.push_back( cur_tree->findId(name)[same_name_index] );
+			err_expected("date");
-		return res;
+
 	}
 
-	void state_neutral(char* it_data)
+	pair<person_id, person_id> processParents(TokenIt& it, TokenIt end)
 	{
-		for(; it_data < end_data; ++it_data)
+		++it;					/* skip attribute */
+
+		pair<person_id, person_id> parents(Nobody, Nobody);
+		if(it->type == REPEAT)
 		{
-			switch(*it_data)
+			parents = prev_parents;
+			++it;
+		}
+		else
+		{
+			string name = parseName(it, end);
+			U32 same_name_index = sameNamedIndex(it);
+			parents.first = findPerson(name, same_name_index);
+			if(it->type == SEPARATOR)
 			{
-			default:
+				++it;
-				state_person(it_data);
+				string name = parseName(it, end);
+				U32 same_name_index = sameNamedIndex(it);
+				parents.second = findPerson(name, same_name_index);
+			}
+			prev_parents = parents;
+		}
+
+		expect_newline(it);
+		return parents;
+	}
+
+	vector<person_id> parseSpouses(TokenIt& it, TokenIt end)
+	{
+		++it;					/* skip attribute */
+		vector<person_id> spouses;
+		while(true)
+		{
+			switch(it->type)
+			{
+			case STRING:
+			{
+				string name = parseName(it, end); /* todo unify in 1 function */
+				U32 same_name_index = sameNamedIndex(it);
+				spouses.push_back(findPerson(name, same_name_index));
+			}
+			break;
+			case SEPARATOR:
+				++it;
 				break;
-			case '\n':
+			case REPEAT:
+				++it;
+				spouses.push_back( cur_tree->last() );
+				break;
+			case NEW_LINE:
 				++line;
-				break;
+				++it;
-			case ',':
+				return spouses;
-			case '-':
+			default:
-			case 0 ... 9:
+				err_unexpected_token(*it);
-			case ':':
-				ERR_UNEXPECTED_CHAR(*it_data)
 			}
 		}
 	}
 
-	void state_person(char*& it_data)
+	void state_neutral(const vector<token>& tokens)
+	{
+		auto end = tokens.cend();
+		for(TokenIt it = tokens.cbegin(); it < end;)
+		{
+			switch(it->type)
+			{
+			case NEW_LINE:
+				++line;
+				++it;
+				break;
+			case STRING:
+				state_person(it, end);
+				break;
+			default:
+				err_unexpected_token(*it);
+
+			}
+		}
+	}
+
+	void state_person(TokenIt& it, TokenIt end)
 	{
 		pair<person_id, person_id> parents(Nobody, Nobody);
 		vector<person_id> spouses;
 		EventTime birth{};
 		EventTime death{};
 
-		string name = proceedToNameEnd(it_data);
+		string name = parseName(it, end);
-		for(char& c: name)
-		{
-			if(c == '_')
-				c = ' ';
-		}
 
-		U32 same_name_index = sameNamedIndex(it_data);
+		if(it->type == NUMBER)
+			++it;
+
+		/* U32 same_name_index = sameNamedIndex(it); */
 		/* todo */
 		/* if(cur_tree->findId(name).size() != same_name_index) */
 			/* err_name_index(name, same_name_index); */
 
-		Sex sex = findSex(it_data);
+		Sex sex = parseSex(it, end);
 
-		for(; it_data < end_data; ++it_data)
+		while(true)
 		{
-			switch(*it_data)
+			switch(it->type)
 			{
-			case '\n':
+			case ATTRIBUTE:
+				switch(it->begin[0])
+				{
+				case 'b':
+					birth = state_date(it, end); /* todo check for redefinition */
+					break;
+				case 'd':
+					death = state_date(it, end);
+					break;
+				case 'p':
+					parents = processParents(it, end);
+					break;
+				case 's':
+					spouses = parseSpouses(it, end);
+					break;
+				default:
+					err_illegal_attribute(*it);
+				}
+				break;
+			case NEW_LINE:
 				++line;
-				break;
+				++it;
+			case STRING:
+				/* todo str prevent copy */
+				cur_tree->addPerson(name.c_str(), sex,
+									parents.first, parents.second, birth, death);
+				for(person_id spouse: spouses)
+					cur_tree->addRelation(spouse, Spouse, cur_tree->last());
+				return;
 			default:
-				if(it_data[1] == ':') /* todo */
+				err_unexpected_token(*it);
-				{
-					switch(*it_data)
-					{
-					case 'b':
-						it_data += 2;
-						birth = processDate(it_data);
-						break;
-					case 'd':
-						it_data += 2;
-						death = processDate(it_data);
-						break;
-					case 'p':
-						it_data += 2;
-						parents = processParents(it_data);
-						break;
-					case 's':
-						it_data += 2;
-						spouses = parseSpouses(it_data);
-						break;
-					default:
-						ERR_UNEXPECTED_CHAR(it_data[-1])
-					}
-
-				}
-				else
-				{
-					cur_tree->addPerson(name.c_str(), sex, parents.first, parents.second, birth, death); /* todo prevent copy */
-					for(person_id spouse: spouses)
-						cur_tree->addRelation(spouse, Spouse, cur_tree->last());
-					--it_data;	/* rewind 1 */
-					return;
-				}
-				break;
-			/* case 0...9: */
-			/* case ':': */
-			/* 	ERR_UNEXPECTED_CHAR(*it_data) */
-			/* default: */
-				/* break; */
 			}
 		}
-		cur_tree->addPerson(name.c_str(), sex, parents.first, parents.second, birth, death);
-		for(person_id spouse: spouses)
-			cur_tree->addRelation(spouse, Spouse, cur_tree->last());
 	}
 
-	alloced preprocess(MappedFile file) /* remove whitespace and comments */
+	alloced removeComments(MappedFile file) /* todo - drop */
 	{
 		alloced res(file.len);
 
@@ -366,25 +497,58 @@ namespace /* internal */
 		char* end = file.data + file.len;
 		for(char* it = file.data; it < end; ++it)
 		{
-			switch(*it)
+			if(*it == '#')
 			{
-			case '#':
 				while(++it != end && *it != '\n'); /* skip comment */
 				*itres++ = '\n';				   /* add \n */
-				break;
+			}
-			case ' ':
+			else
-			case '\t':
+			{
-			case '\r':
-				break;
-			default:
 				*itres = *it;
 				++itres;
 			}
 		}
-		end_data = itres;
+		end_data_ = itres;
 		return res;
 	}
 
+	vector<token> tokenize(alloced file)
+	{
+		vector<token> tokens;
+		tokens.reserve(256);
+		char* token_begin = file.data;
+
+		for(char* it = file.data; it < end_data_; ++it)
+		{
+			switch(*it)
+			{
+			case ':':
+				if(token_begin < it)
+					tokens.emplace_back(token_begin, it+1); /* consume the : */
+				token_begin = it + 1;
+				break;
+			case ' ':
+			case '\t':
+				if(token_begin < it)
+					tokens.emplace_back(token_begin, it);
+				token_begin = it + 1; /* skip the ws */
+				break;
+			case ',':
+			case '\n':
+				if(token_begin < it)
+					tokens.emplace_back(token_begin, it);
+				tokens.emplace_back(it, it+1);
+				token_begin = it+1;
+				break;
+			/* case '0' ... '9': */
+
+			default: break;
+			}
+		}
+		/* tokens.emplace_back(nullptr, nullptr); */
+		/* tokens.back().type = NEW_LINE; */
+		return tokens;
+	}
 }
 
 
@@ -396,9 +560,10 @@ Tree parseTftFile(const char* path)
 	cur_tree = &result;
 
 	MappedFile mapedfile = mapfile(file);
-	alloced preprocessed = preprocess(mapedfile);
+	alloced preprocessed = removeComments(mapedfile);
+	vector<token> tokens = tokenize(preprocessed);
 
-	state_neutral(preprocessed.data);
+	state_neutral(tokens);
 
 	return result;
 }