From 8c5e6ba30ea93e4c0ca94de8fbace7435b435f41 Mon Sep 17 00:00:00 2001 From: Venelin Date: Fri, 10 Mar 2023 19:04:21 +0200 Subject: [PATCH] introduce parse tokens --- build_debug.sh | 11 +- example_tree.tft | 12 +- install.sh | 1 - src/parser.cpp | 635 +++++++++++++++++++++++++++++------------------ 4 files changed, 409 insertions(+), 250 deletions(-) diff --git a/build_debug.sh b/build_debug.sh index 524d57b..e5b5bb1 100755 --- a/build_debug.sh +++ b/build_debug.sh @@ -1,7 +1,5 @@ #!/bin/sh -mkdir -p bin/debug - language=' -std=c++17 -fno-rtti @@ -10,12 +8,9 @@ language=' -fno-common' # force each global variable to be only defined once - warnings=' -Wall - -Wextra - -Wpedantic - -Wmultiple-inheritance - -Wvirtual-inheritance' #-fanalyzer + -Wextra' -g++ -o bin/debug/famt $language -g -O0 -march=native $warnings -pipe *.cpp -lncursesw +cd src +g++ -o ../tft_debug $language -g -O0 -march=native $warnings -pipe *.cpp -lncursesw diff --git a/example_tree.tft b/example_tree.tft index 9134c21..46dda23 100755 --- a/example_tree.tft +++ b/example_tree.tft @@ -1,19 +1,19 @@ # This is a comment # The dating in this tree is according to the Byzantine calendar but relative to Christ's birth -Adam, M # Adam is a male +Adam b: -5509.9.06 # Birthday in format yyyy.mm.dd -Eve, F +Eve, F # Eve is female b: -5509.9.06 s: Adam # Eve's spouse is Adam -Cain, M # Cain's birthday is unknown +Cain # Cain's birthday is unknown p: Adam, Eve # Cain's parents are Adam and Eve -Abel, M +Abel p: - # Abel's parents are the same as the last person's (Cain) -Seth, M -b: -5279 +Seth +b: -5279 # Only the year os Seth's birth is known p: - diff --git a/install.sh b/install.sh index e824054..70eeaf4 100755 --- a/install.sh +++ b/install.sh @@ -26,7 +26,6 @@ warnings=' other='-fno-fat-lto-objects' cd src - g++ -o tft $language $optimizations $warnings $other -DNDEBUG -pipe -s *.cpp -lncursesw #-static -lncursesw -ltinfo diff --git a/src/parser.cpp b/src/parser.cpp index b49a929..e020223 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1,27 +1,17 @@ #include -/* #include */ +#include #include #include "tree.hpp" #include "utils.hpp" +#include using std::cerr; -/* using std::string_view; */ +using std::string_view; using std::string; using std::vector; using std::pair; /* using std::endl; */ -/* macros rather then function to avoid warnings */ -#define ERR_UNEXPECTED_END() { endwin(); \ - cerr << "Unexpected end of file\n"; \ - cerr.flush(); \ - exit(1); } - -#define ERR_UNEXPECTED_CHAR(c) { endwin(); \ - cerr << "Line " << line << ": Unexpected '" << c << "'\n"; \ - cerr.flush(); \ - exit(1); } - namespace /* internal */ { struct alloced @@ -31,27 +21,93 @@ namespace /* internal */ alloced(U32 len) : data(new char[len]) , end(data + len) - { - } + {} ~alloced() { delete[] data; } }; - char* end_data; /* past the end pointer */ + enum TokenType + { + ATTRIBUTE, + STRING, + /* DATE, */ + NUMBER, + NEW_LINE, + SEPARATOR, /* , */ + REPEAT /* - */ + }; + + struct token + { + token(const char* b, const char* e); + U32 size() const + { + return end - begin; + } + + const char* begin; + const char* end; + TokenType type; + }; + + using TokenIt = vector::const_iterator; + + char* end_data_; /* past the end pointer */ unsigned line = 1; pair prev_parents(Nobody, Nobody); /* state */ - void state_neutral(char* it_data); - void state_person(char*& it_data); + /* void state_neutral(char* it_data); */ + void state_person(TokenIt& it, TokenIt end); /* void state_person_attribute() */ /* result */ Tree* cur_tree; - void err_third_parent(const string& extra_name) + [[noreturn]]void err_unexpected_end() + { + endwin(); + cerr << "Unexpected end of file\n"; + cerr.flush(); + exit(1); + } + + [[noreturn]]void err_unexpected_char(char c) + { + endwin(); + cerr << "Line " << line << ": Unexpected '" << c << "'\n"; + cerr.flush(); + exit(1); + } + + [[noreturn]]void err_expected(const char* what) + { + endwin(); + cerr << "Line " << line << ": Expected " << what << "\n"; + cerr.flush(); + exit(1); + } + + + [[noreturn]]void err_unexpected_token(token t) + { + endwin(); + /* cerr << "Line " << line << ": Unexpected token '" << c << "'\n"; */ + cerr.flush(); + exit(1); + } + + [[noreturn]]void err_illegal_attribute(token attr) + { + endwin(); + /* cerr << "Line " << line << ": Unexpected '" << c << "'\n"; */ + cerr.flush(); + exit(1); + } + + [[noreturn]]void err_thirdParent(const string& extra_name) { endwin(); cerr << "Line " << line << ": two parents are already defined, " @@ -60,6 +116,26 @@ namespace /* internal */ exit(1); } + [[noreturn]]void err_personUndefined(const string& name) + { + endwin(); + cerr << "Line " << line << ": '" << name << "' is undefined.\n" + "Members need to be defined before they are used as relations\n"; + cerr.flush(); + exit(1); + } + + [[noreturn]]void err_indexExcedes(U32 index, U32 max) + { + endwin(); + cerr << "Line " << line << ": '" << index << "' is too large.\n" + << "At most the index after a name can be the number of people with the " + "same name already defined - 1, since the count starts from 0.\n" + "In this case the max is " << max << '\n'; + cerr.flush(); + exit(1); + } + /* void err_name_index(const string& name, U32 i) */ /* { */ /* endwin(); */ @@ -74,291 +150,346 @@ namespace /* internal */ return c >= '0' && c <= '9'; } - string proceedToNameEnd(char*& it_data) + TokenType findType(token t) { - char* name_start = it_data; - while(it_data < end_data && - *it_data != ',' && - *it_data != '\n' && - !isNum(*it_data)) + TokenType type = NUMBER; + for(const char* it = t.begin; it < t.end; ++it) { - if(*it_data == ':' /* && *it_data == '-' */) - ERR_UNEXPECTED_CHAR(*it_data); - ++it_data; + if(!isNum(*it)) + { + type = STRING; + break; + } } - return string(name_start, it_data); /* it_data points past end */ + + if(type == STRING) + { + if(t.size() == 1) + { + switch(t.begin[0]) + { + case ',': + type = SEPARATOR; + break; + case '-': + type = REPEAT; + break; + case '\n': + type = NEW_LINE; + } + } + else if(t.size() == 2) + { + if(t.begin[1] == ':') + type = ATTRIBUTE; + } + } + return type; } - U32 sameNamedIndex(char*& it_data) + token::token(const char* b, const char* e) + : begin(b) + , end(e) + { + type = findType(*this); + } + + + U32 calcNumber(token t) + { + assert(t.type == NUMBER); + U32 val = 0; + for(const char* it = t.begin; it < t.end; ++it) + val = val*10 + *it-'0'; + + return val; + } + + void expect_newline(TokenIt& it) + { + if(it->type == NEW_LINE) + { + ++line; + ++it; + } + else + err_unexpected_token(*it); + } + + /* construct name from tokens */ + string parseName(TokenIt& it, TokenIt end) + { + if(it->type != STRING) + err_expected("name"); + + string name; + for(; it < end && it->type == STRING; ++it) + { + name.append(it->begin, it->end); + name += ' '; + } + name.pop_back(); /* remove last ' ' */ + return name; + } + + /* find a person's id in the tree based on his name and index + * among same named */ + person_id findPerson(const string& name, U32 same_name_index) + { + vector same_named = cur_tree->findId(name); + + if(same_named.empty()) + err_personUndefined(name); + else if(same_name_index >= same_named.size()) + err_indexExcedes(same_name_index, same_named.size()-1); + else + return same_named[same_name_index]; + } + + U32 sameNamedIndex(TokenIt& it) { U32 res = 0; - if(isNum(*it_data)) + if(it->type == NUMBER) { - res = *it_data - '0'; /* todo many digits */ - ++it_data; + res = calcNumber(*it); + ++it; } return res; } - Sex findSex(char*& it_data) + Sex parseSex(TokenIt& it, TokenIt end) { Sex res = M; - if(*it_data == ',') + if(it->type == SEPARATOR) { - ++it_data; - if(*it_data == 'M') - res = M; - else if(*it_data == 'F') - res = F; - else - ERR_UNEXPECTED_CHAR(*it_data); + ++it; + if(it == end) + err_unexpected_end(); - ++it_data; - if(*it_data != '\n') - ERR_UNEXPECTED_CHAR(*it_data); + if(it->type == STRING) + { + if(it->size() == 1) + { + if(it->begin[0] == 'M') + res = M; + else if(it->begin[0] == 'F') + res = F; + else + err_expected("sex"); + ++it; + } + else + err_expected("sex"); + + } + else + err_unexpected_token(*it); } + + expect_newline(it); + return res; } - I32 processDatePart(char*& it_data) + I32 processDatePart(const char*& it_date, const char* end_date) { I32 sign = 1; I32 val = 0; - for(; it_data < end_data; ++it_data) + for(; it_date < end_date; ++it_date) { - switch(*it_data) + switch(*it_date) { case '-': sign = -1; break; case '0' ... '9': - val = val*10 + (*it_data - '0'); + val = val*10 + (*it_date - '0'); break; case '.': if(val == 0) - ERR_UNEXPECTED_CHAR('.') + err_unexpected_char('.'); else return sign * val; break; - case '\n': - ++line; - return sign * val; /* case '?': todo */ default: - ERR_UNEXPECTED_CHAR(*it_data) + err_unexpected_char(*it_date); } } - ERR_UNEXPECTED_END() + return sign * val; } - EventTime processDate(char*& it_data) /* todo validate */ + EventTime state_date(TokenIt& it, TokenIt end) /* todo validate */ { - EventTime date; - date.year = processDatePart(it_data); - if(*it_data == '.') + ++it; /* skip attribute */ + if(it->type == STRING || it->type == NUMBER) { - ++it_data; - date.month = processDatePart(it_data); - - if(*it_data == '.') + EventTime date; + const char* it_date = it->begin; + const char* end_date = it->end; + date.year = processDatePart(it_date, end_date); + if(*it_date == '.') { - ++it_data; - date.day = processDatePart(it_data); - } - } - return date; - } + ++it_date; + date.month = processDatePart(it_date, end_date); - pair processParents(char*& it_data) - { - pair res(Nobody, Nobody); - U32 same_name_index = 0; - string name; /* todo dont use string */ - for(; it_data < end_data; ++it_data) - { - switch(*it_data) - { - case ',': - if(res.first == Nobody) - res.first = cur_tree->findId(name)[same_name_index]; /* todo validate index */ - else if(res.second == Nobody) - res.second = cur_tree->findId(name)[same_name_index]; - else - err_third_parent(name); - - same_name_index = 0; - name.clear(); - break; - case '-': - if(name.empty() && res.first == Nobody) - return prev_parents; - else - ERR_UNEXPECTED_CHAR('-') - case '\n': - ++line; - if(res.first == Nobody) - res.first = cur_tree->findId(name)[same_name_index]; /* todo validate index */ - else if(res.second == Nobody) - res.second = cur_tree->findId(name)[same_name_index]; - else - err_third_parent(name); - - return prev_parents = res; - case '_': - name.push_back(' '); - break; - case '0' ... '9': - same_name_index = *it_data - '0'; - break; - default: - name.push_back(*it_data); - } - } - ERR_UNEXPECTED_END() - } - - vector parseSpouses(char*& it_data) - { - vector res; - U32 same_name_index = 0; - string name; - for(; it_data < end_data; ++it_data) - { - switch(*it_data) - { - case '-': - if(name.empty()) - res.push_back( cur_tree->last() ); - else - ERR_UNEXPECTED_CHAR('-') - break; - case ',': - if(!name.empty()) + if(*it_date == '.') { - res.push_back( cur_tree->findId(name)[same_name_index] ); - name.clear(); - same_name_index = 0; + ++it_date; + date.day = processDatePart(it_date, end_date); } - else - ERR_UNEXPECTED_CHAR(',') - break; - case '\n': - ++line; - if(!name.empty()) - res.push_back( cur_tree->findId(name)[same_name_index] ); - return res; - case '_': - name.push_back(' '); - break; - case '0' ... '9': - same_name_index = *it_data - '0'; - break; - default: - name.push_back(*it_data); } + + ++it; + expect_newline(it); + return date; } - if(!name.empty()) - res.push_back( cur_tree->findId(name)[same_name_index] ); - return res; + else + err_expected("date"); + } - void state_neutral(char* it_data) + pair processParents(TokenIt& it, TokenIt end) { - for(; it_data < end_data; ++it_data) + ++it; /* skip attribute */ + + pair parents(Nobody, Nobody); + if(it->type == REPEAT) { - switch(*it_data) + parents = prev_parents; + ++it; + } + else + { + string name = parseName(it, end); + U32 same_name_index = sameNamedIndex(it); + parents.first = findPerson(name, same_name_index); + if(it->type == SEPARATOR) { - default: - state_person(it_data); + ++it; + string name = parseName(it, end); + U32 same_name_index = sameNamedIndex(it); + parents.second = findPerson(name, same_name_index); + } + prev_parents = parents; + } + + expect_newline(it); + return parents; + } + + vector parseSpouses(TokenIt& it, TokenIt end) + { + ++it; /* skip attribute */ + vector spouses; + while(true) + { + switch(it->type) + { + case STRING: + { + string name = parseName(it, end); /* todo unify in 1 function */ + U32 same_name_index = sameNamedIndex(it); + spouses.push_back(findPerson(name, same_name_index)); + } + break; + case SEPARATOR: + ++it; break; - case '\n': + case REPEAT: + ++it; + spouses.push_back( cur_tree->last() ); + break; + case NEW_LINE: ++line; - break; - case ',': - case '-': - case 0 ... 9: - case ':': - ERR_UNEXPECTED_CHAR(*it_data) + ++it; + return spouses; + default: + err_unexpected_token(*it); } } } - void state_person(char*& it_data) + void state_neutral(const vector& tokens) + { + auto end = tokens.cend(); + for(TokenIt it = tokens.cbegin(); it < end;) + { + switch(it->type) + { + case NEW_LINE: + ++line; + ++it; + break; + case STRING: + state_person(it, end); + break; + default: + err_unexpected_token(*it); + + } + } + } + + void state_person(TokenIt& it, TokenIt end) { pair parents(Nobody, Nobody); vector spouses; EventTime birth{}; EventTime death{}; - string name = proceedToNameEnd(it_data); - for(char& c: name) - { - if(c == '_') - c = ' '; - } + string name = parseName(it, end); - U32 same_name_index = sameNamedIndex(it_data); + if(it->type == NUMBER) + ++it; + + /* U32 same_name_index = sameNamedIndex(it); */ /* todo */ /* if(cur_tree->findId(name).size() != same_name_index) */ /* err_name_index(name, same_name_index); */ - Sex sex = findSex(it_data); + Sex sex = parseSex(it, end); - for(; it_data < end_data; ++it_data) + while(true) { - switch(*it_data) + switch(it->type) { - case '\n': + case ATTRIBUTE: + switch(it->begin[0]) + { + case 'b': + birth = state_date(it, end); /* todo check for redefinition */ + break; + case 'd': + death = state_date(it, end); + break; + case 'p': + parents = processParents(it, end); + break; + case 's': + spouses = parseSpouses(it, end); + break; + default: + err_illegal_attribute(*it); + } + break; + case NEW_LINE: ++line; - break; + ++it; + case STRING: + /* todo str prevent copy */ + cur_tree->addPerson(name.c_str(), sex, + parents.first, parents.second, birth, death); + for(person_id spouse: spouses) + cur_tree->addRelation(spouse, Spouse, cur_tree->last()); + return; default: - if(it_data[1] == ':') /* todo */ - { - switch(*it_data) - { - case 'b': - it_data += 2; - birth = processDate(it_data); - break; - case 'd': - it_data += 2; - death = processDate(it_data); - break; - case 'p': - it_data += 2; - parents = processParents(it_data); - break; - case 's': - it_data += 2; - spouses = parseSpouses(it_data); - break; - default: - ERR_UNEXPECTED_CHAR(it_data[-1]) - } - - } - else - { - cur_tree->addPerson(name.c_str(), sex, parents.first, parents.second, birth, death); /* todo prevent copy */ - for(person_id spouse: spouses) - cur_tree->addRelation(spouse, Spouse, cur_tree->last()); - --it_data; /* rewind 1 */ - return; - } - break; - /* case 0...9: */ - /* case ':': */ - /* ERR_UNEXPECTED_CHAR(*it_data) */ - /* default: */ - /* break; */ + err_unexpected_token(*it); } } - cur_tree->addPerson(name.c_str(), sex, parents.first, parents.second, birth, death); - for(person_id spouse: spouses) - cur_tree->addRelation(spouse, Spouse, cur_tree->last()); } - alloced preprocess(MappedFile file) /* remove whitespace and comments */ + alloced removeComments(MappedFile file) /* todo - drop */ { alloced res(file.len); @@ -366,25 +497,58 @@ namespace /* internal */ char* end = file.data + file.len; for(char* it = file.data; it < end; ++it) { - switch(*it) + if(*it == '#') { - case '#': while(++it != end && *it != '\n'); /* skip comment */ *itres++ = '\n'; /* add \n */ - break; - case ' ': - case '\t': - case '\r': - break; - default: + } + else + { *itres = *it; ++itres; } } - end_data = itres; + end_data_ = itres; return res; } + vector tokenize(alloced file) + { + vector tokens; + tokens.reserve(256); + char* token_begin = file.data; + + for(char* it = file.data; it < end_data_; ++it) + { + switch(*it) + { + case ':': + if(token_begin < it) + tokens.emplace_back(token_begin, it+1); /* consume the : */ + token_begin = it + 1; + break; + case ' ': + case '\t': + if(token_begin < it) + tokens.emplace_back(token_begin, it); + token_begin = it + 1; /* skip the ws */ + break; + case ',': + case '\n': + if(token_begin < it) + tokens.emplace_back(token_begin, it); + tokens.emplace_back(it, it+1); + token_begin = it+1; + break; + /* case '0' ... '9': */ + + default: break; + } + } + /* tokens.emplace_back(nullptr, nullptr); */ + /* tokens.back().type = NEW_LINE; */ + return tokens; + } } @@ -396,9 +560,10 @@ Tree parseTftFile(const char* path) cur_tree = &result; MappedFile mapedfile = mapfile(file); - alloced preprocessed = preprocess(mapedfile); + alloced preprocessed = removeComments(mapedfile); + vector tokens = tokenize(preprocessed); - state_neutral(preprocessed.data); + state_neutral(tokens); return result; }