lexer.c - ded - Dramatic EDitor

src/lexer.c (6787B)
Line numbers
   1 #include <assert.h>
   2 #include <stdbool.h>
   3 #include <ctype.h>
   4 #include <string.h>
   5 #include "common.h"
   6 #include "lexer.h"
   7 
   8 typedef struct {
   9     Token_Kind kind;
  10     const char *text;
  11 } Literal_Token;
  12 
  13 Literal_Token literal_tokens[] = {
  14     {.text = "(", .kind = TOKEN_OPEN_PAREN},
  15     {.text = ")", .kind = TOKEN_CLOSE_PAREN},
  16     {.text = "{", .kind = TOKEN_OPEN_CURLY},
  17     {.text = "}", .kind = TOKEN_CLOSE_CURLY},
  18     {.text = ";", .kind = TOKEN_SEMICOLON},
  19 };
  20 #define literal_tokens_count (sizeof(literal_tokens)/sizeof(literal_tokens[0]))
  21 
  22 const char *keywords[] = {
  23     "auto", "break", "case", "char", "const", "continue", "default", "do", "double",
  24     "else", "enum", "extern", "float", "for", "goto", "if", "int", "long", "register",
  25     "return", "short", "signed", "sizeof", "static", "struct", "switch", "typedef",
  26     "union", "unsigned", "void", "volatile", "while", "alignas", "alignof", "and",
  27     "and_eq", "asm", "atomic_cancel", "atomic_commit", "atomic_noexcept", "bitand",
  28     "bitor", "bool", "catch", "char16_t", "char32_t", "char8_t", "class", "co_await",
  29     "co_return", "co_yield", "compl", "concept", "const_cast", "consteval", "constexpr",
  30     "constinit", "decltype", "delete", "dynamic_cast", "explicit", "export", "false",
  31     "friend", "inline", "mutable", "namespace", "new", "noexcept", "not", "not_eq",
  32     "nullptr", "operator", "or", "or_eq", "private", "protected", "public", "reflexpr",
  33     "reinterpret_cast", "requires", "static_assert", "static_cast", "synchronized",
  34     "template", "this", "thread_local", "throw", "true", "try", "typeid", "typename",
  35     "using", "virtual", "wchar_t", "xor", "xor_eq",
  36 };
  37 #define keywords_count (sizeof(keywords)/sizeof(keywords[0]))
  38 
  39 const char *token_kind_name(Token_Kind kind)
  40 {
  41     switch (kind) {
  42     case TOKEN_END:
  43         return "end of content";
  44     case TOKEN_INVALID:
  45         return "invalid token";
  46     case TOKEN_PREPROC:
  47         return "preprocessor directive";
  48     case TOKEN_SYMBOL:
  49         return "symbol";
  50     case TOKEN_OPEN_PAREN:
  51         return "open paren";
  52     case TOKEN_CLOSE_PAREN:
  53         return "close paren";
  54     case TOKEN_OPEN_CURLY:
  55         return "open curly";
  56     case TOKEN_CLOSE_CURLY:
  57         return "close curly";
  58     case TOKEN_SEMICOLON:
  59         return "semicolon";
  60     case TOKEN_KEYWORD:
  61         return "keyword";
  62     default:
  63         UNREACHABLE("token_kind_name");
  64     }
  65     return NULL;
  66 }
  67 
  68 Lexer lexer_new(Free_Glyph_Atlas *atlas, const char *content, size_t content_len)
  69 {
  70     Lexer l = {0};
  71     l.atlas = atlas;
  72     l.content = content;
  73     l.content_len = content_len;
  74     return l;
  75 }
  76 
  77 bool lexer_starts_with(Lexer *l, const char *prefix)
  78 {
  79     size_t prefix_len = strlen(prefix);
  80     if (prefix_len == 0) {
  81         return true;
  82     }
  83     if (l->cursor + prefix_len - 1 >= l->content_len) {
  84         return false;
  85     }
  86     for (size_t i = 0; i < prefix_len; ++i) {
  87         if (prefix[i] != l->content[l->cursor + i]) {
  88             return false;
  89         }
  90     }
  91     return true;
  92 }
  93 
  94 void lexer_chop_char(Lexer *l, size_t len)
  95 {
  96     for (size_t i = 0; i < len; ++i) {
  97         // TODO: get rid of this assert by checking the length of the choped prefix upfront
  98         assert(l->cursor < l->content_len);
  99         char x = l->content[l->cursor];
 100         l->cursor += 1;
 101         if (x == '\n') {
 102             l->line += 1;
 103             l->bol = l->cursor;
 104             l->x = 0;
 105         } else {
 106             if (l->atlas) {
 107                 size_t glyph_index = x;
 108                 // TODO: support for glyphs outside of ASCII range
 109                 if (glyph_index >= GLYPH_METRICS_CAPACITY) {
 110                     glyph_index = '?';
 111                 }
 112                 Glyph_Metric metric = l->atlas->metrics[glyph_index];
 113                 l->x += metric.ax;
 114             }
 115         }
 116     }
 117 }
 118 
 119 void lexer_trim_left(Lexer *l)
 120 {
 121     while (l->cursor < l->content_len && isspace(l->content[l->cursor])) {
 122         lexer_chop_char(l, 1);
 123     }
 124 }
 125 
 126 bool is_symbol_start(char x)
 127 {
 128     return isalpha(x) || x == '_';
 129 }
 130 
 131 bool is_symbol(char x)
 132 {
 133     return isalnum(x) || x == '_';
 134 }
 135 
 136 Token lexer_next(Lexer *l)
 137 {
 138     lexer_trim_left(l);
 139 
 140     Token token = {
 141         .text = &l->content[l->cursor],
 142     };
 143 
 144     token.position.x = l->x;
 145     token.position.y = -(float)l->line * FREE_GLYPH_FONT_SIZE;
 146 
 147     if (l->cursor >= l->content_len) return token;
 148 
 149     if (l->content[l->cursor] == '"') {
 150         // TODO: TOKEN_STRING should also handle escape sequences
 151         token.kind = TOKEN_STRING;
 152         lexer_chop_char(l, 1);
 153         while (l->cursor < l->content_len && l->content[l->cursor] != '"' && l->content[l->cursor] != '\n') {
 154             lexer_chop_char(l, 1);
 155         }
 156         if (l->cursor < l->content_len) {
 157             lexer_chop_char(l, 1);
 158         }
 159         token.text_len = &l->content[l->cursor] - token.text;
 160         return token;
 161     }
 162 
 163     if (l->content[l->cursor] == '#') {
 164         // TODO: preproc should also handle newlines
 165         token.kind = TOKEN_PREPROC;
 166         while (l->cursor < l->content_len && l->content[l->cursor] != '\n') {
 167             lexer_chop_char(l, 1);
 168         }
 169         if (l->cursor < l->content_len) {
 170             lexer_chop_char(l, 1);
 171         }
 172         token.text_len = &l->content[l->cursor] - token.text;
 173         return token;
 174     }
 175 
 176     if (lexer_starts_with(l, "//")) {
 177         token.kind = TOKEN_COMMENT;
 178         while (l->cursor < l->content_len && l->content[l->cursor] != '\n') {
 179             lexer_chop_char(l, 1);
 180         }
 181         if (l->cursor < l->content_len) {
 182             lexer_chop_char(l, 1);
 183         }
 184         token.text_len = &l->content[l->cursor] - token.text;
 185         return token;
 186     }
 187     
 188     for (size_t i = 0; i < literal_tokens_count; ++i) {
 189         if (lexer_starts_with(l, literal_tokens[i].text)) {
 190             // NOTE: this code assumes that there is no newlines in literal_tokens[i].text
 191             size_t text_len = strlen(literal_tokens[i].text);
 192             token.kind = literal_tokens[i].kind;
 193             token.text_len = text_len;
 194             lexer_chop_char(l, text_len);
 195             return token;
 196         }
 197     }
 198 
 199     if (is_symbol_start(l->content[l->cursor])) {
 200         token.kind = TOKEN_SYMBOL;
 201         while (l->cursor < l->content_len && is_symbol(l->content[l->cursor])) {
 202             lexer_chop_char(l, 1);
 203             token.text_len += 1;
 204         }
 205 
 206         for (size_t i = 0; i < keywords_count; ++i) {
 207             size_t keyword_len = strlen(keywords[i]);
 208             if (keyword_len == token.text_len && memcmp(keywords[i], token.text, keyword_len) == 0) {
 209                 token.kind = TOKEN_KEYWORD;
 210                 break;
 211             }
 212         }
 213 
 214         return token;
 215     }
 216 
 217     lexer_chop_char(l, 1);
 218     token.kind = TOKEN_INVALID;
 219     token.text_len = 1;
 220     return token;
 221 }