Repository navigation
Expand file tree
/
Copy pathlexer.h
More file actions
140 lines (126 loc) · 4.23 KB
/
Copy pathlexer.h
File metadata and controls
140 lines (126 loc) · 4.23 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
/* lexer.h -- simple tokeniser for Micro Python
*
* Uses (byte) length instead of null termination.
* Tokens are the same - UTF-8 with (byte) length.
*/
typedef enum _mp_token_kind_t {
MP_TOKEN_END, // 0
MP_TOKEN_INVALID,
MP_TOKEN_DEDENT_MISMATCH,
MP_TOKEN_LONELY_STRING_OPEN,
MP_TOKEN_NEWLINE, // 4
MP_TOKEN_INDENT, // 5
MP_TOKEN_DEDENT, // 6
MP_TOKEN_NAME, // 7
MP_TOKEN_NUMBER,
MP_TOKEN_STRING,
MP_TOKEN_BYTES,
MP_TOKEN_ELLIPSES,
MP_TOKEN_KW_FALSE, // 12
MP_TOKEN_KW_NONE,
MP_TOKEN_KW_TRUE,
MP_TOKEN_KW_AND,
MP_TOKEN_KW_AS,
MP_TOKEN_KW_ASSERT,
MP_TOKEN_KW_BREAK,
MP_TOKEN_KW_CLASS,
MP_TOKEN_KW_CONTINUE,
MP_TOKEN_KW_DEF, // 21
MP_TOKEN_KW_DEL,
MP_TOKEN_KW_ELIF,
MP_TOKEN_KW_ELSE,
MP_TOKEN_KW_EXCEPT,
MP_TOKEN_KW_FINALLY,
MP_TOKEN_KW_FOR,
MP_TOKEN_KW_FROM,
MP_TOKEN_KW_GLOBAL,
MP_TOKEN_KW_IF,
MP_TOKEN_KW_IMPORT, // 31
MP_TOKEN_KW_IN,
MP_TOKEN_KW_IS,
MP_TOKEN_KW_LAMBDA,
MP_TOKEN_KW_NONLOCAL,
MP_TOKEN_KW_NOT,
MP_TOKEN_KW_OR,
MP_TOKEN_KW_PASS,
MP_TOKEN_KW_RAISE,
MP_TOKEN_KW_RETURN,
MP_TOKEN_KW_TRY, // 41
MP_TOKEN_KW_WHILE,
MP_TOKEN_KW_WITH,
MP_TOKEN_KW_YIELD,
MP_TOKEN_OP_PLUS, // 45
MP_TOKEN_OP_MINUS,
MP_TOKEN_OP_STAR,
MP_TOKEN_OP_DBL_STAR,
MP_TOKEN_OP_SLASH,
MP_TOKEN_OP_DBL_SLASH,
MP_TOKEN_OP_PERCENT,
MP_TOKEN_OP_LESS,
MP_TOKEN_OP_DBL_LESS,
MP_TOKEN_OP_MORE,
MP_TOKEN_OP_DBL_MORE, // 55
MP_TOKEN_OP_AMPERSAND,
MP_TOKEN_OP_PIPE,
MP_TOKEN_OP_CARET,
MP_TOKEN_OP_TILDE,
MP_TOKEN_OP_LESS_EQUAL,
MP_TOKEN_OP_MORE_EQUAL,
MP_TOKEN_OP_DBL_EQUAL,
MP_TOKEN_OP_NOT_EQUAL,
MP_TOKEN_DEL_PAREN_OPEN, // 64
MP_TOKEN_DEL_PAREN_CLOSE,
MP_TOKEN_DEL_BRACKET_OPEN,
MP_TOKEN_DEL_BRACKET_CLOSE,
MP_TOKEN_DEL_BRACE_OPEN,
MP_TOKEN_DEL_BRACE_CLOSE,
MP_TOKEN_DEL_COMMA,
MP_TOKEN_DEL_COLON,
MP_TOKEN_DEL_PERIOD,
MP_TOKEN_DEL_SEMICOLON,
MP_TOKEN_DEL_AT, // 74
MP_TOKEN_DEL_EQUAL,
MP_TOKEN_DEL_PLUS_EQUAL,
MP_TOKEN_DEL_MINUS_EQUAL,
MP_TOKEN_DEL_STAR_EQUAL,
MP_TOKEN_DEL_SLASH_EQUAL,
MP_TOKEN_DEL_DBL_SLASH_EQUAL,
MP_TOKEN_DEL_PERCENT_EQUAL,
MP_TOKEN_DEL_AMPERSAND_EQUAL,
MP_TOKEN_DEL_PIPE_EQUAL,
MP_TOKEN_DEL_CARET_EQUAL, // 84
MP_TOKEN_DEL_DBL_MORE_EQUAL,
MP_TOKEN_DEL_DBL_LESS_EQUAL,
MP_TOKEN_DEL_DBL_STAR_EQUAL,
MP_TOKEN_DEL_MINUS_MORE,
} mp_token_kind_t;
typedef struct _mp_token_t {
const char *src_name; // name of source
uint src_line; // source line
uint src_column; // source column
mp_token_kind_t kind; // kind of token
const char *str; // string of token (valid only while this token is current token)
uint len; // (byte) length of string of token
} mp_token_t;
// the next-char function must return the next character in the stream
// it must return MP_LEXER_CHAR_EOF if end of stream
// it can be called again after returning MP_LEXER_CHAR_EOF, and in that case must return MP_LEXER_CHAR_EOF
#define MP_LEXER_CHAR_EOF (-1)
typedef unichar (*mp_lexer_stream_next_char_t)(void*);
typedef void (*mp_lexer_stream_close_t)(void*);
typedef struct _mp_lexer_t mp_lexer_t;
void mp_token_show(const mp_token_t *tok);
void mp_token_show_error_prefix(const mp_token_t *tok);
bool mp_token_show_error(const mp_token_t *tok, const char *msg);
mp_lexer_t *mp_lexer_new(const char *src_name, void *stream_data, mp_lexer_stream_next_char_t stream_next_char, mp_lexer_stream_close_t stream_close);
void mp_lexer_free(mp_lexer_t *lex);
void mp_lexer_to_next(mp_lexer_t *lex);
const mp_token_t *mp_lexer_cur(const mp_lexer_t *lex);
bool mp_lexer_is_kind(mp_lexer_t *lex, mp_token_kind_t kind);
/* unused
bool mp_lexer_is_str(mp_lexer_t *lex, const char *str);
bool mp_lexer_opt_kind(mp_lexer_t *lex, mp_token_kind_t kind);
bool mp_lexer_opt_str(mp_lexer_t *lex, const char *str);
*/
bool mp_lexer_show_error(mp_lexer_t *lex, const char *msg);
bool mp_lexer_show_error_pythonic(mp_lexer_t *lex, const char *msg);