-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathparser-generator-lex-analyzer.el
269 lines (225 loc) · 7.54 KB
/
parser-generator-lex-analyzer.el
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
;;; parser-generator-lex-analyzer.el --- Lex-analyzer library -*- lexical-binding: t -*-
;; Copyright (C) 2020-2024 Free Software Foundation, Inc.
;;; Commentary:
;;; Code:
(require 'parser-generator)
;;; Variables:
(defvar
parser-generator-lex-analyzer--get-function
nil
"Get token contents. Any return is valid even nil.")
(defvar
parser-generator-lex-analyzer--function
nil
"Get next token like \='(a b . c) or nil, expects signal if input-tape is invalid.")
(defvar
parser-generator-lex-analyzer--state-init
nil
"Initial value of state.")
(defvar
parser-generator-lex-analyzer--reset-function
nil
"Function used when resetting lex-analyzer.")
(defvar
parser-generator-lex-analyzer--index-init
1
"Initial value of index.")
;;; Buffer-Local Variables:
(defvar-local
parser-generator-lex-analyzer--buffered-response
nil
"Buffered tokens of lex-analyzer.")
(defvar-local
parser-generator-lex-analyzer--index
nil
"Index in lex-analyzer.")
(defvar-local
parser-generator-lex-analyzer--state
nil
"State of lex-analyzer.")
;; Functions
(defun parser-generator-lex-analyzer--get-function (token)
"Get information about TOKEN."
(unless parser-generator-lex-analyzer--get-function
(signal
'error
(list
(format "Missing lex-analyzer get function! Token: %s" token)
token)))
(let ((meta-information))
(condition-case error
(progn
(setq
meta-information
(funcall
parser-generator-lex-analyzer--get-function
token)))
(error
(signal
'error
(list
(format
"Lex-analyze failed to get token meta-data of %s, error: %s"
token
(car (cdr error)))
token
(car (cdr error))))))
meta-information))
(defun parser-generator-lex-analyzer--peek-next-look-ahead ()
"Peek next look-ahead number of tokens via lex-analyzer."
(unless parser-generator-lex-analyzer--index
(error "Missing lex-analyzer index when peeking!"))
(unless parser-generator--look-ahead-number
(error "Missing look-ahead-number when peeking!"))
(let ((look-ahead)
(look-ahead-length
0)
(index
parser-generator-lex-analyzer--index)
(state
parser-generator-lex-analyzer--state)
(k
(max
1
parser-generator--look-ahead-number)))
(while (<
look-ahead-length
k)
(let* ((result-list
(parser-generator-lex-analyzer--get-buffered-lex
index
state))
(token
(nth 0 result-list))
(new-index
(nth 2 result-list))
(new-state
(nth 3 result-list)))
(push
token
look-ahead)
(setq
look-ahead-length
(1+ look-ahead-length))
(setq
index
new-index)
(setq
state
new-state)))
(nreverse look-ahead)))
(defun parser-generator-lex-analyzer--pop-token ()
"Pop next token via lex-analyzer."
(unless parser-generator-lex-analyzer--index
(error "Missing lex-analyzer index when popping!"))
(unless parser-generator--look-ahead-number
(error "Missing look-ahead-number when popping!"))
(let* ((result-list
(parser-generator-lex-analyzer--get-buffered-lex
parser-generator-lex-analyzer--index
parser-generator-lex-analyzer--state))
(token
(nth 0 result-list))
(new-index
(nth 2 result-list))
(new-state
(nth 3 result-list)))
(setq-local
parser-generator-lex-analyzer--index
new-index)
(setq-local
parser-generator-lex-analyzer--state
new-state)
(list token)))
(defun parser-generator-lex-analyzer--reset ()
"Reset lex-analyzer."
(setq
parser-generator-lex-analyzer--buffered-response
(make-hash-table :test 'equal))
(setq
parser-generator-lex-analyzer--index
parser-generator-lex-analyzer--index-init)
(setq
parser-generator-lex-analyzer--state
parser-generator-lex-analyzer--state-init)
(when parser-generator-lex-analyzer--reset-function
(funcall parser-generator-lex-analyzer--reset-function)))
(defun parser-generator-lex-analyzer--get-buffered-lex (index state)
"Get next token in stream, use buffer to only call function when needed."
(unless (gethash
index
parser-generator-lex-analyzer--buffered-response)
(let ((continue t)
(tmp-index index)
(tmp-state state))
(unless parser-generator-lex-analyzer--function
(error "Missing lex-analyzer function!"))
(while continue
(condition-case error
(progn
(let* ((result-list
(funcall
parser-generator-lex-analyzer--function
tmp-index
tmp-state))
(tokens
(nth 0 result-list))
(move-to-index-flag
(nth 1 result-list))
(new-state
(nth 2 result-list)))
(if move-to-index-flag
(progn
(setq
tmp-index
move-to-index-flag)
(setq
tmp-state
new-state))
(if tokens
(progn
(unless (listp (car tokens))
(setq tokens (list tokens)))
(let* ((first-token (car tokens))
(first-token-start (car (cdr first-token)))
(first-token-end (cdr (cdr first-token))))
(when (< index first-token-start)
(let ((token-start index))
(while (< token-start first-token-start)
(puthash
token-start
(list
first-token
nil
first-token-end
new-state)
parser-generator-lex-analyzer--buffered-response)
(setq
token-start
(1+ token-start))))))
(dolist (token tokens)
(let ((token-start (car (cdr token)))
(token-end (cdr (cdr token))))
(puthash
token-start
(list token nil token-end new-state)
parser-generator-lex-analyzer--buffered-response))))
;; Fill up look-ahead with EOF-identifier if we found nothing
(puthash
index
(list (list parser-generator--eof-identifier) nil (1+ index) nil)
parser-generator-lex-analyzer--buffered-response))
(setq
continue
nil))))
(error
(error
"Lex-analyze failed to get next token at: %s in state: %s, error: %s"
index
state
error))))))
(gethash
index
parser-generator-lex-analyzer--buffered-response))
(provide 'parser-generator-lex-analyzer)
;;; parser-generator-lex-analyzer.el ends here