equal
deleted
inserted
replaced
16 val pos_of: token -> string |
16 val pos_of: token -> string |
17 val kind_of: token -> token_kind |
17 val kind_of: token -> token_kind |
18 val content_of: token -> string |
18 val content_of: token -> string |
19 val keywords: string list |
19 val keywords: string list |
20 val source: (Symbol.symbol, 'a) Source.source -> |
20 val source: (Symbol.symbol, 'a) Source.source -> |
21 (token, (SymbolPos.T, Position.T * (Symbol.symbol, 'a) Source.source) |
21 (token, (Symbol_Pos.T, Position.T * (Symbol.symbol, 'a) Source.source) |
22 Source.source) Source.source |
22 Source.source) Source.source |
23 end; |
23 end; |
24 |
24 |
25 structure ML_Lex: ML_LEX = |
25 structure ML_Lex: ML_LEX = |
26 struct |
26 struct |
73 |
73 |
74 |
74 |
75 |
75 |
76 (** scanners **) |
76 (** scanners **) |
77 |
77 |
78 open BasicSymbolPos; |
78 open Basic_Symbol_Pos; |
79 |
79 |
80 fun !!! msg = SymbolPos.!!! ("SML lexical error: " ^ msg); |
80 fun !!! msg = Symbol_Pos.!!! ("SML lexical error: " ^ msg); |
81 |
81 |
82 |
82 |
83 (* blanks *) |
83 (* blanks *) |
84 |
84 |
85 val scan_blank = Scan.one (Symbol.is_ascii_blank o symbol); |
85 val scan_blank = Scan.one (Symbol.is_ascii_blank o symbol); |
181 |
181 |
182 (* token source *) |
182 (* token source *) |
183 |
183 |
184 local |
184 local |
185 |
185 |
186 fun token k ss = Token (SymbolPos.range ss, (k, SymbolPos.implode ss)); |
186 fun token k ss = Token (Symbol_Pos.range ss, (k, Symbol_Pos.implode ss)); |
187 |
187 |
188 val scan = !!! "bad input" |
188 val scan = !!! "bad input" |
189 (scan_char >> token Char || |
189 (scan_char >> token Char || |
190 scan_string >> token String || |
190 scan_string >> token String || |
191 scan_blanks1 >> token Space || |
191 scan_blanks1 >> token Space || |
192 SymbolPos.scan_comment !!! >> token Comment || |
192 Symbol_Pos.scan_comment !!! >> token Comment || |
193 Scan.max token_leq |
193 Scan.max token_leq |
194 (scan_keyword >> token Keyword) |
194 (scan_keyword >> token Keyword) |
195 (scan_word >> token Word || |
195 (scan_word >> token Word || |
196 scan_real >> token Real || |
196 scan_real >> token Real || |
197 scan_int >> token Int || |
197 scan_int >> token Int || |
204 >> (fn cs => [token (Error msg) cs]); |
204 >> (fn cs => [token (Error msg) cs]); |
205 |
205 |
206 in |
206 in |
207 |
207 |
208 fun source src = |
208 fun source src = |
209 SymbolPos.source (Position.line 1) src |
209 Symbol_Pos.source (Position.line 1) src |
210 |> Source.source SymbolPos.stopper (Scan.bulk scan) (SOME (false, recover)); |
210 |> Source.source Symbol_Pos.stopper (Scan.bulk scan) (SOME (false, recover)); |
211 |
211 |
212 end; |
212 end; |
213 |
213 |
214 end; |
214 end; |
215 |
215 |