equal
deleted
inserted
replaced
244 |
244 |
245 fun implode_str ss = enclose "''" "''" (implode (map (fn "'" => "\\'" | s => s) ss)); |
245 fun implode_str ss = enclose "''" "''" (implode (map (fn "'" => "\\'" | s => s) ss)); |
246 val explode_str = explode_literal scan_str_body; |
246 val explode_str = explode_literal scan_str_body; |
247 |
247 |
248 |
248 |
249 (* comment cartouche *) |
|
250 |
|
251 val scan_comment_cartouche = |
|
252 $$$ "\<comment>" @@@ Scan.many (Symbol.is_blank o Symbol_Pos.symbol) @@@ |
|
253 !!! "cartouche expected after \"\<comment>\"" (Symbol_Pos.scan_cartouche err_prefix); |
|
254 |
|
255 |
249 |
256 (** tokenize **) |
250 (** tokenize **) |
257 |
251 |
258 fun token_leq (Token (_, s1, _), Token (_, s2, _)) = s1 <= s2; |
252 fun token_leq (Token (_, s1, _), Token (_, s2, _)) = s1 <= s2; |
259 fun token kind ss = Token (kind, Symbol_Pos.content ss, Symbol_Pos.range ss); |
253 fun token kind ss = Token (kind, Symbol_Pos.content ss, Symbol_Pos.range ss); |
276 val scan_lit = Scan.literal lex >> token Literal; |
270 val scan_lit = Scan.literal lex >> token Literal; |
277 |
271 |
278 val scan_token = |
272 val scan_token = |
279 Symbol_Pos.scan_cartouche err_prefix >> token Cartouche || |
273 Symbol_Pos.scan_cartouche err_prefix >> token Cartouche || |
280 Symbol_Pos.scan_comment err_prefix >> token Comment || |
274 Symbol_Pos.scan_comment err_prefix >> token Comment || |
281 scan_comment_cartouche >> token Comment_Cartouche || |
275 Symbol_Pos.scan_comment_cartouche err_prefix >> token Comment_Cartouche || |
282 Scan.max token_leq scan_lit scan_val || |
276 Scan.max token_leq scan_lit scan_val || |
283 scan_string >> token StringSy || |
277 scan_string >> token StringSy || |
284 scan_str >> token StrSy || |
278 scan_str >> token StrSy || |
285 Scan.many1 (Symbol.is_blank o Symbol_Pos.symbol) >> token Space; |
279 Scan.many1 (Symbol.is_blank o Symbol_Pos.symbol) >> token Space; |
286 in |
280 in |