107 (Scan.pass 0 (Scan.repeat scan_cmt) |-- keep_line ($$ "*" -- $$ ")") >> K ""); |
107 (Scan.pass 0 (Scan.repeat scan_cmt) |-- keep_line ($$ "*" -- $$ ")") >> K ""); |
108 |
108 |
109 |
109 |
110 (* scan token *) |
110 (* scan token *) |
111 |
111 |
112 fun token k None x = (k, x, 0) |
112 fun token k NONE x = (k, x, 0) |
113 | token k (Some n) x = (k, x, n); |
113 | token k (SOME n) x = (k, x, n); |
114 |
114 |
115 fun scan_tok lex (n, cs) = |
115 fun scan_tok lex (n, cs) = |
116 (scan_string >> token String n || |
116 (scan_string >> token String n || |
117 scan_verbatim >> token Verbatim n || |
117 scan_verbatim >> token Verbatim n || |
118 Scan.repeat1 scan_blank >> (token Ignore n o implode) || |
118 Scan.repeat1 scan_blank >> (token Ignore n o implode) || |
127 |
127 |
128 val scan_rest = Scan.any Symbol.not_eof >> implode; |
128 val scan_rest = Scan.any Symbol.not_eof >> implode; |
129 |
129 |
130 fun scan_token lex x = |
130 fun scan_token lex x = |
131 (case scan_tok lex x of |
131 (case scan_tok lex x of |
132 ((Keyword, "ML", n), x') => (keep_line scan_rest >> token Verbatim (Some n)) x' |
132 ((Keyword, "ML", n), x') => (keep_line scan_rest >> token Verbatim (SOME n)) x' |
133 | tok_x' => tok_x'); |
133 | tok_x' => tok_x'); |
134 |
134 |
135 |
135 |
136 (* tokenize *) |
136 (* tokenize *) |
137 |
137 |
138 fun tokenize lex chs = |
138 fun tokenize lex chs = |
139 let |
139 let |
140 val scan_toks = Scan.repeat (scan_token lex); |
140 val scan_toks = Scan.repeat (scan_token lex); |
141 val ignore = fn (Ignore, _, _) => true | _ => false; |
141 val ignore = fn (Ignore, _, _) => true | _ => false; |
142 in |
142 in |
143 (case Scan.error (Scan.finite' Symbol.stopper scan_toks) (Some 1, chs) of |
143 (case Scan.error (Scan.finite' Symbol.stopper scan_toks) (SOME 1, chs) of |
144 (toks, (n, [])) => filter_out ignore toks @ [token EOF n "end-of-file"] |
144 (toks, (n, [])) => filter_out ignore toks @ [token EOF n "end-of-file"] |
145 | (_, (n, cs)) => error (lex_err n ("Bad input " ^ quote (Symbol.beginning 10 cs)))) |
145 | (_, (n, cs)) => error (lex_err n ("Bad input " ^ quote (Symbol.beginning 10 cs)))) |
146 end; |
146 end; |
147 |
147 |
148 |
148 |