53 "let", "local", "nonfix", "of", "op", "open", "orelse", "raise", |
53 "let", "local", "nonfix", "of", "op", "open", "orelse", "raise", |
54 "rec", "sharing", "sig", "signature", "struct", "structure", "then", |
54 "rec", "sharing", "sig", "signature", "struct", "structure", "then", |
55 "type", "val", "where", "while", "with", "withtype"]; |
55 "type", "val", "where", "while", "with", "withtype"]; |
56 |
56 |
57 val keywords2 = |
57 val keywords2 = |
58 ["and", "case", "do", "else", "end", "if", "in", "let", "local", "of", |
58 Symset.make ["and", "case", "do", "else", "end", "if", "in", "let", "local", "of", |
59 "sig", "struct", "then", "while", "with"]; |
59 "sig", "struct", "then", "while", "with"]; |
60 |
60 |
61 val keywords3 = |
61 val keywords3 = |
62 ["handle", "open", "raise"]; |
62 Symset.make ["handle", "open", "raise"]; |
63 |
63 |
64 val lexicon = Scan.make_lexicon (map raw_explode keywords); |
64 val lexicon = Scan.make_lexicon (map raw_explode keywords); |
65 |
65 |
66 |
66 |
67 |
67 |
166 fun token_report (tok as Token ((pos, _), (kind, x))) = |
166 fun token_report (tok as Token ((pos, _), (kind, x))) = |
167 let |
167 let |
168 val (markup, txt) = |
168 val (markup, txt) = |
169 if not (is_keyword tok) then token_kind_markup kind |
169 if not (is_keyword tok) then token_kind_markup kind |
170 else if is_delimiter tok then (Markup.ML_delimiter, "") |
170 else if is_delimiter tok then (Markup.ML_delimiter, "") |
171 else if member (op =) keywords2 x then (Markup.ML_keyword2 |> Markup.keyword_properties, "") |
171 else if Symset.member keywords2 x then (Markup.ML_keyword2 |> Markup.keyword_properties, "") |
172 else if member (op =) keywords3 x then (Markup.ML_keyword3 |> Markup.keyword_properties, "") |
172 else if Symset.member keywords3 x then (Markup.ML_keyword3 |> Markup.keyword_properties, "") |
173 else (Markup.ML_keyword1 |> Markup.keyword_properties, ""); |
173 else (Markup.ML_keyword1 |> Markup.keyword_properties, ""); |
174 in ((pos, markup), txt) end; |
174 in ((pos, markup), txt) end; |
175 |
175 |
176 end; |
176 end; |
177 |
177 |
194 Scan.many (Symbol.is_ascii_letdig o Symbol_Pos.symbol); |
194 Scan.many (Symbol.is_ascii_letdig o Symbol_Pos.symbol); |
195 |
195 |
196 val scan_alphanumeric = |
196 val scan_alphanumeric = |
197 Scan.one (Symbol.is_ascii_letter o Symbol_Pos.symbol) ::: scan_letdigs; |
197 Scan.one (Symbol.is_ascii_letter o Symbol_Pos.symbol) ::: scan_letdigs; |
198 |
198 |
199 val scan_symbolic = |
199 val symbolic = Symset.make (raw_explode "!#$%&*+-/:<=>?@\\^`|~"); |
200 Scan.many1 (member (op =) (raw_explode "!#$%&*+-/:<=>?@\\^`|~") o Symbol_Pos.symbol); |
200 |
|
201 val scan_symbolic = Scan.many1 (Symset.member symbolic o Symbol_Pos.symbol); |
201 |
202 |
202 in |
203 in |
203 |
204 |
204 val scan_ident = scan_alphanumeric || scan_symbolic; |
205 val scan_ident = scan_alphanumeric || scan_symbolic; |
205 |
206 |
242 |
243 |
243 val scan_blanks1 = Scan.many1 (Symbol.is_ascii_blank o Symbol_Pos.symbol); |
244 val scan_blanks1 = Scan.many1 (Symbol.is_ascii_blank o Symbol_Pos.symbol); |
244 |
245 |
245 local |
246 local |
246 |
247 |
|
248 val escape = Symset.make (raw_explode "\"\\abtnvfr"); |
|
249 |
247 val scan_escape = |
250 val scan_escape = |
248 Scan.one (member (op =) (raw_explode "\"\\abtnvfr") o Symbol_Pos.symbol) >> single || |
251 Scan.one (Symset.member escape o Symbol_Pos.symbol) >> single || |
249 $$$ "^" @@@ |
252 $$$ "^" @@@ |
250 (Scan.one (fn (s, _) => Char.ord #"@" <= ord s andalso ord s <= Char.ord #"_") >> single) || |
253 (Scan.one (fn (s, _) => Char.ord #"@" <= ord s andalso ord s <= Char.ord #"_") >> single) || |
251 Scan.one (Symbol.is_ascii_digit o Symbol_Pos.symbol) -- |
254 Scan.one (Symbol.is_ascii_digit o Symbol_Pos.symbol) -- |
252 Scan.one (Symbol.is_ascii_digit o Symbol_Pos.symbol) -- |
255 Scan.one (Symbol.is_ascii_digit o Symbol_Pos.symbol) -- |
253 Scan.one (Symbol.is_ascii_digit o Symbol_Pos.symbol) >> (fn ((a, b), c) => [a, b, c]); |
256 Scan.one (Symbol.is_ascii_digit o Symbol_Pos.symbol) >> (fn ((a, b), c) => [a, b, c]); |