equal
deleted
inserted
replaced
20 local (* Lexer *) |
20 local (* Lexer *) |
21 |
21 |
22 open Basic_Symbol_Pos |
22 open Basic_Symbol_Pos |
23 |
23 |
24 val keywords = |
24 val keywords = |
25 Scan.make_lexicon (map Symbol.explode ["code", "font", "abbrev", ":"]); |
25 Scan.make_lexicon (map Symbol.explode ["code", "group", "font", "abbrev", ":"]); |
26 |
26 |
27 datatype token_kind = |
27 datatype token_kind = |
28 Symbol | AsciiSymbol | Keyword | Name | Code | Space | Comment | EOF; |
28 Symbol | AsciiSymbol | Keyword | Name | Code | Space | Comment | EOF; |
29 |
29 |
30 datatype token = Token of token_kind * string * Position.range; |
30 datatype token = Token of token_kind * string * Position.range; |
126 val ascii_sym = Scan.one is_ascii_sym >> str_of_token; |
126 val ascii_sym = Scan.one is_ascii_sym >> str_of_token; |
127 val symbol = Scan.one is_symbol >> (fn t => (str_of_token t, pos_of_token t)); |
127 val symbol = Scan.one is_symbol >> (fn t => (str_of_token t, pos_of_token t)); |
128 val name = Scan.one is_name >> str_of_token; |
128 val name = Scan.one is_name >> str_of_token; |
129 |
129 |
130 val unicode = $$$ "code" -- $$$ ":" |-- hex_code; |
130 val unicode = $$$ "code" -- $$$ ":" |-- hex_code; |
|
131 val group = Scan.option ($$$ "group" -- $$$ ":" |-- name); |
131 val font = Scan.option ($$$ "font" -- $$$ ":" |-- name); |
132 val font = Scan.option ($$$ "font" -- $$$ ":" |-- name); |
132 val abbr = Scan.option ($$$ "abbrev" -- $$$ ":" |
133 val abbr = Scan.option ($$$ "abbrev" -- $$$ ":" |
133 |-- (ascii_sym || $$$ ":" || name)); |
134 |-- (ascii_sym || $$$ ":" || name)); |
134 |
135 |
135 in |
136 in |
136 |
137 |
137 val line = (symbol -- unicode -- font -- abbr) >> (fn (((a, b), c), d) => (a, b, d)); |
138 val line = (symbol -- unicode -- group -- font -- abbr) |
|
139 >> (fn ((((a, b), _), _), c) => (a, b, c)); |
138 |
140 |
139 end; |
141 end; |
140 |
142 |
141 local (* build tables *) |
143 local (* build tables *) |
142 |
144 |