author | wenzelm |
Sat, 11 Aug 2012 20:54:06 +0200 | |
changeset 48773 | 0e1bab274672 |
parent 48704 | 85a3de10567d |
child 48774 | c4bd5bb3ae69 |
permissions | -rw-r--r-- |
6118 | 1 |
(* Title: Pure/General/symbol.ML |
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
2 |
Author: Markus Wenzel, TU Muenchen |
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
3 |
|
21897 | 4 |
Generalized characters with infinitely many named symbols. |
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
5 |
*) |
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
6 |
|
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
7 |
signature SYMBOL = |
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
8 |
sig |
40509 | 9 |
type symbol = string |
26524 | 10 |
val STX: symbol |
11 |
val DEL: symbol |
|
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
12 |
val space: symbol |
14678 | 13 |
val is_char: symbol -> bool |
37533
d775bd70f571
explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents:
34095
diff
changeset
|
14 |
val is_utf8: symbol -> bool |
14678 | 15 |
val is_symbolic: symbol -> bool |
16 |
val is_printable: symbol -> bool |
|
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
17 |
val eof: symbol |
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
18 |
val is_eof: symbol -> bool |
27766 | 19 |
val not_eof: symbol -> bool |
27732 | 20 |
val stopper: symbol Scan.stopper |
14678 | 21 |
val sync: symbol |
22 |
val is_sync: symbol -> bool |
|
23784
75e6b9dd5336
Symbol.not_eof/sync is superceded by Symbol.is_regular (rules out further control symbols);
wenzelm
parents:
23728
diff
changeset
|
23 |
val is_regular: symbol -> bool |
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
24 |
val is_malformed: symbol -> bool |
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
25 |
val malformed_msg: symbol -> string |
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
26 |
val is_ascii: symbol -> bool |
14678 | 27 |
val is_ascii_letter: symbol -> bool |
28 |
val is_ascii_digit: symbol -> bool |
|
24580
916259859344
replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents:
23784
diff
changeset
|
29 |
val is_ascii_hex: symbol -> bool |
14678 | 30 |
val is_ascii_quasi: symbol -> bool |
31 |
val is_ascii_blank: symbol -> bool |
|
34095
c2f176a38448
robust representation of low ASCII control characters within XML/YXML text;
wenzelm
parents:
33955
diff
changeset
|
32 |
val is_ascii_control: symbol -> bool |
20200 | 33 |
val is_ascii_lower: symbol -> bool |
34 |
val is_ascii_upper: symbol -> bool |
|
35 |
val to_ascii_lower: symbol -> symbol |
|
36 |
val to_ascii_upper: symbol -> symbol |
|
14834 | 37 |
val is_raw: symbol -> bool |
38 |
val decode_raw: symbol -> string |
|
14977
77d88064991a
added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents:
14961
diff
changeset
|
39 |
val encode_raw: string -> string |
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
40 |
datatype sym = |
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
41 |
Char of string | UTF8 of string | Sym of string | Ctrl of string | Raw of string | |
43485 | 42 |
Malformed of string | EOF |
14873 | 43 |
val decode: symbol -> sym |
14678 | 44 |
datatype kind = Letter | Digit | Quasi | Blank | Other |
45 |
val kind: symbol -> kind |
|
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
46 |
val is_letter: symbol -> bool |
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
47 |
val is_digit: symbol -> bool |
12904 | 48 |
val is_quasi: symbol -> bool |
14678 | 49 |
val is_blank: symbol -> bool |
47850
c638127b4653
avoid interference of markup for literal tokens, which may contain slightly odd \<^bsub> \<^esub> counted as pseudo-markup (especially relevant for HTML output, e.g. of thm power3_eq_cube);
wenzelm
parents:
43947
diff
changeset
|
50 |
val is_block_ctrl: symbol -> bool |
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
51 |
val is_quasi_letter: symbol -> bool |
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
52 |
val is_letdig: symbol -> bool |
16138 | 53 |
val is_ident: symbol list -> bool |
14728 | 54 |
val beginning: int -> symbol list -> string |
14678 | 55 |
val scanner: string -> (string list -> 'a * string list) -> symbol list -> 'a |
13730 | 56 |
val scan_id: string list -> string * string list |
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
57 |
val source: (string, 'a) Source.source -> (symbol, (string, 'a) Source.source) Source.source |
6272 | 58 |
val explode: string -> symbol list |
31425 | 59 |
val esc: symbol -> string |
14977
77d88064991a
added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents:
14961
diff
changeset
|
60 |
val escape: string -> string |
14678 | 61 |
val strip_blanks: string -> string |
62 |
val bump_init: string -> string |
|
12904 | 63 |
val bump_string: string -> string |
14678 | 64 |
val length: symbol list -> int |
6692 | 65 |
val xsymbolsN: string |
40131
7cbebd636e79
explicitly qualify type Output.output, which is a slightly odd internal feature;
wenzelm
parents:
37728
diff
changeset
|
66 |
val output: string -> Output.output * int |
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
67 |
end; |
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
68 |
|
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
69 |
structure Symbol: SYMBOL = |
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
70 |
struct |
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
71 |
|
14678 | 72 |
(** type symbol **) |
6272 | 73 |
|
14678 | 74 |
(*Symbols, which are considered the smallest entities of any Isabelle |
6272 | 75 |
string, may be of the following form: |
14678 | 76 |
|
14834 | 77 |
(1) ASCII symbols: a |
17823 | 78 |
(2) regular symbols: \<ident> |
14834 | 79 |
(3) control symbols: \<^ident> |
80 |
(4) raw control symbols: \<^raw:...>, where "..." may be any printable |
|
20205
7b2958d3d575
raw symbols: disallow dot to avoid confusion in NameSpace.unpack;
wenzelm
parents:
20200
diff
changeset
|
81 |
character (excluding ".", ">"), or \<^raw000> |
6272 | 82 |
|
14678 | 83 |
Output is subject to the print_mode variable (default: verbatim), |
84 |
actual interpretation in display is up to front-end tools. |
|
6272 | 85 |
*) |
86 |
||
87 |
type symbol = string; |
|
88 |
||
26524 | 89 |
val STX = chr 2; |
90 |
val DEL = chr 127; |
|
91 |
||
92 |
val space = chr 32; |
|
17063 | 93 |
|
14678 | 94 |
fun is_char s = size s = 1; |
95 |
||
37533
d775bd70f571
explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents:
34095
diff
changeset
|
96 |
fun is_utf8 s = size s > 0 andalso forall_string (fn c => ord c >= 128) s; |
d775bd70f571
explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents:
34095
diff
changeset
|
97 |
|
14678 | 98 |
fun is_symbolic s = |
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
99 |
String.isPrefix "\\<" s andalso String.isSuffix ">" s andalso not (String.isPrefix "\\<^" s); |
14678 | 100 |
|
101 |
fun is_printable s = |
|
102 |
if is_char s then ord space <= ord s andalso ord s <= ord "~" |
|
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
103 |
else is_utf8 s orelse is_symbolic s; |
26632 | 104 |
|
6272 | 105 |
|
14678 | 106 |
(* input source control *) |
6272 | 107 |
|
14678 | 108 |
val eof = ""; |
6272 | 109 |
fun is_eof s = s = eof; |
110 |
fun not_eof s = s <> eof; |
|
27732 | 111 |
val stopper = Scan.stopper (K eof) is_eof; |
6272 | 112 |
|
14678 | 113 |
val sync = "\\<^sync>"; |
114 |
fun is_sync s = s = sync; |
|
115 |
||
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
116 |
fun is_regular s = not_eof s andalso s <> sync; |
25641 | 117 |
|
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
118 |
fun is_malformed s = String.isPrefix "\\<" s andalso not (String.isSuffix ">" s); |
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
119 |
fun malformed_msg s = "Malformed symbolic character: " ^ quote s; |
14678 | 120 |
|
121 |
||
43418 | 122 |
(* ASCII symbols *) |
14678 | 123 |
|
124 |
fun is_ascii s = is_char s andalso ord s < 128; |
|
125 |
||
126 |
fun is_ascii_letter s = |
|
127 |
is_char s andalso |
|
128 |
(ord "A" <= ord s andalso ord s <= ord "Z" orelse |
|
129 |
ord "a" <= ord s andalso ord s <= ord "z"); |
|
130 |
||
131 |
fun is_ascii_digit s = |
|
132 |
is_char s andalso ord "0" <= ord s andalso ord s <= ord "9"; |
|
133 |
||
24580
916259859344
replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents:
23784
diff
changeset
|
134 |
fun is_ascii_hex s = |
916259859344
replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents:
23784
diff
changeset
|
135 |
is_char s andalso |
916259859344
replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents:
23784
diff
changeset
|
136 |
(ord "0" <= ord s andalso ord s <= ord "9" orelse |
916259859344
replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents:
23784
diff
changeset
|
137 |
ord "A" <= ord s andalso ord s <= ord "F" orelse |
916259859344
replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents:
23784
diff
changeset
|
138 |
ord "a" <= ord s andalso ord s <= ord "f"); |
916259859344
replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents:
23784
diff
changeset
|
139 |
|
14678 | 140 |
fun is_ascii_quasi "_" = true |
141 |
| is_ascii_quasi "'" = true |
|
142 |
| is_ascii_quasi _ = false; |
|
143 |
||
144 |
val is_ascii_blank = |
|
43845
d89353d17f54
added File.fold_pages for streaming of large files;
wenzelm
parents:
43777
diff
changeset
|
145 |
fn " " => true | "\t" => true | "\n" => true | "\^K" => true | "\f" => true | "\^M" => true |
14678 | 146 |
| _ => false; |
147 |
||
34095
c2f176a38448
robust representation of low ASCII control characters within XML/YXML text;
wenzelm
parents:
33955
diff
changeset
|
148 |
fun is_ascii_control s = is_char s andalso ord s < 32 andalso not (is_ascii_blank s); |
c2f176a38448
robust representation of low ASCII control characters within XML/YXML text;
wenzelm
parents:
33955
diff
changeset
|
149 |
|
20200 | 150 |
fun is_ascii_lower s = is_char s andalso (ord "a" <= ord s andalso ord s <= ord "z"); |
151 |
fun is_ascii_upper s = is_char s andalso (ord "A" <= ord s andalso ord s <= ord "Z"); |
|
152 |
||
153 |
fun to_ascii_lower s = if is_ascii_upper s then chr (ord s + ord "a" - ord "A") else s; |
|
154 |
fun to_ascii_upper s = if is_ascii_lower s then chr (ord s + ord "A" - ord "a") else s; |
|
155 |
||
14678 | 156 |
|
14956
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
157 |
(* encode_raw *) |
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
158 |
|
20205
7b2958d3d575
raw symbols: disallow dot to avoid confusion in NameSpace.unpack;
wenzelm
parents:
20200
diff
changeset
|
159 |
fun raw_chr c = |
48773
0e1bab274672
more liberal scanning of potentially malformed symbols;
wenzelm
parents:
48704
diff
changeset
|
160 |
is_char c andalso |
0e1bab274672
more liberal scanning of potentially malformed symbols;
wenzelm
parents:
48704
diff
changeset
|
161 |
(ord space <= ord c andalso ord c <= ord "~" andalso c <> "." andalso c <> ">" |
0e1bab274672
more liberal scanning of potentially malformed symbols;
wenzelm
parents:
48704
diff
changeset
|
162 |
orelse ord c >= 128); |
14956
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
163 |
|
29324 | 164 |
fun encode_raw "" = "" |
165 |
| encode_raw str = |
|
166 |
let |
|
167 |
val raw0 = enclose "\\<^raw:" ">"; |
|
168 |
val raw1 = raw0 o implode; |
|
169 |
val raw2 = enclose "\\<^raw" ">" o string_of_int o ord; |
|
170 |
||
33955 | 171 |
fun encode cs = enc (take_prefix raw_chr cs) |
29324 | 172 |
and enc ([], []) = [] |
173 |
| enc (cs, []) = [raw1 cs] |
|
174 |
| enc ([], d :: ds) = raw2 d :: encode ds |
|
175 |
| enc (cs, d :: ds) = raw1 cs :: raw2 d :: encode ds; |
|
176 |
in |
|
40627
becf5d5187cc
renamed raw "explode" function to "raw_explode" to emphasize its meaning;
wenzelm
parents:
40523
diff
changeset
|
177 |
if exists_string (not o raw_chr) str then implode (encode (raw_explode str)) |
29324 | 178 |
else raw0 str |
179 |
end; |
|
14956
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
180 |
|
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
181 |
|
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
182 |
(* diagnostics *) |
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
183 |
|
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
184 |
fun beginning n cs = |
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
185 |
let |
33955 | 186 |
val drop_blanks = #1 o take_suffix is_ascii_blank; |
14956
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
187 |
val all_cs = drop_blanks cs; |
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
188 |
val dots = if length all_cs > n then " ..." else ""; |
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
189 |
in |
33955 | 190 |
(drop_blanks (take n all_cs) |
14956
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
191 |
|> map (fn c => if is_ascii_blank c then space else c) |
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
192 |
|> implode) ^ dots |
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
193 |
end; |
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
194 |
|
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
195 |
|
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
196 |
(* decode_raw *) |
14834 | 197 |
|
198 |
fun is_raw s = |
|
17063 | 199 |
String.isPrefix "\\<^raw" s andalso String.isSuffix ">" s; |
14834 | 200 |
|
201 |
fun decode_raw s = |
|
23676
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
202 |
if not (is_raw s) then error (malformed_msg s) |
14834 | 203 |
else if String.isPrefix "\\<^raw:" s then String.substring (s, 7, size s - 8) |
40627
becf5d5187cc
renamed raw "explode" function to "raw_explode" to emphasize its meaning;
wenzelm
parents:
40523
diff
changeset
|
204 |
else chr (#1 (Library.read_int (raw_explode (String.substring (s, 6, size s - 7))))); |
14834 | 205 |
|
206 |
||
14873 | 207 |
(* symbol variants *) |
208 |
||
37533
d775bd70f571
explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents:
34095
diff
changeset
|
209 |
datatype sym = |
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
210 |
Char of string | UTF8 of string | Sym of string | Ctrl of string | Raw of string | |
43485 | 211 |
Malformed of string | EOF; |
14873 | 212 |
|
213 |
fun decode s = |
|
43485 | 214 |
if s = "" then EOF |
215 |
else if is_char s then Char s |
|
37533
d775bd70f571
explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents:
34095
diff
changeset
|
216 |
else if is_utf8 s then UTF8 s |
14873 | 217 |
else if is_raw s then Raw (decode_raw s) |
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
218 |
else if is_malformed s then Malformed s |
14873 | 219 |
else if String.isPrefix "\\<^" s then Ctrl (String.substring (s, 3, size s - 4)) |
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
220 |
else Sym (String.substring (s, 2, size s - 3)); |
14873 | 221 |
|
222 |
||
14678 | 223 |
(* standard symbol kinds *) |
224 |
||
225 |
datatype kind = Letter | Digit | Quasi | Blank | Other; |
|
6272 | 226 |
|
14171
0cab06e3bbd0
Extended the notion of letter and digit, such that now one may use greek,
skalberg
parents:
13730
diff
changeset
|
227 |
local |
14678 | 228 |
val symbol_kinds = Symtab.make |
229 |
[("\\<A>", Letter), |
|
230 |
("\\<B>", Letter), |
|
231 |
("\\<C>", Letter), |
|
232 |
("\\<D>", Letter), |
|
233 |
("\\<E>", Letter), |
|
234 |
("\\<F>", Letter), |
|
235 |
("\\<G>", Letter), |
|
236 |
("\\<H>", Letter), |
|
237 |
("\\<I>", Letter), |
|
238 |
("\\<J>", Letter), |
|
239 |
("\\<K>", Letter), |
|
240 |
("\\<L>", Letter), |
|
241 |
("\\<M>", Letter), |
|
242 |
("\\<N>", Letter), |
|
243 |
("\\<O>", Letter), |
|
244 |
("\\<P>", Letter), |
|
245 |
("\\<Q>", Letter), |
|
246 |
("\\<R>", Letter), |
|
247 |
("\\<S>", Letter), |
|
248 |
("\\<T>", Letter), |
|
249 |
("\\<U>", Letter), |
|
250 |
("\\<V>", Letter), |
|
251 |
("\\<W>", Letter), |
|
252 |
("\\<X>", Letter), |
|
253 |
("\\<Y>", Letter), |
|
254 |
("\\<Z>", Letter), |
|
255 |
("\\<a>", Letter), |
|
256 |
("\\<b>", Letter), |
|
257 |
("\\<c>", Letter), |
|
258 |
("\\<d>", Letter), |
|
259 |
("\\<e>", Letter), |
|
260 |
("\\<f>", Letter), |
|
261 |
("\\<g>", Letter), |
|
262 |
("\\<h>", Letter), |
|
263 |
("\\<i>", Letter), |
|
264 |
("\\<j>", Letter), |
|
265 |
("\\<k>", Letter), |
|
266 |
("\\<l>", Letter), |
|
267 |
("\\<m>", Letter), |
|
268 |
("\\<n>", Letter), |
|
269 |
("\\<o>", Letter), |
|
270 |
("\\<p>", Letter), |
|
271 |
("\\<q>", Letter), |
|
272 |
("\\<r>", Letter), |
|
273 |
("\\<s>", Letter), |
|
274 |
("\\<t>", Letter), |
|
275 |
("\\<u>", Letter), |
|
276 |
("\\<v>", Letter), |
|
277 |
("\\<w>", Letter), |
|
278 |
("\\<x>", Letter), |
|
279 |
("\\<y>", Letter), |
|
280 |
("\\<z>", Letter), |
|
281 |
("\\<AA>", Letter), |
|
282 |
("\\<BB>", Letter), |
|
283 |
("\\<CC>", Letter), |
|
284 |
("\\<DD>", Letter), |
|
285 |
("\\<EE>", Letter), |
|
286 |
("\\<FF>", Letter), |
|
287 |
("\\<GG>", Letter), |
|
288 |
("\\<HH>", Letter), |
|
289 |
("\\<II>", Letter), |
|
290 |
("\\<JJ>", Letter), |
|
291 |
("\\<KK>", Letter), |
|
292 |
("\\<LL>", Letter), |
|
293 |
("\\<MM>", Letter), |
|
294 |
("\\<NN>", Letter), |
|
295 |
("\\<OO>", Letter), |
|
296 |
("\\<PP>", Letter), |
|
297 |
("\\<QQ>", Letter), |
|
298 |
("\\<RR>", Letter), |
|
299 |
("\\<SS>", Letter), |
|
300 |
("\\<TT>", Letter), |
|
301 |
("\\<UU>", Letter), |
|
302 |
("\\<VV>", Letter), |
|
303 |
("\\<WW>", Letter), |
|
304 |
("\\<XX>", Letter), |
|
305 |
("\\<YY>", Letter), |
|
306 |
("\\<ZZ>", Letter), |
|
307 |
("\\<aa>", Letter), |
|
308 |
("\\<bb>", Letter), |
|
309 |
("\\<cc>", Letter), |
|
310 |
("\\<dd>", Letter), |
|
311 |
("\\<ee>", Letter), |
|
312 |
("\\<ff>", Letter), |
|
313 |
("\\<gg>", Letter), |
|
314 |
("\\<hh>", Letter), |
|
315 |
("\\<ii>", Letter), |
|
316 |
("\\<jj>", Letter), |
|
317 |
("\\<kk>", Letter), |
|
318 |
("\\<ll>", Letter), |
|
319 |
("\\<mm>", Letter), |
|
320 |
("\\<nn>", Letter), |
|
321 |
("\\<oo>", Letter), |
|
322 |
("\\<pp>", Letter), |
|
323 |
("\\<qq>", Letter), |
|
324 |
("\\<rr>", Letter), |
|
325 |
("\\<ss>", Letter), |
|
326 |
("\\<tt>", Letter), |
|
327 |
("\\<uu>", Letter), |
|
328 |
("\\<vv>", Letter), |
|
329 |
("\\<ww>", Letter), |
|
330 |
("\\<xx>", Letter), |
|
331 |
("\\<yy>", Letter), |
|
332 |
("\\<zz>", Letter), |
|
333 |
("\\<alpha>", Letter), |
|
334 |
("\\<beta>", Letter), |
|
335 |
("\\<gamma>", Letter), |
|
336 |
("\\<delta>", Letter), |
|
337 |
("\\<epsilon>", Letter), |
|
338 |
("\\<zeta>", Letter), |
|
339 |
("\\<eta>", Letter), |
|
340 |
("\\<theta>", Letter), |
|
341 |
("\\<iota>", Letter), |
|
342 |
("\\<kappa>", Letter), |
|
343 |
("\\<lambda>", Other), (*sic!*) |
|
344 |
("\\<mu>", Letter), |
|
345 |
("\\<nu>", Letter), |
|
346 |
("\\<xi>", Letter), |
|
347 |
("\\<pi>", Letter), |
|
348 |
("\\<rho>", Letter), |
|
349 |
("\\<sigma>", Letter), |
|
350 |
("\\<tau>", Letter), |
|
351 |
("\\<upsilon>", Letter), |
|
352 |
("\\<phi>", Letter), |
|
25521 | 353 |
("\\<chi>", Letter), |
14678 | 354 |
("\\<psi>", Letter), |
355 |
("\\<omega>", Letter), |
|
356 |
("\\<Gamma>", Letter), |
|
357 |
("\\<Delta>", Letter), |
|
358 |
("\\<Theta>", Letter), |
|
359 |
("\\<Lambda>", Letter), |
|
360 |
("\\<Xi>", Letter), |
|
361 |
("\\<Pi>", Letter), |
|
362 |
("\\<Sigma>", Letter), |
|
363 |
("\\<Upsilon>", Letter), |
|
364 |
("\\<Phi>", Letter), |
|
365 |
("\\<Psi>", Letter), |
|
366 |
("\\<Omega>", Letter), |
|
14961 | 367 |
("\\<^isub>", Letter), |
368 |
("\\<^isup>", Letter), |
|
14678 | 369 |
("\\<spacespace>", Blank)]; |
14171
0cab06e3bbd0
Extended the notion of letter and digit, such that now one may use greek,
skalberg
parents:
13730
diff
changeset
|
370 |
in |
14678 | 371 |
fun kind s = |
372 |
if is_ascii_letter s then Letter |
|
373 |
else if is_ascii_digit s then Digit |
|
374 |
else if is_ascii_quasi s then Quasi |
|
375 |
else if is_ascii_blank s then Blank |
|
376 |
else if is_char s then Other |
|
18939 | 377 |
else the_default Other (Symtab.lookup symbol_kinds s); |
14678 | 378 |
end; |
14173 | 379 |
|
14678 | 380 |
fun is_letter s = kind s = Letter; |
381 |
fun is_digit s = kind s = Digit; |
|
382 |
fun is_quasi s = kind s = Quasi; |
|
383 |
fun is_blank s = kind s = Blank; |
|
6272 | 384 |
|
47850
c638127b4653
avoid interference of markup for literal tokens, which may contain slightly odd \<^bsub> \<^esub> counted as pseudo-markup (especially relevant for HTML output, e.g. of thm power3_eq_cube);
wenzelm
parents:
43947
diff
changeset
|
385 |
val is_block_ctrl = member (op =) ["\\<^bsub>", "\\<^esub>", "\\<^bsup>", "\\<^esup>"]; |
c638127b4653
avoid interference of markup for literal tokens, which may contain slightly odd \<^bsub> \<^esub> counted as pseudo-markup (especially relevant for HTML output, e.g. of thm power3_eq_cube);
wenzelm
parents:
43947
diff
changeset
|
386 |
|
14678 | 387 |
fun is_quasi_letter s = let val k = kind s in k = Letter orelse k = Quasi end; |
388 |
fun is_letdig s = let val k = kind s in k = Letter orelse k = Digit orelse k = Quasi end; |
|
11010 | 389 |
|
16138 | 390 |
fun is_ident [] = false |
391 |
| is_ident (c :: cs) = is_letter c andalso forall is_letdig cs; |
|
392 |
||
6272 | 393 |
|
394 |
||
14678 | 395 |
(** symbol input **) |
396 |
||
397 |
(* scanning through symbols *) |
|
6272 | 398 |
|
6640 | 399 |
fun scanner msg scan chs = |
400 |
let |
|
43947
9b00f09f7721
defer evaluation of Scan.message, for improved performance in the frequent situation where failure is handled later (e.g. via ||);
wenzelm
parents:
43845
diff
changeset
|
401 |
fun message (cs, NONE) = (fn () => msg ^ ": " ^ quote (beginning 10 cs)) |
9b00f09f7721
defer evaluation of Scan.message, for improved performance in the frequent situation where failure is handled later (e.g. via ||);
wenzelm
parents:
43845
diff
changeset
|
402 |
| message (cs, SOME msg') = (fn () => msg ^ ", " ^ msg' () ^ ": " ^ quote (beginning 10 cs)); |
14961 | 403 |
val fin_scan = Scan.error (Scan.finite stopper (!! message scan)); |
6640 | 404 |
in |
405 |
(case fin_scan chs of |
|
406 |
(result, []) => result |
|
43947
9b00f09f7721
defer evaluation of Scan.message, for improved performance in the frequent situation where failure is handled later (e.g. via ||);
wenzelm
parents:
43845
diff
changeset
|
407 |
| (_, rest) => error (message (rest, NONE) ())) |
6640 | 408 |
end; |
409 |
||
21858
05f57309170c
avoid conflict with Alice keywords: renamed pack -> implode, unpack -> explode, any -> many, avoided assert;
wenzelm
parents:
21495
diff
changeset
|
410 |
val scan_id = Scan.one is_letter ^^ (Scan.many is_letdig >> implode); |
14678 | 411 |
|
23676
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
412 |
|
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
413 |
(* source *) |
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
414 |
|
14678 | 415 |
local |
14561
c53396af770e
* raw control symbols are of the form \<^raw:...> now.
schirmer
parents:
14559
diff
changeset
|
416 |
|
37533
d775bd70f571
explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents:
34095
diff
changeset
|
417 |
fun is_plain s = is_ascii s andalso s <> "\^M" andalso s <> "\\"; |
d775bd70f571
explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents:
34095
diff
changeset
|
418 |
|
d775bd70f571
explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents:
34095
diff
changeset
|
419 |
fun is_utf8_trailer s = is_char s andalso 128 <= ord s andalso ord s < 192; |
23676
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
420 |
|
37728
5d2b3e827371
implode pseudo utf8, i.e. decode byte-stuffed low ASCII characters;
wenzelm
parents:
37534
diff
changeset
|
421 |
fun implode_pseudo_utf8 (cs as ["\192", c]) = |
5d2b3e827371
implode pseudo utf8, i.e. decode byte-stuffed low ASCII characters;
wenzelm
parents:
37534
diff
changeset
|
422 |
if ord c < 160 then chr (ord c - 128) else implode cs |
5d2b3e827371
implode pseudo utf8, i.e. decode byte-stuffed low ASCII characters;
wenzelm
parents:
37534
diff
changeset
|
423 |
| implode_pseudo_utf8 cs = implode cs; |
5d2b3e827371
implode pseudo utf8, i.e. decode byte-stuffed low ASCII characters;
wenzelm
parents:
37534
diff
changeset
|
424 |
|
14678 | 425 |
val scan_encoded_newline = |
17756 | 426 |
$$ "\^M" -- $$ "\n" >> K "\n" || |
427 |
$$ "\^M" >> K "\n" || |
|
31545
5f1f0a20af4d
discontinued escaped symbols such as \\<forall> -- only one backslash should be used;
wenzelm
parents:
31425
diff
changeset
|
428 |
Scan.this_string "\\<^newline>" >> K "\n"; |
14956
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
429 |
|
70ec4b8aef8d
prevent looping of error messages involving malformed symbols;
wenzelm
parents:
14908
diff
changeset
|
430 |
val scan_raw = |
21858
05f57309170c
avoid conflict with Alice keywords: renamed pack -> implode, unpack -> explode, any -> many, avoided assert;
wenzelm
parents:
21495
diff
changeset
|
431 |
Scan.this_string "raw:" ^^ (Scan.many raw_chr >> implode) || |
05f57309170c
avoid conflict with Alice keywords: renamed pack -> implode, unpack -> explode, any -> many, avoided assert;
wenzelm
parents:
21495
diff
changeset
|
432 |
Scan.this_string "raw" ^^ (Scan.many1 is_ascii_digit >> implode); |
14678 | 433 |
|
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
434 |
val scan_total = |
23676
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
435 |
Scan.one is_plain || |
37728
5d2b3e827371
implode pseudo utf8, i.e. decode byte-stuffed low ASCII characters;
wenzelm
parents:
37534
diff
changeset
|
436 |
Scan.one is_utf8 ::: Scan.many is_utf8_trailer >> implode_pseudo_utf8 || |
23676
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
437 |
scan_encoded_newline || |
48773
0e1bab274672
more liberal scanning of potentially malformed symbols;
wenzelm
parents:
48704
diff
changeset
|
438 |
($$ "\\" ^^ $$ "<" ^^ |
0e1bab274672
more liberal scanning of potentially malformed symbols;
wenzelm
parents:
48704
diff
changeset
|
439 |
(($$ "^" ^^ (scan_raw || scan_id) || scan_id) ^^ Scan.optional ($$ ">") "")) || |
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
440 |
Scan.this_string "\\<^" || |
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
441 |
Scan.this_string "\\<" || |
23676
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
442 |
Scan.one not_eof; |
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
443 |
|
14678 | 444 |
in |
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
445 |
|
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
446 |
fun source src = Source.source stopper (Scan.bulk scan_total) NONE src; |
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
447 |
|
14678 | 448 |
end; |
449 |
||
14562
980da32f4617
proper handling of lines terminated by CRLF or CR;
wenzelm
parents:
14561
diff
changeset
|
450 |
|
23676
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
451 |
(* explode *) |
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
452 |
|
23676
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
453 |
local |
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
454 |
|
14562
980da32f4617
proper handling of lines terminated by CRLF or CR;
wenzelm
parents:
14561
diff
changeset
|
455 |
fun no_explode [] = true |
980da32f4617
proper handling of lines terminated by CRLF or CR;
wenzelm
parents:
14561
diff
changeset
|
456 |
| no_explode ("\\" :: "<" :: _) = false |
17756 | 457 |
| no_explode ("\^M" :: _) = false |
37533
d775bd70f571
explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents:
34095
diff
changeset
|
458 |
| no_explode (c :: cs) = is_ascii c andalso no_explode cs; |
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
459 |
|
23676
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
460 |
in |
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
461 |
|
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
462 |
fun sym_explode str = |
40627
becf5d5187cc
renamed raw "explode" function to "raw_explode" to emphasize its meaning;
wenzelm
parents:
40523
diff
changeset
|
463 |
let val chs = raw_explode str in |
14562
980da32f4617
proper handling of lines terminated by CRLF or CR;
wenzelm
parents:
14561
diff
changeset
|
464 |
if no_explode chs then chs |
40523
1050315f6ee2
simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents:
40509
diff
changeset
|
465 |
else Source.exhaust (source (Source.of_list chs)) |
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
466 |
end; |
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
467 |
|
23676
ea9b7e9c2301
scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents:
23618
diff
changeset
|
468 |
end; |
14994 | 469 |
|
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
470 |
|
14977
77d88064991a
added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents:
14961
diff
changeset
|
471 |
(* escape *) |
77d88064991a
added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents:
14961
diff
changeset
|
472 |
|
37534 | 473 |
val esc = fn s => |
474 |
if is_char s then s |
|
475 |
else if is_utf8 s then translate_string (fn c => "\\" ^ string_of_int (ord c)) s |
|
476 |
else "\\" ^ s; |
|
477 |
||
31425 | 478 |
val escape = implode o map esc o sym_explode; |
14977
77d88064991a
added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents:
14961
diff
changeset
|
479 |
|
77d88064991a
added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents:
14961
diff
changeset
|
480 |
|
14678 | 481 |
(* blanks *) |
482 |
||
483 |
fun strip_blanks s = |
|
484 |
sym_explode s |
|
33955 | 485 |
|> take_prefix is_blank |> #2 |
486 |
|> take_suffix is_blank |> #1 |
|
14678 | 487 |
|> implode; |
488 |
||
489 |
||
490 |
(* bump string -- treat as base 26 or base 1 numbers *) |
|
491 |
||
15979 | 492 |
fun symbolic_end (_ :: "\\<^isub>" :: _) = true |
493 |
| symbolic_end (_ :: "\\<^isup>" :: _) = true |
|
14908 | 494 |
| symbolic_end (s :: _) = is_symbolic s |
495 |
| symbolic_end [] = false; |
|
14678 | 496 |
|
497 |
fun bump_init str = |
|
14908 | 498 |
if symbolic_end (rev (sym_explode str)) then str ^ "'" |
14678 | 499 |
else str ^ "a"; |
12904 | 500 |
|
501 |
fun bump_string str = |
|
502 |
let |
|
503 |
fun bump [] = ["a"] |
|
504 |
| bump ("z" :: ss) = "a" :: bump ss |
|
505 |
| bump (s :: ss) = |
|
14678 | 506 |
if is_char s andalso ord "a" <= ord s andalso ord s < ord "z" |
12904 | 507 |
then chr (ord s + 1) :: ss |
508 |
else "a" :: s :: ss; |
|
14678 | 509 |
|
33955 | 510 |
val (ss, qs) = apfst rev (take_suffix is_quasi (sym_explode str)); |
14908 | 511 |
val ss' = if symbolic_end ss then "'" :: ss else bump ss; |
14678 | 512 |
in implode (rev ss' @ qs) end; |
513 |
||
12904 | 514 |
|
6272 | 515 |
|
29324 | 516 |
(** symbol output **) |
14977
77d88064991a
added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents:
14961
diff
changeset
|
517 |
|
29324 | 518 |
(* length *) |
6272 | 519 |
|
14678 | 520 |
fun sym_len s = |
24593
1547ea587f5a
added some int constraints (ML_Parse.fix_ints not active here);
wenzelm
parents:
24580
diff
changeset
|
521 |
if not (is_printable s) then (0: int) |
14678 | 522 |
else if String.isPrefix "\\<long" s then 2 |
523 |
else if String.isPrefix "\\<Long" s then 2 |
|
524 |
else if s = "\\<spacespace>" then 2 |
|
525 |
else 1; |
|
526 |
||
19473 | 527 |
fun sym_length ss = fold (fn s => fn n => sym_len s + n) ss 0; |
14678 | 528 |
|
29324 | 529 |
|
530 |
(* print mode *) |
|
531 |
||
532 |
val xsymbolsN = "xsymbols"; |
|
533 |
||
534 |
fun output s = (s, sym_length (sym_explode s)); |
|
535 |
||
536 |
||
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
537 |
(*final declarations of this structure!*) |
29324 | 538 |
val explode = sym_explode; |
6272 | 539 |
val length = sym_length; |
6116
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
540 |
|
8ba2f25610f7
files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff
changeset
|
541 |
end; |