src/Pure/General/symbol.ML
author wenzelm
Tue, 13 Aug 2013 17:26:22 +0200
changeset 53016 fa9c38891cf2
parent 52920 4539e4a06339
child 53198 06b096cf89ca
permissions -rw-r--r--
disable old identifier syntax by default, legacy_isub_isup := true may be used temporarily as fall-back;
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
6118
caa439435666 fixed titles;
wenzelm
parents: 6116
diff changeset
     1
(*  Title:      Pure/General/symbol.ML
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
     2
    Author:     Markus Wenzel, TU Muenchen
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
     3
21897
d0c67d715deb fixed typo in comment
paulson
parents: 21858
diff changeset
     4
Generalized characters with infinitely many named symbols.
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
     5
*)
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
     6
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
     7
signature SYMBOL =
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
     8
sig
40509
0bc83ae22789 tuned signatures;
wenzelm
parents: 40131
diff changeset
     9
  type symbol = string
26524
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    10
  val STX: symbol
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    11
  val DEL: symbol
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    12
  val space: symbol
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    13
  val is_char: symbol -> bool
37533
d775bd70f571 explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents: 34095
diff changeset
    14
  val is_utf8: symbol -> bool
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    15
  val is_symbolic: symbol -> bool
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    16
  val is_printable: symbol -> bool
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    17
  val eof: symbol
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    18
  val is_eof: symbol -> bool
27766
1ae745357856 export not_eof;
wenzelm
parents: 27745
diff changeset
    19
  val not_eof: symbol -> bool
27732
8dbf5761a24a abstract type Scan.stopper;
wenzelm
parents: 26632
diff changeset
    20
  val stopper: symbol Scan.stopper
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    21
  val sync: symbol
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    22
  val is_sync: symbol -> bool
23784
75e6b9dd5336 Symbol.not_eof/sync is superceded by Symbol.is_regular (rules out further control symbols);
wenzelm
parents: 23728
diff changeset
    23
  val is_regular: symbol -> bool
40523
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
    24
  val is_malformed: symbol -> bool
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
    25
  val malformed_msg: symbol -> string
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    26
  val is_ascii: symbol -> bool
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    27
  val is_ascii_letter: symbol -> bool
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    28
  val is_ascii_digit: symbol -> bool
24580
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
    29
  val is_ascii_hex: symbol -> bool
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    30
  val is_ascii_quasi: symbol -> bool
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    31
  val is_ascii_blank: symbol -> bool
34095
c2f176a38448 robust representation of low ASCII control characters within XML/YXML text;
wenzelm
parents: 33955
diff changeset
    32
  val is_ascii_control: symbol -> bool
50236
476a3350589c clarified Symbol.scan_ascii_id;
wenzelm
parents: 50235
diff changeset
    33
  val is_ascii_letdig: symbol -> bool
20200
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
    34
  val is_ascii_lower: symbol -> bool
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
    35
  val is_ascii_upper: symbol -> bool
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
    36
  val to_ascii_lower: symbol -> symbol
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
    37
  val to_ascii_upper: symbol -> symbol
50238
98d35a7368bd more uniform Symbol.is_ascii_identifier in ML/Scala;
wenzelm
parents: 50237
diff changeset
    38
  val is_ascii_identifier: string -> bool
50236
476a3350589c clarified Symbol.scan_ascii_id;
wenzelm
parents: 50235
diff changeset
    39
  val scan_ascii_id: string list -> string * string list
14834
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
    40
  val is_raw: symbol -> bool
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
    41
  val decode_raw: symbol -> string
14977
77d88064991a added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents: 14961
diff changeset
    42
  val encode_raw: string -> string
40523
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
    43
  datatype sym =
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
    44
    Char of string | UTF8 of string | Sym of string | Ctrl of string | Raw of string |
43485
33a24212a72d more tolerant Symbol.decode;
wenzelm
parents: 43418
diff changeset
    45
    Malformed of string | EOF
14873
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
    46
  val decode: symbol -> sym
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    47
  datatype kind = Letter | Digit | Quasi | Blank | Other
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    48
  val kind: symbol -> kind
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    49
  val is_letter: symbol -> bool
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    50
  val is_digit: symbol -> bool
12904
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
    51
  val is_quasi: symbol -> bool
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    52
  val is_blank: symbol -> bool
47850
c638127b4653 avoid interference of markup for literal tokens, which may contain slightly odd \<^bsub> \<^esub> counted as pseudo-markup (especially relevant for HTML output, e.g. of thm power3_eq_cube);
wenzelm
parents: 43947
diff changeset
    53
  val is_block_ctrl: symbol -> bool
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    54
  val is_quasi_letter: symbol -> bool
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    55
  val is_letdig: symbol -> bool
14728
wenzelm
parents: 14678
diff changeset
    56
  val beginning: int -> symbol list -> string
40523
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
    57
  val source: (string, 'a) Source.source -> (symbol, (string, 'a) Source.source) Source.source
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    58
  val explode: string -> symbol list
50237
wenzelm
parents: 50236
diff changeset
    59
  val esc: symbol -> string
wenzelm
parents: 50236
diff changeset
    60
  val escape: string -> string
wenzelm
parents: 50236
diff changeset
    61
  val scanner: string -> (string list -> 'a * string list) -> symbol list -> 'a
50162
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
    62
  val split_words: symbol list -> string list
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
    63
  val explode_words: string -> string list
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    64
  val strip_blanks: string -> string
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    65
  val bump_init: string -> string
12904
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
    66
  val bump_string: string -> string
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    67
  val length: symbol list -> int
6692
05c56f41e661 avoid string constants;
wenzelm
parents: 6640
diff changeset
    68
  val xsymbolsN: string
40131
7cbebd636e79 explicitly qualify type Output.output, which is a slightly odd internal feature;
wenzelm
parents: 37728
diff changeset
    69
  val output: string -> Output.output * int
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    70
end;
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    71
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    72
structure Symbol: SYMBOL =
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    73
struct
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    74
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    75
(** type symbol **)
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    76
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    77
(*Symbols, which are considered the smallest entities of any Isabelle
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    78
  string, may be of the following form:
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    79
14834
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
    80
    (1) ASCII symbols: a
17823
4735c07399c8 raw symbols: allow non-ASCII chars (e.g. UTF-8);
wenzelm
parents: 17756
diff changeset
    81
    (2) regular symbols: \<ident>
14834
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
    82
    (3) control symbols: \<^ident>
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
    83
    (4) raw control symbols: \<^raw:...>, where "..." may be any printable
20205
7b2958d3d575 raw symbols: disallow dot to avoid confusion in NameSpace.unpack;
wenzelm
parents: 20200
diff changeset
    84
        character (excluding ".", ">"), or \<^raw000>
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    85
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    86
  Output is subject to the print_mode variable (default: verbatim),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    87
  actual interpretation in display is up to front-end tools.
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    88
*)
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    89
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    90
type symbol = string;
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    91
26524
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    92
val STX = chr 2;
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    93
val DEL = chr 127;
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    94
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    95
val space = chr 32;
17063
502105583951 tuned Symbol.spaces;
wenzelm
parents: 16138
diff changeset
    96
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    97
fun is_char s = size s = 1;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    98
37533
d775bd70f571 explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents: 34095
diff changeset
    99
fun is_utf8 s = size s > 0 andalso forall_string (fn c => ord c >= 128) s;
d775bd70f571 explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents: 34095
diff changeset
   100
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   101
fun is_symbolic s =
40523
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
   102
  String.isPrefix "\\<" s andalso String.isSuffix ">" s andalso not (String.isPrefix "\\<^" s);
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   103
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   104
fun is_printable s =
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   105
  if is_char s then ord space <= ord s andalso ord s <= ord "~"
40523
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
   106
  else is_utf8 s orelse is_symbolic s;
26632
90c0b075c0d3 added is_utf8_trailer;
wenzelm
parents: 26538
diff changeset
   107
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   108
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   109
(* input source control *)
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   110
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   111
val eof = "";
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   112
fun is_eof s = s = eof;
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   113
fun not_eof s = s <> eof;
27732
8dbf5761a24a abstract type Scan.stopper;
wenzelm
parents: 26632
diff changeset
   114
val stopper = Scan.stopper (K eof) is_eof;
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   115
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   116
val sync = "\\<^sync>";
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   117
fun is_sync s = s = sync;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   118
40523
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
   119
fun is_regular s = not_eof s andalso s <> sync;
25641
615aecd485ad added separate_chars;
wenzelm
parents: 25521
diff changeset
   120
48774
c4bd5bb3ae69 further clarification of malformed symbols;
wenzelm
parents: 48773
diff changeset
   121
fun is_malformed s =
c4bd5bb3ae69 further clarification of malformed symbols;
wenzelm
parents: 48773
diff changeset
   122
  String.isPrefix "\\<" s andalso not (String.isSuffix ">" s)
c4bd5bb3ae69 further clarification of malformed symbols;
wenzelm
parents: 48773
diff changeset
   123
  orelse s = "\\<>" orelse s = "\\<^>";
c4bd5bb3ae69 further clarification of malformed symbols;
wenzelm
parents: 48773
diff changeset
   124
40523
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
   125
fun malformed_msg s = "Malformed symbolic character: " ^ quote s;
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   126
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   127
43418
c69e9fbb81a8 recovered markup for non-alphabetic keywords;
wenzelm
parents: 40627
diff changeset
   128
(* ASCII symbols *)
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   129
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   130
fun is_ascii s = is_char s andalso ord s < 128;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   131
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   132
fun is_ascii_letter s =
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   133
  is_char s andalso
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   134
   (ord "A" <= ord s andalso ord s <= ord "Z" orelse
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   135
    ord "a" <= ord s andalso ord s <= ord "z");
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   136
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   137
fun is_ascii_digit s =
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   138
  is_char s andalso ord "0" <= ord s andalso ord s <= ord "9";
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   139
24580
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
   140
fun is_ascii_hex s =
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
   141
  is_char s andalso
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
   142
   (ord "0" <= ord s andalso ord s <= ord "9" orelse
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
   143
    ord "A" <= ord s andalso ord s <= ord "F" orelse
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
   144
    ord "a" <= ord s andalso ord s <= ord "f");
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
   145
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   146
fun is_ascii_quasi "_" = true
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   147
  | is_ascii_quasi "'" = true
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   148
  | is_ascii_quasi _ = false;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   149
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   150
val is_ascii_blank =
43845
d89353d17f54 added File.fold_pages for streaming of large files;
wenzelm
parents: 43777
diff changeset
   151
  fn " " => true | "\t" => true | "\n" => true | "\^K" => true | "\f" => true | "\^M" => true
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   152
    | _ => false;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   153
34095
c2f176a38448 robust representation of low ASCII control characters within XML/YXML text;
wenzelm
parents: 33955
diff changeset
   154
fun is_ascii_control s = is_char s andalso ord s < 32 andalso not (is_ascii_blank s);
c2f176a38448 robust representation of low ASCII control characters within XML/YXML text;
wenzelm
parents: 33955
diff changeset
   155
50236
476a3350589c clarified Symbol.scan_ascii_id;
wenzelm
parents: 50235
diff changeset
   156
fun is_ascii_letdig s = is_ascii_letter s orelse is_ascii_digit s orelse is_ascii_quasi s;
476a3350589c clarified Symbol.scan_ascii_id;
wenzelm
parents: 50235
diff changeset
   157
20200
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
   158
fun is_ascii_lower s = is_char s andalso (ord "a" <= ord s andalso ord s <= ord "z");
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
   159
fun is_ascii_upper s = is_char s andalso (ord "A" <= ord s andalso ord s <= ord "Z");
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
   160
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
   161
fun to_ascii_lower s = if is_ascii_upper s then chr (ord s + ord "a" - ord "A") else s;
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
   162
fun to_ascii_upper s = if is_ascii_lower s then chr (ord s + ord "A" - ord "a") else s;
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
   163
50238
98d35a7368bd more uniform Symbol.is_ascii_identifier in ML/Scala;
wenzelm
parents: 50237
diff changeset
   164
fun is_ascii_identifier s =
98d35a7368bd more uniform Symbol.is_ascii_identifier in ML/Scala;
wenzelm
parents: 50237
diff changeset
   165
  size s > 0 andalso is_ascii_letter (String.substring (s, 0, 1)) andalso
98d35a7368bd more uniform Symbol.is_ascii_identifier in ML/Scala;
wenzelm
parents: 50237
diff changeset
   166
  forall_string is_ascii_letdig s;
98d35a7368bd more uniform Symbol.is_ascii_identifier in ML/Scala;
wenzelm
parents: 50237
diff changeset
   167
50236
476a3350589c clarified Symbol.scan_ascii_id;
wenzelm
parents: 50235
diff changeset
   168
val scan_ascii_id = Scan.one is_ascii_letter ^^ (Scan.many is_ascii_letdig >> implode);
476a3350589c clarified Symbol.scan_ascii_id;
wenzelm
parents: 50235
diff changeset
   169
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   170
14956
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   171
(* encode_raw *)
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   172
20205
7b2958d3d575 raw symbols: disallow dot to avoid confusion in NameSpace.unpack;
wenzelm
parents: 20200
diff changeset
   173
fun raw_chr c =
48773
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
   174
  is_char c andalso
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
   175
    (ord space <= ord c andalso ord c <= ord "~" andalso c <> "." andalso c <> ">"
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
   176
      orelse ord c >= 128);
14956
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   177
29324
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   178
fun encode_raw "" = ""
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   179
  | encode_raw str =
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   180
      let
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   181
        val raw0 = enclose "\\<^raw:" ">";
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   182
        val raw1 = raw0 o implode;
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   183
        val raw2 = enclose "\\<^raw" ">" o string_of_int o ord;
50162
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
   184
33955
fff6f11b1f09 curried take/drop
haftmann
parents: 33548
diff changeset
   185
        fun encode cs = enc (take_prefix raw_chr cs)
29324
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   186
        and enc ([], []) = []
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   187
          | enc (cs, []) = [raw1 cs]
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   188
          | enc ([], d :: ds) = raw2 d :: encode ds
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   189
          | enc (cs, d :: ds) = raw1 cs :: raw2 d :: encode ds;
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   190
      in
40627
becf5d5187cc renamed raw "explode" function to "raw_explode" to emphasize its meaning;
wenzelm
parents: 40523
diff changeset
   191
        if exists_string (not o raw_chr) str then implode (encode (raw_explode str))
29324
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   192
        else raw0 str
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   193
      end;
14956
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   194
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   195
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   196
(* diagnostics *)
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   197
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   198
fun beginning n cs =
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   199
  let
33955
fff6f11b1f09 curried take/drop
haftmann
parents: 33548
diff changeset
   200
    val drop_blanks = #1 o take_suffix is_ascii_blank;
14956
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   201
    val all_cs = drop_blanks cs;
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   202
    val dots = if length all_cs > n then " ..." else "";
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   203
  in
33955
fff6f11b1f09 curried take/drop
haftmann
parents: 33548
diff changeset
   204
    (drop_blanks (take n all_cs)
14956
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   205
      |> map (fn c => if is_ascii_blank c then space else c)
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   206
      |> implode) ^ dots
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   207
  end;
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   208
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   209
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   210
(* decode_raw *)
14834
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   211
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   212
fun is_raw s =
17063
502105583951 tuned Symbol.spaces;
wenzelm
parents: 16138
diff changeset
   213
  String.isPrefix "\\<^raw" s andalso String.isSuffix ">" s;
14834
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   214
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   215
fun decode_raw s =
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   216
  if not (is_raw s) then error (malformed_msg s)
14834
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   217
  else if String.isPrefix "\\<^raw:" s then String.substring (s, 7, size s - 8)
40627
becf5d5187cc renamed raw "explode" function to "raw_explode" to emphasize its meaning;
wenzelm
parents: 40523
diff changeset
   218
  else chr (#1 (Library.read_int (raw_explode (String.substring (s, 6, size s - 7)))));
14834
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   219
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   220
14873
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   221
(* symbol variants *)
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   222
37533
d775bd70f571 explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents: 34095
diff changeset
   223
datatype sym =
40523
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
   224
  Char of string | UTF8 of string | Sym of string | Ctrl of string | Raw of string |
43485
33a24212a72d more tolerant Symbol.decode;
wenzelm
parents: 43418
diff changeset
   225
  Malformed of string | EOF;
14873
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   226
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   227
fun decode s =
43485
33a24212a72d more tolerant Symbol.decode;
wenzelm
parents: 43418
diff changeset
   228
  if s = "" then EOF
33a24212a72d more tolerant Symbol.decode;
wenzelm
parents: 43418
diff changeset
   229
  else if is_char s then Char s
37533
d775bd70f571 explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents: 34095
diff changeset
   230
  else if is_utf8 s then UTF8 s
14873
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   231
  else if is_raw s then Raw (decode_raw s)
40523
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
   232
  else if is_malformed s then Malformed s
14873
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   233
  else if String.isPrefix "\\<^" s then Ctrl (String.substring (s, 3, size s - 4))
40523
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
   234
  else Sym (String.substring (s, 2, size s - 3));
14873
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   235
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   236
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   237
(* standard symbol kinds *)
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   238
14171
0cab06e3bbd0 Extended the notion of letter and digit, such that now one may use greek,
skalberg
parents: 13730
diff changeset
   239
local
50235
wenzelm
parents: 50233
diff changeset
   240
  val letter_symbols =
wenzelm
parents: 50233
diff changeset
   241
    Symtab.make_set [
wenzelm
parents: 50233
diff changeset
   242
      "\\<A>",
wenzelm
parents: 50233
diff changeset
   243
      "\\<B>",
wenzelm
parents: 50233
diff changeset
   244
      "\\<C>",
wenzelm
parents: 50233
diff changeset
   245
      "\\<D>",
wenzelm
parents: 50233
diff changeset
   246
      "\\<E>",
wenzelm
parents: 50233
diff changeset
   247
      "\\<F>",
wenzelm
parents: 50233
diff changeset
   248
      "\\<G>",
wenzelm
parents: 50233
diff changeset
   249
      "\\<H>",
wenzelm
parents: 50233
diff changeset
   250
      "\\<I>",
wenzelm
parents: 50233
diff changeset
   251
      "\\<J>",
wenzelm
parents: 50233
diff changeset
   252
      "\\<K>",
wenzelm
parents: 50233
diff changeset
   253
      "\\<L>",
wenzelm
parents: 50233
diff changeset
   254
      "\\<M>",
wenzelm
parents: 50233
diff changeset
   255
      "\\<N>",
wenzelm
parents: 50233
diff changeset
   256
      "\\<O>",
wenzelm
parents: 50233
diff changeset
   257
      "\\<P>",
wenzelm
parents: 50233
diff changeset
   258
      "\\<Q>",
wenzelm
parents: 50233
diff changeset
   259
      "\\<R>",
wenzelm
parents: 50233
diff changeset
   260
      "\\<S>",
wenzelm
parents: 50233
diff changeset
   261
      "\\<T>",
wenzelm
parents: 50233
diff changeset
   262
      "\\<U>",
wenzelm
parents: 50233
diff changeset
   263
      "\\<V>",
wenzelm
parents: 50233
diff changeset
   264
      "\\<W>",
wenzelm
parents: 50233
diff changeset
   265
      "\\<X>",
wenzelm
parents: 50233
diff changeset
   266
      "\\<Y>",
wenzelm
parents: 50233
diff changeset
   267
      "\\<Z>",
wenzelm
parents: 50233
diff changeset
   268
      "\\<a>",
wenzelm
parents: 50233
diff changeset
   269
      "\\<b>",
wenzelm
parents: 50233
diff changeset
   270
      "\\<c>",
wenzelm
parents: 50233
diff changeset
   271
      "\\<d>",
wenzelm
parents: 50233
diff changeset
   272
      "\\<e>",
wenzelm
parents: 50233
diff changeset
   273
      "\\<f>",
wenzelm
parents: 50233
diff changeset
   274
      "\\<g>",
wenzelm
parents: 50233
diff changeset
   275
      "\\<h>",
wenzelm
parents: 50233
diff changeset
   276
      "\\<i>",
wenzelm
parents: 50233
diff changeset
   277
      "\\<j>",
wenzelm
parents: 50233
diff changeset
   278
      "\\<k>",
wenzelm
parents: 50233
diff changeset
   279
      "\\<l>",
wenzelm
parents: 50233
diff changeset
   280
      "\\<m>",
wenzelm
parents: 50233
diff changeset
   281
      "\\<n>",
wenzelm
parents: 50233
diff changeset
   282
      "\\<o>",
wenzelm
parents: 50233
diff changeset
   283
      "\\<p>",
wenzelm
parents: 50233
diff changeset
   284
      "\\<q>",
wenzelm
parents: 50233
diff changeset
   285
      "\\<r>",
wenzelm
parents: 50233
diff changeset
   286
      "\\<s>",
wenzelm
parents: 50233
diff changeset
   287
      "\\<t>",
wenzelm
parents: 50233
diff changeset
   288
      "\\<u>",
wenzelm
parents: 50233
diff changeset
   289
      "\\<v>",
wenzelm
parents: 50233
diff changeset
   290
      "\\<w>",
wenzelm
parents: 50233
diff changeset
   291
      "\\<x>",
wenzelm
parents: 50233
diff changeset
   292
      "\\<y>",
wenzelm
parents: 50233
diff changeset
   293
      "\\<z>",
wenzelm
parents: 50233
diff changeset
   294
      "\\<AA>",
wenzelm
parents: 50233
diff changeset
   295
      "\\<BB>",
wenzelm
parents: 50233
diff changeset
   296
      "\\<CC>",
wenzelm
parents: 50233
diff changeset
   297
      "\\<DD>",
wenzelm
parents: 50233
diff changeset
   298
      "\\<EE>",
wenzelm
parents: 50233
diff changeset
   299
      "\\<FF>",
wenzelm
parents: 50233
diff changeset
   300
      "\\<GG>",
wenzelm
parents: 50233
diff changeset
   301
      "\\<HH>",
wenzelm
parents: 50233
diff changeset
   302
      "\\<II>",
wenzelm
parents: 50233
diff changeset
   303
      "\\<JJ>",
wenzelm
parents: 50233
diff changeset
   304
      "\\<KK>",
wenzelm
parents: 50233
diff changeset
   305
      "\\<LL>",
wenzelm
parents: 50233
diff changeset
   306
      "\\<MM>",
wenzelm
parents: 50233
diff changeset
   307
      "\\<NN>",
wenzelm
parents: 50233
diff changeset
   308
      "\\<OO>",
wenzelm
parents: 50233
diff changeset
   309
      "\\<PP>",
wenzelm
parents: 50233
diff changeset
   310
      "\\<QQ>",
wenzelm
parents: 50233
diff changeset
   311
      "\\<RR>",
wenzelm
parents: 50233
diff changeset
   312
      "\\<SS>",
wenzelm
parents: 50233
diff changeset
   313
      "\\<TT>",
wenzelm
parents: 50233
diff changeset
   314
      "\\<UU>",
wenzelm
parents: 50233
diff changeset
   315
      "\\<VV>",
wenzelm
parents: 50233
diff changeset
   316
      "\\<WW>",
wenzelm
parents: 50233
diff changeset
   317
      "\\<XX>",
wenzelm
parents: 50233
diff changeset
   318
      "\\<YY>",
wenzelm
parents: 50233
diff changeset
   319
      "\\<ZZ>",
wenzelm
parents: 50233
diff changeset
   320
      "\\<aa>",
wenzelm
parents: 50233
diff changeset
   321
      "\\<bb>",
wenzelm
parents: 50233
diff changeset
   322
      "\\<cc>",
wenzelm
parents: 50233
diff changeset
   323
      "\\<dd>",
wenzelm
parents: 50233
diff changeset
   324
      "\\<ee>",
wenzelm
parents: 50233
diff changeset
   325
      "\\<ff>",
wenzelm
parents: 50233
diff changeset
   326
      "\\<gg>",
wenzelm
parents: 50233
diff changeset
   327
      "\\<hh>",
wenzelm
parents: 50233
diff changeset
   328
      "\\<ii>",
wenzelm
parents: 50233
diff changeset
   329
      "\\<jj>",
wenzelm
parents: 50233
diff changeset
   330
      "\\<kk>",
wenzelm
parents: 50233
diff changeset
   331
      "\\<ll>",
wenzelm
parents: 50233
diff changeset
   332
      "\\<mm>",
wenzelm
parents: 50233
diff changeset
   333
      "\\<nn>",
wenzelm
parents: 50233
diff changeset
   334
      "\\<oo>",
wenzelm
parents: 50233
diff changeset
   335
      "\\<pp>",
wenzelm
parents: 50233
diff changeset
   336
      "\\<qq>",
wenzelm
parents: 50233
diff changeset
   337
      "\\<rr>",
wenzelm
parents: 50233
diff changeset
   338
      "\\<ss>",
wenzelm
parents: 50233
diff changeset
   339
      "\\<tt>",
wenzelm
parents: 50233
diff changeset
   340
      "\\<uu>",
wenzelm
parents: 50233
diff changeset
   341
      "\\<vv>",
wenzelm
parents: 50233
diff changeset
   342
      "\\<ww>",
wenzelm
parents: 50233
diff changeset
   343
      "\\<xx>",
wenzelm
parents: 50233
diff changeset
   344
      "\\<yy>",
wenzelm
parents: 50233
diff changeset
   345
      "\\<zz>",
wenzelm
parents: 50233
diff changeset
   346
      "\\<alpha>",
wenzelm
parents: 50233
diff changeset
   347
      "\\<beta>",
wenzelm
parents: 50233
diff changeset
   348
      "\\<gamma>",
wenzelm
parents: 50233
diff changeset
   349
      "\\<delta>",
wenzelm
parents: 50233
diff changeset
   350
      "\\<epsilon>",
wenzelm
parents: 50233
diff changeset
   351
      "\\<zeta>",
wenzelm
parents: 50233
diff changeset
   352
      "\\<eta>",
wenzelm
parents: 50233
diff changeset
   353
      "\\<theta>",
wenzelm
parents: 50233
diff changeset
   354
      "\\<iota>",
wenzelm
parents: 50233
diff changeset
   355
      "\\<kappa>",
wenzelm
parents: 50233
diff changeset
   356
      (*"\\<lambda>", sic!*)
wenzelm
parents: 50233
diff changeset
   357
      "\\<mu>",
wenzelm
parents: 50233
diff changeset
   358
      "\\<nu>",
wenzelm
parents: 50233
diff changeset
   359
      "\\<xi>",
wenzelm
parents: 50233
diff changeset
   360
      "\\<pi>",
wenzelm
parents: 50233
diff changeset
   361
      "\\<rho>",
wenzelm
parents: 50233
diff changeset
   362
      "\\<sigma>",
wenzelm
parents: 50233
diff changeset
   363
      "\\<tau>",
wenzelm
parents: 50233
diff changeset
   364
      "\\<upsilon>",
wenzelm
parents: 50233
diff changeset
   365
      "\\<phi>",
wenzelm
parents: 50233
diff changeset
   366
      "\\<chi>",
wenzelm
parents: 50233
diff changeset
   367
      "\\<psi>",
wenzelm
parents: 50233
diff changeset
   368
      "\\<omega>",
wenzelm
parents: 50233
diff changeset
   369
      "\\<Gamma>",
wenzelm
parents: 50233
diff changeset
   370
      "\\<Delta>",
wenzelm
parents: 50233
diff changeset
   371
      "\\<Theta>",
wenzelm
parents: 50233
diff changeset
   372
      "\\<Lambda>",
wenzelm
parents: 50233
diff changeset
   373
      "\\<Xi>",
wenzelm
parents: 50233
diff changeset
   374
      "\\<Pi>",
wenzelm
parents: 50233
diff changeset
   375
      "\\<Sigma>",
wenzelm
parents: 50233
diff changeset
   376
      "\\<Upsilon>",
wenzelm
parents: 50233
diff changeset
   377
      "\\<Phi>",
wenzelm
parents: 50233
diff changeset
   378
      "\\<Psi>",
52616
3ac2878764f9 more robust identifier syntax: sub/superscript counts as modifier of LETDIG part instead of LETTER, both isub/isup and sub/sup are allowed;
wenzelm
parents: 50242
diff changeset
   379
      "\\<Omega>"
50235
wenzelm
parents: 50233
diff changeset
   380
    ];
14171
0cab06e3bbd0 Extended the notion of letter and digit, such that now one may use greek,
skalberg
parents: 13730
diff changeset
   381
in
50242
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   382
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   383
val is_letter_symbol = Symtab.defined letter_symbols;
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   384
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   385
end;
14173
a3690aeb79d4 Removed the extended digits again.
skalberg
parents: 14171
diff changeset
   386
50242
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   387
datatype kind = Letter | Digit | Quasi | Blank | Other;
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   388
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   389
fun kind s =
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   390
  if is_ascii_letter s then Letter
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   391
  else if is_ascii_digit s then Digit
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   392
  else if is_ascii_quasi s then Quasi
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   393
  else if is_ascii_blank s then Blank
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   394
  else if is_char s then Other
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   395
  else if is_letter_symbol s then Letter
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   396
  else Other;
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   397
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   398
fun is_letter s = kind s = Letter;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   399
fun is_digit s = kind s = Digit;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   400
fun is_quasi s = kind s = Quasi;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   401
fun is_blank s = kind s = Blank;
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   402
47850
c638127b4653 avoid interference of markup for literal tokens, which may contain slightly odd \<^bsub> \<^esub> counted as pseudo-markup (especially relevant for HTML output, e.g. of thm power3_eq_cube);
wenzelm
parents: 43947
diff changeset
   403
val is_block_ctrl = member (op =) ["\\<^bsub>", "\\<^esub>", "\\<^bsup>", "\\<^esup>"];
c638127b4653 avoid interference of markup for literal tokens, which may contain slightly odd \<^bsub> \<^esub> counted as pseudo-markup (especially relevant for HTML output, e.g. of thm power3_eq_cube);
wenzelm
parents: 43947
diff changeset
   404
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   405
fun is_quasi_letter s = let val k = kind s in k = Letter orelse k = Quasi end;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   406
fun is_letdig s = let val k = kind s in k = Letter orelse k = Digit orelse k = Quasi end;
11010
2c6559297be3 added strip_blanks;
wenzelm
parents: 10953
diff changeset
   407
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   408
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   409
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   410
(** symbol input **)
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   411
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   412
(* source *)
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   413
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   414
local
14561
c53396af770e * raw control symbols are of the form \<^raw:...> now.
schirmer
parents: 14559
diff changeset
   415
37533
d775bd70f571 explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents: 34095
diff changeset
   416
fun is_plain s = is_ascii s andalso s <> "\^M" andalso s <> "\\";
d775bd70f571 explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents: 34095
diff changeset
   417
d775bd70f571 explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents: 34095
diff changeset
   418
fun is_utf8_trailer s = is_char s andalso 128 <= ord s andalso ord s < 192;
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   419
37728
5d2b3e827371 implode pseudo utf8, i.e. decode byte-stuffed low ASCII characters;
wenzelm
parents: 37534
diff changeset
   420
fun implode_pseudo_utf8 (cs as ["\192", c]) =
5d2b3e827371 implode pseudo utf8, i.e. decode byte-stuffed low ASCII characters;
wenzelm
parents: 37534
diff changeset
   421
      if ord c < 160 then chr (ord c - 128) else implode cs
5d2b3e827371 implode pseudo utf8, i.e. decode byte-stuffed low ASCII characters;
wenzelm
parents: 37534
diff changeset
   422
  | implode_pseudo_utf8 cs = implode cs;
5d2b3e827371 implode pseudo utf8, i.e. decode byte-stuffed low ASCII characters;
wenzelm
parents: 37534
diff changeset
   423
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   424
val scan_encoded_newline =
17756
d4a35f82fbb4 minor tweaks for Poplog/ML;
wenzelm
parents: 17412
diff changeset
   425
  $$ "\^M" -- $$ "\n" >> K "\n" ||
d4a35f82fbb4 minor tweaks for Poplog/ML;
wenzelm
parents: 17412
diff changeset
   426
  $$ "\^M" >> K "\n" ||
31545
5f1f0a20af4d discontinued escaped symbols such as \\<forall> -- only one backslash should be used;
wenzelm
parents: 31425
diff changeset
   427
  Scan.this_string "\\<^newline>" >> K "\n";
14956
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   428
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   429
val scan_raw =
21858
05f57309170c avoid conflict with Alice keywords: renamed pack -> implode, unpack -> explode, any -> many, avoided assert;
wenzelm
parents: 21495
diff changeset
   430
  Scan.this_string "raw:" ^^ (Scan.many raw_chr >> implode) ||
05f57309170c avoid conflict with Alice keywords: renamed pack -> implode, unpack -> explode, any -> many, avoided assert;
wenzelm
parents: 21495
diff changeset
   431
  Scan.this_string "raw" ^^ (Scan.many1 is_ascii_digit >> implode);
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   432
40523
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
   433
val scan_total =
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   434
  Scan.one is_plain ||
37728
5d2b3e827371 implode pseudo utf8, i.e. decode byte-stuffed low ASCII characters;
wenzelm
parents: 37534
diff changeset
   435
  Scan.one is_utf8 ::: Scan.many is_utf8_trailer >> implode_pseudo_utf8 ||
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   436
  scan_encoded_newline ||
48773
0e1bab274672 more liberal scanning of potentially malformed symbols;
wenzelm
parents: 48704
diff changeset
   437
  ($$ "\\" ^^ $$ "<" ^^
50236
476a3350589c clarified Symbol.scan_ascii_id;
wenzelm
parents: 50235
diff changeset
   438
    (($$ "^" ^^ Scan.optional (scan_raw || scan_ascii_id) "" || Scan.optional scan_ascii_id "") ^^
48774
c4bd5bb3ae69 further clarification of malformed symbols;
wenzelm
parents: 48773
diff changeset
   439
      Scan.optional ($$ ">") "")) ||
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   440
  Scan.one not_eof;
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   441
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   442
in
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   443
40523
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
   444
fun source src = Source.source stopper (Scan.bulk scan_total) NONE src;
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   445
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   446
end;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   447
14562
980da32f4617 proper handling of lines terminated by CRLF or CR;
wenzelm
parents: 14561
diff changeset
   448
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   449
(* explode *)
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   450
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   451
local
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   452
14562
980da32f4617 proper handling of lines terminated by CRLF or CR;
wenzelm
parents: 14561
diff changeset
   453
fun no_explode [] = true
980da32f4617 proper handling of lines terminated by CRLF or CR;
wenzelm
parents: 14561
diff changeset
   454
  | no_explode ("\\" :: "<" :: _) = false
17756
d4a35f82fbb4 minor tweaks for Poplog/ML;
wenzelm
parents: 17412
diff changeset
   455
  | no_explode ("\^M" :: _) = false
37533
d775bd70f571 explicit treatment of UTF8 character sequences as Isabelle symbols;
wenzelm
parents: 34095
diff changeset
   456
  | no_explode (c :: cs) = is_ascii c andalso no_explode cs;
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   457
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   458
in
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   459
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   460
fun sym_explode str =
40627
becf5d5187cc renamed raw "explode" function to "raw_explode" to emphasize its meaning;
wenzelm
parents: 40523
diff changeset
   461
  let val chs = raw_explode str in
14562
980da32f4617 proper handling of lines terminated by CRLF or CR;
wenzelm
parents: 14561
diff changeset
   462
    if no_explode chs then chs
40523
1050315f6ee2 simplified/robustified treatment of malformed symbols, which are now fully internalized (total Symbol.explode etc.);
wenzelm
parents: 40509
diff changeset
   463
    else Source.exhaust (source (Source.of_list chs))
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   464
  end;
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   465
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   466
end;
14994
7d8501843146 added chars_only, symbol_output;
wenzelm
parents: 14981
diff changeset
   467
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   468
50237
wenzelm
parents: 50236
diff changeset
   469
(* escape *)
wenzelm
parents: 50236
diff changeset
   470
wenzelm
parents: 50236
diff changeset
   471
val esc = fn s =>
wenzelm
parents: 50236
diff changeset
   472
  if is_char s then s
wenzelm
parents: 50236
diff changeset
   473
  else if is_utf8 s then translate_string (fn c => "\\" ^ string_of_int (ord c)) s
wenzelm
parents: 50236
diff changeset
   474
  else "\\" ^ s;
wenzelm
parents: 50236
diff changeset
   475
wenzelm
parents: 50236
diff changeset
   476
val escape = implode o map esc o sym_explode;
wenzelm
parents: 50236
diff changeset
   477
wenzelm
parents: 50236
diff changeset
   478
wenzelm
parents: 50236
diff changeset
   479
wenzelm
parents: 50236
diff changeset
   480
(** scanning through symbols **)
wenzelm
parents: 50236
diff changeset
   481
wenzelm
parents: 50236
diff changeset
   482
(* scanner *)
wenzelm
parents: 50236
diff changeset
   483
wenzelm
parents: 50236
diff changeset
   484
fun scanner msg scan syms =
wenzelm
parents: 50236
diff changeset
   485
  let
wenzelm
parents: 50236
diff changeset
   486
    fun message (ss, NONE) = (fn () => msg ^ ": " ^ quote (beginning 10 ss))
wenzelm
parents: 50236
diff changeset
   487
      | message (ss, SOME msg') = (fn () => msg ^ ", " ^ msg' () ^ ": " ^ quote (beginning 10 ss));
wenzelm
parents: 50236
diff changeset
   488
    val finite_scan = Scan.error (Scan.finite stopper (!! message scan));
wenzelm
parents: 50236
diff changeset
   489
  in
wenzelm
parents: 50236
diff changeset
   490
    (case finite_scan syms of
wenzelm
parents: 50236
diff changeset
   491
      (result, []) => result
wenzelm
parents: 50236
diff changeset
   492
    | (_, rest) => error (message (rest, NONE) ()))
wenzelm
parents: 50236
diff changeset
   493
  end;
wenzelm
parents: 50236
diff changeset
   494
wenzelm
parents: 50236
diff changeset
   495
50162
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
   496
(* space-separated words *)
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
   497
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
   498
val scan_word =
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
   499
  Scan.many1 is_ascii_blank >> K NONE ||
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
   500
  Scan.many1 (fn s => not (is_ascii_blank s) andalso not_eof s) >> (SOME o implode);
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
   501
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
   502
val split_words = scanner "Bad text" (Scan.repeat scan_word >> map_filter I);
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
   503
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
   504
val explode_words = split_words o sym_explode;
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
   505
e06eabc421e7 some support for breakable text and paragraphs;
wenzelm
parents: 48774
diff changeset
   506
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   507
(* blanks *)
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   508
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   509
fun strip_blanks s =
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   510
  sym_explode s
33955
fff6f11b1f09 curried take/drop
haftmann
parents: 33548
diff changeset
   511
  |> take_prefix is_blank |> #2
fff6f11b1f09 curried take/drop
haftmann
parents: 33548
diff changeset
   512
  |> take_suffix is_blank |> #1
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   513
  |> implode;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   514
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   515
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   516
(* bump string -- treat as base 26 or base 1 numbers *)
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   517
50242
56b9c792a98b support for sub-structured identifier syntax (inactive);
wenzelm
parents: 50239
diff changeset
   518
fun symbolic_end (_ :: "\\<^sub>" :: _) = true
53016
fa9c38891cf2 disable old identifier syntax by default, legacy_isub_isup := true may be used temporarily as fall-back;
wenzelm
parents: 52920
diff changeset
   519
  | symbolic_end (_ :: "\\<^isub>" :: _) = true  (*legacy*)
fa9c38891cf2 disable old identifier syntax by default, legacy_isub_isup := true may be used temporarily as fall-back;
wenzelm
parents: 52920
diff changeset
   520
  | symbolic_end (_ :: "\\<^isup>" :: _) = true  (*legacy*)
14908
wenzelm
parents: 14873
diff changeset
   521
  | symbolic_end (s :: _) = is_symbolic s
wenzelm
parents: 14873
diff changeset
   522
  | symbolic_end [] = false;
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   523
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   524
fun bump_init str =
14908
wenzelm
parents: 14873
diff changeset
   525
  if symbolic_end (rev (sym_explode str)) then str ^ "'"
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   526
  else str ^ "a";
12904
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   527
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   528
fun bump_string str =
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   529
  let
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   530
    fun bump [] = ["a"]
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   531
      | bump ("z" :: ss) = "a" :: bump ss
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   532
      | bump (s :: ss) =
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   533
          if is_char s andalso ord "a" <= ord s andalso ord s < ord "z"
12904
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   534
          then chr (ord s + 1) :: ss
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   535
          else "a" :: s :: ss;
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   536
33955
fff6f11b1f09 curried take/drop
haftmann
parents: 33548
diff changeset
   537
    val (ss, qs) = apfst rev (take_suffix is_quasi (sym_explode str));
14908
wenzelm
parents: 14873
diff changeset
   538
    val ss' = if symbolic_end ss then "'" :: ss else bump ss;
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   539
  in implode (rev ss' @ qs) end;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   540
12904
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   541
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   542
29324
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   543
(** symbol output **)
14977
77d88064991a added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents: 14961
diff changeset
   544
29324
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   545
(* length *)
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   546
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   547
fun sym_len s =
24593
1547ea587f5a added some int constraints (ML_Parse.fix_ints not active here);
wenzelm
parents: 24580
diff changeset
   548
  if not (is_printable s) then (0: int)
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   549
  else if String.isPrefix "\\<long" s then 2
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   550
  else if String.isPrefix "\\<Long" s then 2
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   551
  else 1;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   552
19473
wenzelm
parents: 19305
diff changeset
   553
fun sym_length ss = fold (fn s => fn n => sym_len s + n) ss 0;
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   554
29324
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   555
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   556
(* print mode *)
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   557
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   558
val xsymbolsN = "xsymbols";
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   559
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   560
fun output s = (s, sym_length (sym_explode s));
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   561
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   562
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   563
(*final declarations of this structure!*)
29324
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   564
val explode = sym_explode;
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   565
val length = sym_length;
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   566
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   567
end;