src/Pure/General/symbol.ML
author wenzelm
Fri, 02 Jan 2009 22:52:42 +0100
changeset 29324 e07fc65e296b
parent 27903 af1b39debf30
child 29606 fedb8be05f24
permissions -rw-r--r--
added output; improved encode_raw: map empty to empty; tuned;
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
6118
caa439435666 fixed titles;
wenzelm
parents: 6116
diff changeset
     1
(*  Title:      Pure/General/symbol.ML
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
     2
    ID:         $Id$
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
     3
    Author:     Markus Wenzel, TU Muenchen
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
     4
21897
d0c67d715deb fixed typo in comment
paulson
parents: 21858
diff changeset
     5
Generalized characters with infinitely many named symbols.
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
     6
*)
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
     7
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
     8
signature SYMBOL =
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
     9
sig
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    10
  type symbol
26524
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    11
  val SOH: symbol
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    12
  val STX: symbol
26538
d65504ffb47d replaced ETX/EOT by ENQ/ACK, which are less likely to be interpreted by tty etc.;
wenzelm
parents: 26524
diff changeset
    13
  val ENQ: symbol
d65504ffb47d replaced ETX/EOT by ENQ/ACK, which are less likely to be interpreted by tty etc.;
wenzelm
parents: 26524
diff changeset
    14
  val ACK: symbol
26524
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    15
  val DEL: symbol
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    16
  val space: symbol
23618
32ee8cac5c02 simplified output mode setup;
wenzelm
parents: 21897
diff changeset
    17
  val spaces: int -> string
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    18
  val is_char: symbol -> bool
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    19
  val is_symbolic: symbol -> bool
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    20
  val is_printable: symbol -> bool
26632
90c0b075c0d3 added is_utf8_trailer;
wenzelm
parents: 26538
diff changeset
    21
  val is_utf8_trailer: symbol -> bool
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    22
  val eof: symbol
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    23
  val is_eof: symbol -> bool
27766
1ae745357856 export not_eof;
wenzelm
parents: 27745
diff changeset
    24
  val not_eof: symbol -> bool
27732
8dbf5761a24a abstract type Scan.stopper;
wenzelm
parents: 26632
diff changeset
    25
  val stopper: symbol Scan.stopper
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    26
  val sync: symbol
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    27
  val is_sync: symbol -> bool
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    28
  val malformed: symbol
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
    29
  val end_malformed: symbol
25641
615aecd485ad added separate_chars;
wenzelm
parents: 25521
diff changeset
    30
  val separate_chars: string -> string
23784
75e6b9dd5336 Symbol.not_eof/sync is superceded by Symbol.is_regular (rules out further control symbols);
wenzelm
parents: 23728
diff changeset
    31
  val is_regular: symbol -> bool
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    32
  val is_ascii: symbol -> bool
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    33
  val is_ascii_letter: symbol -> bool
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    34
  val is_ascii_digit: symbol -> bool
24580
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
    35
  val is_ascii_hex: symbol -> bool
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    36
  val is_ascii_quasi: symbol -> bool
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    37
  val is_ascii_blank: symbol -> bool
20200
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
    38
  val is_ascii_lower: symbol -> bool
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
    39
  val is_ascii_upper: symbol -> bool
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
    40
  val to_ascii_lower: symbol -> symbol
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
    41
  val to_ascii_upper: symbol -> symbol
14834
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
    42
  val is_raw: symbol -> bool
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
    43
  val decode_raw: symbol -> string
14977
77d88064991a added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents: 14961
diff changeset
    44
  val encode_raw: string -> string
14873
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
    45
  datatype sym = Char of string | Sym of string | Ctrl of string | Raw of string
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
    46
  val decode: symbol -> sym
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    47
  datatype kind = Letter | Digit | Quasi | Blank | Other
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    48
  val kind: symbol -> kind
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    49
  val is_letter: symbol -> bool
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    50
  val is_digit: symbol -> bool
12904
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
    51
  val is_quasi: symbol -> bool
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    52
  val is_blank: symbol -> bool
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    53
  val is_quasi_letter: symbol -> bool
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    54
  val is_letdig: symbol -> bool
16138
727c5e32930e added is_ident (from Syntax/lexicon.ML);
wenzelm
parents: 16002
diff changeset
    55
  val is_ident: symbol list -> bool
14728
wenzelm
parents: 14678
diff changeset
    56
  val beginning: int -> symbol list -> string
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    57
  val scanner: string -> (string list -> 'a * string list) -> symbol list -> 'a
13730
09aeb7346d3f default_output now escapes \'s more carefully.
berghofe
parents: 13559
diff changeset
    58
  val scan_id: string list -> string * string list
27835
ff8b8513965a Symbol.source/OuterLex.source: more explicit do_recover argument;
wenzelm
parents: 27766
diff changeset
    59
  val source: {do_recover: bool} -> (string, 'a) Source.source ->
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    60
    (symbol, (string, 'a) Source.source) Source.source
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    61
  val explode: string -> symbol list
14977
77d88064991a added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents: 14961
diff changeset
    62
  val escape: string -> string
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    63
  val strip_blanks: string -> string
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    64
  val bump_init: string -> string
12904
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
    65
  val bump_string: string -> string
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    66
  val length: symbol list -> int
6692
05c56f41e661 avoid string constants;
wenzelm
parents: 6640
diff changeset
    67
  val xsymbolsN: string
29324
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
    68
  val output: string -> output * int
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    69
end;
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    70
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    71
structure Symbol: SYMBOL =
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    72
struct
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
    73
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    74
(** type symbol **)
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    75
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    76
(*Symbols, which are considered the smallest entities of any Isabelle
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    77
  string, may be of the following form:
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    78
14834
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
    79
    (1) ASCII symbols: a
17823
4735c07399c8 raw symbols: allow non-ASCII chars (e.g. UTF-8);
wenzelm
parents: 17756
diff changeset
    80
    (2) regular symbols: \<ident>
14834
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
    81
    (3) control symbols: \<^ident>
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
    82
    (4) raw control symbols: \<^raw:...>, where "..." may be any printable
20205
7b2958d3d575 raw symbols: disallow dot to avoid confusion in NameSpace.unpack;
wenzelm
parents: 20200
diff changeset
    83
        character (excluding ".", ">"), or \<^raw000>
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    84
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    85
  Output is subject to the print_mode variable (default: verbatim),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
    86
  actual interpretation in display is up to front-end tools.
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    87
*)
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    88
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    89
type symbol = string;
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
    90
26524
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    91
val SOH = chr 1;
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    92
val STX = chr 2;
26538
d65504ffb47d replaced ETX/EOT by ENQ/ACK, which are less likely to be interpreted by tty etc.;
wenzelm
parents: 26524
diff changeset
    93
val ENQ = chr 5;
d65504ffb47d replaced ETX/EOT by ENQ/ACK, which are less likely to be interpreted by tty etc.;
wenzelm
parents: 26524
diff changeset
    94
val ACK = chr 6;
26524
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    95
val DEL = chr 127;
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    96
63953bec4c98 added some ASCII control symbols;
wenzelm
parents: 25644
diff changeset
    97
val space = chr 32;
17063
502105583951 tuned Symbol.spaces;
wenzelm
parents: 16138
diff changeset
    98
502105583951 tuned Symbol.spaces;
wenzelm
parents: 16138
diff changeset
    99
local
502105583951 tuned Symbol.spaces;
wenzelm
parents: 16138
diff changeset
   100
  val small_spaces = Vector.tabulate (65, fn i => Library.replicate_string i space);
502105583951 tuned Symbol.spaces;
wenzelm
parents: 16138
diff changeset
   101
in
502105583951 tuned Symbol.spaces;
wenzelm
parents: 16138
diff changeset
   102
  fun spaces k =
502105583951 tuned Symbol.spaces;
wenzelm
parents: 16138
diff changeset
   103
    if k < 64 then Vector.sub (small_spaces, k)
502105583951 tuned Symbol.spaces;
wenzelm
parents: 16138
diff changeset
   104
    else Library.replicate_string (k div 64) (Vector.sub (small_spaces, 64)) ^
502105583951 tuned Symbol.spaces;
wenzelm
parents: 16138
diff changeset
   105
      Vector.sub (small_spaces, k mod 64);
502105583951 tuned Symbol.spaces;
wenzelm
parents: 16138
diff changeset
   106
end;
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   107
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   108
fun is_char s = size s = 1;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   109
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   110
fun is_symbolic s =
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   111
  String.isPrefix "\\<" s andalso not (String.isPrefix "\\<^" s);
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   112
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   113
fun is_printable s =
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   114
  if is_char s then ord space <= ord s andalso ord s <= ord "~"
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   115
  else not (String.isPrefix "\\<^" s);
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   116
26632
90c0b075c0d3 added is_utf8_trailer;
wenzelm
parents: 26538
diff changeset
   117
fun is_utf8_trailer s = is_char s andalso 128 <= ord s andalso ord s < 192;
90c0b075c0d3 added is_utf8_trailer;
wenzelm
parents: 26538
diff changeset
   118
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   119
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   120
(* input source control *)
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   121
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   122
val eof = "";
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   123
fun is_eof s = s = eof;
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   124
fun not_eof s = s <> eof;
27732
8dbf5761a24a abstract type Scan.stopper;
wenzelm
parents: 26632
diff changeset
   125
val stopper = Scan.stopper (K eof) is_eof;
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   126
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   127
val sync = "\\<^sync>";
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   128
fun is_sync s = s = sync;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   129
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   130
val malformed = "[[";
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   131
val end_malformed = "]]";
25641
615aecd485ad added separate_chars;
wenzelm
parents: 25521
diff changeset
   132
615aecd485ad added separate_chars;
wenzelm
parents: 25521
diff changeset
   133
val separate_chars = explode #> space_implode space;
615aecd485ad added separate_chars;
wenzelm
parents: 25521
diff changeset
   134
fun malformed_msg s = "Malformed symbolic character: " ^ quote (separate_chars s);
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   135
23784
75e6b9dd5336 Symbol.not_eof/sync is superceded by Symbol.is_regular (rules out further control symbols);
wenzelm
parents: 23728
diff changeset
   136
fun is_regular s =
75e6b9dd5336 Symbol.not_eof/sync is superceded by Symbol.is_regular (rules out further control symbols);
wenzelm
parents: 23728
diff changeset
   137
  not_eof s andalso s <> sync andalso s <> malformed andalso s <> end_malformed;
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   138
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   139
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   140
(* ascii symbols *)
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   141
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   142
fun is_ascii s = is_char s andalso ord s < 128;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   143
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   144
fun is_ascii_letter s =
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   145
  is_char s andalso
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   146
   (ord "A" <= ord s andalso ord s <= ord "Z" orelse
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   147
    ord "a" <= ord s andalso ord s <= ord "z");
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   148
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   149
fun is_ascii_digit s =
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   150
  is_char s andalso ord "0" <= ord s andalso ord s <= ord "9";
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   151
24580
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
   152
fun is_ascii_hex s =
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
   153
  is_char s andalso
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
   154
   (ord "0" <= ord s andalso ord s <= ord "9" orelse
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
   155
    ord "A" <= ord s andalso ord s <= ord "F" orelse
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
   156
    ord "a" <= ord s andalso ord s <= ord "f");
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
   157
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   158
fun is_ascii_quasi "_" = true
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   159
  | is_ascii_quasi "'" = true
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   160
  | is_ascii_quasi _ = false;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   161
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   162
val is_ascii_blank =
24580
916259859344 replaced Symbol.is_hex_letter to Symbol.is_ascii_hex;
wenzelm
parents: 23784
diff changeset
   163
  fn " " => true | "\t" => true | "\n" => true | "\^K" => true | "\^L" => true | "\^M" => true
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   164
    | _ => false;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   165
20200
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
   166
fun is_ascii_lower s = is_char s andalso (ord "a" <= ord s andalso ord s <= ord "z");
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
   167
fun is_ascii_upper s = is_char s andalso (ord "A" <= ord s andalso ord s <= ord "Z");
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
   168
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
   169
fun to_ascii_lower s = if is_ascii_upper s then chr (ord s + ord "a" - ord "A") else s;
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
   170
fun to_ascii_upper s = if is_ascii_lower s then chr (ord s + ord "A" - ord "a") else s;
c6fb50dbbc30 added is/to_ascii_lower/upper;
wenzelm
parents: 20192
diff changeset
   171
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   172
14956
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   173
(* encode_raw *)
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   174
20205
7b2958d3d575 raw symbols: disallow dot to avoid confusion in NameSpace.unpack;
wenzelm
parents: 20200
diff changeset
   175
fun raw_chr c =
7b2958d3d575 raw symbols: disallow dot to avoid confusion in NameSpace.unpack;
wenzelm
parents: 20200
diff changeset
   176
  ord space <= ord c andalso ord c <= ord "~" andalso c <> "." andalso c <> ">"
17823
4735c07399c8 raw symbols: allow non-ASCII chars (e.g. UTF-8);
wenzelm
parents: 17756
diff changeset
   177
  orelse ord c >= 128;
14956
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   178
29324
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   179
fun encode_raw "" = ""
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   180
  | encode_raw str =
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   181
      let
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   182
        val raw0 = enclose "\\<^raw:" ">";
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   183
        val raw1 = raw0 o implode;
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   184
        val raw2 = enclose "\\<^raw" ">" o string_of_int o ord;
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   185
    
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   186
        fun encode cs = enc (Library.take_prefix raw_chr cs)
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   187
        and enc ([], []) = []
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   188
          | enc (cs, []) = [raw1 cs]
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   189
          | enc ([], d :: ds) = raw2 d :: encode ds
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   190
          | enc (cs, d :: ds) = raw1 cs :: raw2 d :: encode ds;
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   191
      in
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   192
        if exists_string (not o raw_chr) str then implode (encode (explode str))
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   193
        else raw0 str
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   194
      end;
14956
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   195
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   196
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   197
(* diagnostics *)
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   198
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   199
fun beginning n cs =
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   200
  let
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   201
    val drop_blanks = #1 o Library.take_suffix is_ascii_blank;
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   202
    val all_cs = drop_blanks cs;
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   203
    val dots = if length all_cs > n then " ..." else "";
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   204
  in
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   205
    (drop_blanks (Library.take (n, all_cs))
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   206
      |> map (fn c => if is_ascii_blank c then space else c)
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   207
      |> implode) ^ dots
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   208
  end;
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   209
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   210
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   211
(* decode_raw *)
14834
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   212
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   213
fun is_raw s =
17063
502105583951 tuned Symbol.spaces;
wenzelm
parents: 16138
diff changeset
   214
  String.isPrefix "\\<^raw" s andalso String.isSuffix ">" s;
14834
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   215
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   216
fun decode_raw s =
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   217
  if not (is_raw s) then error (malformed_msg s)
14834
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   218
  else if String.isPrefix "\\<^raw:" s then String.substring (s, 7, size s - 8)
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   219
  else chr (#1 (Library.read_int (explode (String.substring (s, 6, size s - 7)))));
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   220
e47744683560 improved support for raw symbols;
wenzelm
parents: 14728
diff changeset
   221
14873
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   222
(* symbol variants *)
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   223
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   224
datatype sym = Char of string | Sym of string | Ctrl of string | Raw of string;
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   225
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   226
fun decode s =
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   227
  if is_char s then Char s
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   228
  else if is_raw s then Raw (decode_raw s)
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   229
  else if String.isPrefix "\\<^" s then Ctrl (String.substring (s, 3, size s - 4))
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   230
  else if String.isPrefix "\\<" s then Sym (String.substring (s, 2, size s - 3))
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   231
  else error (malformed_msg s);
14873
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   232
7efc14398e82 added datatype sym, val decode: symbol -> sym;
wenzelm
parents: 14834
diff changeset
   233
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   234
(* standard symbol kinds *)
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   235
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   236
datatype kind = Letter | Digit | Quasi | Blank | Other;
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   237
14171
0cab06e3bbd0 Extended the notion of letter and digit, such that now one may use greek,
skalberg
parents: 13730
diff changeset
   238
local
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   239
  val symbol_kinds = Symtab.make
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   240
   [("\\<A>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   241
    ("\\<B>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   242
    ("\\<C>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   243
    ("\\<D>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   244
    ("\\<E>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   245
    ("\\<F>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   246
    ("\\<G>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   247
    ("\\<H>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   248
    ("\\<I>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   249
    ("\\<J>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   250
    ("\\<K>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   251
    ("\\<L>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   252
    ("\\<M>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   253
    ("\\<N>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   254
    ("\\<O>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   255
    ("\\<P>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   256
    ("\\<Q>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   257
    ("\\<R>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   258
    ("\\<S>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   259
    ("\\<T>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   260
    ("\\<U>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   261
    ("\\<V>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   262
    ("\\<W>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   263
    ("\\<X>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   264
    ("\\<Y>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   265
    ("\\<Z>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   266
    ("\\<a>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   267
    ("\\<b>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   268
    ("\\<c>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   269
    ("\\<d>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   270
    ("\\<e>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   271
    ("\\<f>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   272
    ("\\<g>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   273
    ("\\<h>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   274
    ("\\<i>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   275
    ("\\<j>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   276
    ("\\<k>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   277
    ("\\<l>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   278
    ("\\<m>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   279
    ("\\<n>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   280
    ("\\<o>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   281
    ("\\<p>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   282
    ("\\<q>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   283
    ("\\<r>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   284
    ("\\<s>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   285
    ("\\<t>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   286
    ("\\<u>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   287
    ("\\<v>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   288
    ("\\<w>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   289
    ("\\<x>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   290
    ("\\<y>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   291
    ("\\<z>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   292
    ("\\<AA>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   293
    ("\\<BB>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   294
    ("\\<CC>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   295
    ("\\<DD>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   296
    ("\\<EE>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   297
    ("\\<FF>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   298
    ("\\<GG>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   299
    ("\\<HH>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   300
    ("\\<II>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   301
    ("\\<JJ>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   302
    ("\\<KK>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   303
    ("\\<LL>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   304
    ("\\<MM>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   305
    ("\\<NN>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   306
    ("\\<OO>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   307
    ("\\<PP>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   308
    ("\\<QQ>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   309
    ("\\<RR>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   310
    ("\\<SS>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   311
    ("\\<TT>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   312
    ("\\<UU>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   313
    ("\\<VV>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   314
    ("\\<WW>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   315
    ("\\<XX>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   316
    ("\\<YY>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   317
    ("\\<ZZ>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   318
    ("\\<aa>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   319
    ("\\<bb>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   320
    ("\\<cc>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   321
    ("\\<dd>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   322
    ("\\<ee>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   323
    ("\\<ff>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   324
    ("\\<gg>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   325
    ("\\<hh>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   326
    ("\\<ii>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   327
    ("\\<jj>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   328
    ("\\<kk>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   329
    ("\\<ll>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   330
    ("\\<mm>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   331
    ("\\<nn>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   332
    ("\\<oo>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   333
    ("\\<pp>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   334
    ("\\<qq>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   335
    ("\\<rr>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   336
    ("\\<ss>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   337
    ("\\<tt>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   338
    ("\\<uu>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   339
    ("\\<vv>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   340
    ("\\<ww>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   341
    ("\\<xx>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   342
    ("\\<yy>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   343
    ("\\<zz>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   344
    ("\\<alpha>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   345
    ("\\<beta>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   346
    ("\\<gamma>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   347
    ("\\<delta>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   348
    ("\\<epsilon>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   349
    ("\\<zeta>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   350
    ("\\<eta>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   351
    ("\\<theta>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   352
    ("\\<iota>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   353
    ("\\<kappa>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   354
    ("\\<lambda>", Other),      (*sic!*)
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   355
    ("\\<mu>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   356
    ("\\<nu>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   357
    ("\\<xi>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   358
    ("\\<pi>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   359
    ("\\<rho>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   360
    ("\\<sigma>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   361
    ("\\<tau>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   362
    ("\\<upsilon>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   363
    ("\\<phi>", Letter),
25521
6cebd2ff3ab7 symbol chi is also a letter;
wenzelm
parents: 24593
diff changeset
   364
    ("\\<chi>", Letter),
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   365
    ("\\<psi>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   366
    ("\\<omega>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   367
    ("\\<Gamma>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   368
    ("\\<Delta>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   369
    ("\\<Theta>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   370
    ("\\<Lambda>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   371
    ("\\<Xi>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   372
    ("\\<Pi>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   373
    ("\\<Sigma>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   374
    ("\\<Upsilon>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   375
    ("\\<Phi>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   376
    ("\\<Psi>", Letter),
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   377
    ("\\<Omega>", Letter),
14961
8092a0319927 isub/isup quasi letter (again); tuned;
wenzelm
parents: 14956
diff changeset
   378
    ("\\<^isub>", Letter),
8092a0319927 isub/isup quasi letter (again); tuned;
wenzelm
parents: 14956
diff changeset
   379
    ("\\<^isup>", Letter),
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   380
    ("\\<spacespace>", Blank)];
14171
0cab06e3bbd0 Extended the notion of letter and digit, such that now one may use greek,
skalberg
parents: 13730
diff changeset
   381
in
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   382
  fun kind s =
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   383
    if is_ascii_letter s then Letter
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   384
    else if is_ascii_digit s then Digit
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   385
    else if is_ascii_quasi s then Quasi
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   386
    else if is_ascii_blank s then Blank
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   387
    else if is_char s then Other
18939
wenzelm
parents: 17823
diff changeset
   388
    else the_default Other (Symtab.lookup symbol_kinds s);
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   389
end;
14173
a3690aeb79d4 Removed the extended digits again.
skalberg
parents: 14171
diff changeset
   390
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   391
fun is_letter s = kind s = Letter;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   392
fun is_digit s = kind s = Digit;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   393
fun is_quasi s = kind s = Quasi;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   394
fun is_blank s = kind s = Blank;
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   395
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   396
fun is_quasi_letter s = let val k = kind s in k = Letter orelse k = Quasi end;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   397
fun is_letdig s = let val k = kind s in k = Letter orelse k = Digit orelse k = Quasi end;
11010
2c6559297be3 added strip_blanks;
wenzelm
parents: 10953
diff changeset
   398
16138
727c5e32930e added is_ident (from Syntax/lexicon.ML);
wenzelm
parents: 16002
diff changeset
   399
fun is_ident [] = false
727c5e32930e added is_ident (from Syntax/lexicon.ML);
wenzelm
parents: 16002
diff changeset
   400
  | is_ident (c :: cs) = is_letter c andalso forall is_letdig cs;
727c5e32930e added is_ident (from Syntax/lexicon.ML);
wenzelm
parents: 16002
diff changeset
   401
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   402
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   403
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   404
(** symbol input **)
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   405
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   406
(* scanning through symbols *)
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   407
6640
d2e8342bf5c3 rearranged order of modules;
wenzelm
parents: 6320
diff changeset
   408
fun scanner msg scan chs =
d2e8342bf5c3 rearranged order of modules;
wenzelm
parents: 6320
diff changeset
   409
  let
15531
08c8dad8e399 Deleted Library.option type.
skalberg
parents: 14994
diff changeset
   410
    fun message (cs, NONE) = msg ^ ": " ^ quote (beginning 10 cs)
08c8dad8e399 Deleted Library.option type.
skalberg
parents: 14994
diff changeset
   411
      | message (cs, SOME msg') = msg ^ ", " ^ msg' ^ ": " ^ quote (beginning 10 cs);
14961
8092a0319927 isub/isup quasi letter (again); tuned;
wenzelm
parents: 14956
diff changeset
   412
    val fin_scan = Scan.error (Scan.finite stopper (!! message scan));
6640
d2e8342bf5c3 rearranged order of modules;
wenzelm
parents: 6320
diff changeset
   413
  in
d2e8342bf5c3 rearranged order of modules;
wenzelm
parents: 6320
diff changeset
   414
    (case fin_scan chs of
d2e8342bf5c3 rearranged order of modules;
wenzelm
parents: 6320
diff changeset
   415
      (result, []) => result
15531
08c8dad8e399 Deleted Library.option type.
skalberg
parents: 14994
diff changeset
   416
    | (_, rest) => error (message (rest, NONE)))
6640
d2e8342bf5c3 rearranged order of modules;
wenzelm
parents: 6320
diff changeset
   417
  end;
d2e8342bf5c3 rearranged order of modules;
wenzelm
parents: 6320
diff changeset
   418
21858
05f57309170c avoid conflict with Alice keywords: renamed pack -> implode, unpack -> explode, any -> many, avoided assert;
wenzelm
parents: 21495
diff changeset
   419
val scan_id = Scan.one is_letter ^^ (Scan.many is_letdig >> implode);
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   420
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   421
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   422
(* source *)
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   423
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   424
local
14561
c53396af770e * raw control symbols are of the form \<^raw:...> now.
schirmer
parents: 14559
diff changeset
   425
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   426
fun is_plain s = s <> "\^M" andalso s <> "\\" andalso not_eof s;
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   427
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   428
val scan_encoded_newline =
17756
d4a35f82fbb4 minor tweaks for Poplog/ML;
wenzelm
parents: 17412
diff changeset
   429
  $$ "\^M" -- $$ "\n" >> K "\n" ||
d4a35f82fbb4 minor tweaks for Poplog/ML;
wenzelm
parents: 17412
diff changeset
   430
  $$ "\^M" >> K "\n" ||
14956
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   431
  $$ "\\" -- Scan.optional ($$ "\\") "" -- Scan.this_string "<^newline>" >> K "\n";
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   432
70ec4b8aef8d prevent looping of error messages involving malformed symbols;
wenzelm
parents: 14908
diff changeset
   433
val scan_raw =
21858
05f57309170c avoid conflict with Alice keywords: renamed pack -> implode, unpack -> explode, any -> many, avoided assert;
wenzelm
parents: 21495
diff changeset
   434
  Scan.this_string "raw:" ^^ (Scan.many raw_chr >> implode) ||
05f57309170c avoid conflict with Alice keywords: renamed pack -> implode, unpack -> explode, any -> many, avoided assert;
wenzelm
parents: 21495
diff changeset
   435
  Scan.this_string "raw" ^^ (Scan.many1 is_ascii_digit >> implode);
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   436
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   437
val scan =
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   438
  Scan.one is_plain ||
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   439
  scan_encoded_newline ||
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   440
  (($$ "\\" --| Scan.optional ($$ "\\") "") ^^ $$ "<" ^^
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   441
    !! (fn (cs, _) => malformed_msg (beginning 10 ("\\" :: "<" :: cs)))
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   442
      (($$ "^" ^^ (scan_raw || scan_id) || scan_id) ^^ $$ ">")) ||
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   443
  Scan.one not_eof;
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   444
27745
31899d977a89 improved recover: also resync on symbol/comment/verbatim delimiters;
wenzelm
parents: 27732
diff changeset
   445
val scan_resync =
31899d977a89 improved recover: also resync on symbol/comment/verbatim delimiters;
wenzelm
parents: 27732
diff changeset
   446
  Scan.one is_ascii_blank || $$ "\"" || $$ "`" || $$ "\\" ||
31899d977a89 improved recover: also resync on symbol/comment/verbatim delimiters;
wenzelm
parents: 27732
diff changeset
   447
  Scan.this_string "(*" || Scan.this_string "*)" ||
31899d977a89 improved recover: also resync on symbol/comment/verbatim delimiters;
wenzelm
parents: 27732
diff changeset
   448
  Scan.this_string "{*" || Scan.this_string "*}";
31899d977a89 improved recover: also resync on symbol/comment/verbatim delimiters;
wenzelm
parents: 27732
diff changeset
   449
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   450
val recover =
27903
af1b39debf30 scan: proper recovery for escaped \\< symbols;
wenzelm
parents: 27835
diff changeset
   451
  (Scan.this (explode "\\\\<") || Scan.this (explode "\\<")) @@@
27745
31899d977a89 improved recover: also resync on symbol/comment/verbatim delimiters;
wenzelm
parents: 27732
diff changeset
   452
    Scan.repeat (Scan.unless scan_resync (Scan.one not_eof))
31899d977a89 improved recover: also resync on symbol/comment/verbatim delimiters;
wenzelm
parents: 27732
diff changeset
   453
  >> (fn ss => malformed :: ss @ [end_malformed]);
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   454
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   455
in
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   456
27835
ff8b8513965a Symbol.source/OuterLex.source: more explicit do_recover argument;
wenzelm
parents: 27766
diff changeset
   457
fun source {do_recover} src =
23682
cf4773532006 nested source: explicit interactive flag for recover avoids duplicate errors;
wenzelm
parents: 23676
diff changeset
   458
  Source.source stopper (Scan.bulk scan)
cf4773532006 nested source: explicit interactive flag for recover avoids duplicate errors;
wenzelm
parents: 23676
diff changeset
   459
    (if do_recover then SOME (false, K recover) else NONE) src;
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   460
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   461
end;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   462
14562
980da32f4617 proper handling of lines terminated by CRLF or CR;
wenzelm
parents: 14561
diff changeset
   463
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   464
(* explode *)
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   465
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   466
local
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   467
14562
980da32f4617 proper handling of lines terminated by CRLF or CR;
wenzelm
parents: 14561
diff changeset
   468
fun no_explode [] = true
980da32f4617 proper handling of lines terminated by CRLF or CR;
wenzelm
parents: 14561
diff changeset
   469
  | no_explode ("\\" :: "<" :: _) = false
17756
d4a35f82fbb4 minor tweaks for Poplog/ML;
wenzelm
parents: 17412
diff changeset
   470
  | no_explode ("\^M" :: _) = false
14562
980da32f4617 proper handling of lines terminated by CRLF or CR;
wenzelm
parents: 14561
diff changeset
   471
  | no_explode (_ :: cs) = no_explode cs;
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   472
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   473
in
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   474
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   475
fun sym_explode str =
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   476
  let val chs = explode str in
14562
980da32f4617 proper handling of lines terminated by CRLF or CR;
wenzelm
parents: 14561
diff changeset
   477
    if no_explode chs then chs
27835
ff8b8513965a Symbol.source/OuterLex.source: more explicit do_recover argument;
wenzelm
parents: 27766
diff changeset
   478
    else Source.exhaust (source {do_recover = false} (Source.of_list chs))
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   479
  end;
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   480
23676
ea9b7e9c2301 scan: changed treatment of malformed symbols, passed to next stage;
wenzelm
parents: 23618
diff changeset
   481
end;
14994
7d8501843146 added chars_only, symbol_output;
wenzelm
parents: 14981
diff changeset
   482
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   483
14977
77d88064991a added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents: 14961
diff changeset
   484
(* escape *)
77d88064991a added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents: 14961
diff changeset
   485
77d88064991a added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents: 14961
diff changeset
   486
val escape = implode o map (fn s => if is_char s then s else "\\" ^ s) o sym_explode;
77d88064991a added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents: 14961
diff changeset
   487
77d88064991a added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents: 14961
diff changeset
   488
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   489
(* blanks *)
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   490
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   491
fun strip_blanks s =
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   492
  sym_explode s
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   493
  |> Library.take_prefix is_blank |> #2
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   494
  |> Library.take_suffix is_blank |> #1
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   495
  |> implode;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   496
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   497
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   498
(* bump string -- treat as base 26 or base 1 numbers *)
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   499
15979
wenzelm
parents: 15570
diff changeset
   500
fun symbolic_end (_ :: "\\<^isub>" :: _) = true
wenzelm
parents: 15570
diff changeset
   501
  | symbolic_end (_ :: "\\<^isup>" :: _) = true
14908
wenzelm
parents: 14873
diff changeset
   502
  | symbolic_end (s :: _) = is_symbolic s
wenzelm
parents: 14873
diff changeset
   503
  | symbolic_end [] = false;
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   504
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   505
fun bump_init str =
14908
wenzelm
parents: 14873
diff changeset
   506
  if symbolic_end (rev (sym_explode str)) then str ^ "'"
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   507
  else str ^ "a";
12904
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   508
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   509
fun bump_string str =
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   510
  let
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   511
    fun bump [] = ["a"]
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   512
      | bump ("z" :: ss) = "a" :: bump ss
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   513
      | bump (s :: ss) =
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   514
          if is_char s andalso ord "a" <= ord s andalso ord s < ord "z"
12904
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   515
          then chr (ord s + 1) :: ss
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   516
          else "a" :: s :: ss;
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   517
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   518
    val (ss, qs) = apfst rev (Library.take_suffix is_quasi (sym_explode str));
14908
wenzelm
parents: 14873
diff changeset
   519
    val ss' = if symbolic_end ss then "'" :: ss else bump ss;
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   520
  in implode (rev ss' @ qs) end;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   521
12904
c208d71702d1 added is_quasi;
wenzelm
parents: 12116
diff changeset
   522
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   523
29324
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   524
(** symbol output **)
14977
77d88064991a added escape, export encode_raw, default mode now trivial, tuned;
wenzelm
parents: 14961
diff changeset
   525
29324
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   526
(* length *)
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   527
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   528
fun sym_len s =
24593
1547ea587f5a added some int constraints (ML_Parse.fix_ints not active here);
wenzelm
parents: 24580
diff changeset
   529
  if not (is_printable s) then (0: int)
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   530
  else if String.isPrefix "\\<long" s then 2
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   531
  else if String.isPrefix "\\<Long" s then 2
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   532
  else if s = "\\<spacespace>" then 2
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   533
  else 1;
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   534
19473
wenzelm
parents: 19305
diff changeset
   535
fun sym_length ss = fold (fn s => fn n => sym_len s + n) ss 0;
14678
662b181cae05 clarification of ascii vs. symbolic entities;
wenzelm
parents: 14632
diff changeset
   536
29324
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   537
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   538
(* print mode *)
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   539
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   540
val xsymbolsN = "xsymbols";
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   541
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   542
fun output s = (s, sym_length (sym_explode s));
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   543
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   544
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   545
(*final declarations of this structure!*)
29324
e07fc65e296b added output;
wenzelm
parents: 27903
diff changeset
   546
val explode = sym_explode;
6272
52d99d68d3be added output_width;
wenzelm
parents: 6225
diff changeset
   547
val length = sym_length;
6116
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   548
8ba2f25610f7 files scan.ML, source.ML, symbol.ML, pretty.ML moved to Pure/General;
wenzelm
parents:
diff changeset
   549
end;