src/Pure/Isar/token.ML
author wenzelm
Thu Aug 23 17:46:03 2012 +0200 (2012-08-23)
changeset 48911 5debc3e4fa81
parent 48905 04576657cf94
child 48992 0518bf89c777
permissions -rw-r--r--
tuned messages: end-of-input rarely means physical end-of-file from the past;
wenzelm@36959
     1
(*  Title:      Pure/Isar/token.ML
wenzelm@5825
     2
    Author:     Markus Wenzel, TU Muenchen
wenzelm@5825
     3
wenzelm@36959
     4
Outer token syntax for Isabelle/Isar.
wenzelm@5825
     5
*)
wenzelm@5825
     6
wenzelm@36959
     7
signature TOKEN =
wenzelm@5825
     8
sig
wenzelm@36959
     9
  datatype kind =
wenzelm@27814
    10
    Command | Keyword | Ident | LongIdent | SymIdent | Var | TypeIdent | TypeVar |
wenzelm@40290
    11
    Nat | Float | String | AltString | Verbatim | Space | Comment | InternalValue |
wenzelm@40958
    12
    Error of string | Sync | EOF
wenzelm@48905
    13
  type file = {src_path: Path.T, text: string, pos: Position.T}
wenzelm@27814
    14
  datatype value =
wenzelm@27814
    15
    Text of string | Typ of typ | Term of term | Fact of thm list |
wenzelm@48905
    16
    Attribute of morphism -> attribute | Files of file list
wenzelm@36959
    17
  type T
wenzelm@36959
    18
  val str_of_kind: kind -> string
wenzelm@36959
    19
  val position_of: T -> Position.T
wenzelm@36959
    20
  val end_position_of: T -> Position.T
wenzelm@36959
    21
  val pos_of: T -> string
wenzelm@36959
    22
  val eof: T
wenzelm@36959
    23
  val is_eof: T -> bool
wenzelm@36959
    24
  val not_eof: T -> bool
wenzelm@36959
    25
  val not_sync: T -> bool
wenzelm@36959
    26
  val stopper: T Scan.stopper
wenzelm@36959
    27
  val kind_of: T -> kind
wenzelm@36959
    28
  val is_kind: kind -> T -> bool
wenzelm@36959
    29
  val keyword_with: (string -> bool) -> T -> bool
wenzelm@36959
    30
  val ident_with: (string -> bool) -> T -> bool
wenzelm@46811
    31
  val is_command: T -> bool
wenzelm@48867
    32
  val is_name: T -> bool
wenzelm@36959
    33
  val is_proper: T -> bool
wenzelm@36959
    34
  val is_semicolon: T -> bool
wenzelm@36959
    35
  val is_comment: T -> bool
wenzelm@36959
    36
  val is_begin_ignore: T -> bool
wenzelm@36959
    37
  val is_end_ignore: T -> bool
wenzelm@48749
    38
  val is_error: T -> bool
wenzelm@48771
    39
  val is_space: T -> bool
wenzelm@36959
    40
  val is_blank: T -> bool
wenzelm@36959
    41
  val is_newline: T -> bool
wenzelm@36959
    42
  val source_of: T -> string
wenzelm@36959
    43
  val source_position_of: T -> Symbol_Pos.text * Position.T
wenzelm@36959
    44
  val content_of: T -> string
wenzelm@36959
    45
  val unparse: T -> string
wenzelm@36959
    46
  val text_of: T -> string * string
wenzelm@48905
    47
  val get_files: T -> file list option
wenzelm@48905
    48
  val put_files: file list -> T -> T
wenzelm@36959
    49
  val get_value: T -> value option
wenzelm@36959
    50
  val map_value: (value -> value) -> T -> T
wenzelm@36959
    51
  val mk_text: string -> T
wenzelm@36959
    52
  val mk_typ: typ -> T
wenzelm@36959
    53
  val mk_term: term -> T
wenzelm@36959
    54
  val mk_fact: thm list -> T
wenzelm@36959
    55
  val mk_attribute: (morphism -> attribute) -> T
wenzelm@36959
    56
  val assignable: T -> T
wenzelm@36959
    57
  val assign: value option -> T -> unit
wenzelm@36959
    58
  val closure: T -> T
wenzelm@27814
    59
  val ident_or_symbolic: string -> bool
wenzelm@36959
    60
  val source_proper: (T, 'a) Source.source -> (T, (T, 'a) Source.source) Source.source
wenzelm@48741
    61
  val source': {do_recover: bool option} -> (unit -> Scan.lexicon * Scan.lexicon) ->
wenzelm@36959
    62
    (Symbol_Pos.T, 'a) Source.source -> (T, (Symbol_Pos.T, 'a) Source.source) Source.source
wenzelm@48741
    63
  val source: {do_recover: bool option} -> (unit -> Scan.lexicon * Scan.lexicon) ->
wenzelm@36959
    64
    Position.T -> (Symbol.symbol, 'a) Source.source -> (T,
wenzelm@30573
    65
      (Symbol_Pos.T, Position.T * (Symbol.symbol, 'a) Source.source) Source.source) Source.source
wenzelm@36959
    66
  val read_antiq: Scan.lexicon -> (T list -> 'a * T list) -> Symbol_Pos.T list * Position.T -> 'a
wenzelm@5825
    67
end;
wenzelm@5825
    68
wenzelm@36959
    69
structure Token: TOKEN =
wenzelm@5825
    70
struct
wenzelm@5825
    71
wenzelm@5825
    72
(** tokens **)
wenzelm@5825
    73
wenzelm@27814
    74
(* token values *)
wenzelm@27814
    75
wenzelm@27814
    76
(*The value slot assigns an (optional) internal value to a token,
wenzelm@27814
    77
  usually as a side-effect of special scanner setup (see also
wenzelm@27814
    78
  args.ML).  Note that an assignable ref designates an intermediate
wenzelm@27814
    79
  state of internalization -- it is NOT meant to persist.*)
wenzelm@27814
    80
wenzelm@48905
    81
type file = {src_path: Path.T, text: string, pos: Position.T};
wenzelm@48867
    82
wenzelm@27814
    83
datatype value =
wenzelm@27814
    84
  Text of string |
wenzelm@27814
    85
  Typ of typ |
wenzelm@27814
    86
  Term of term |
wenzelm@27814
    87
  Fact of thm list |
wenzelm@48867
    88
  Attribute of morphism -> attribute |
wenzelm@48905
    89
  Files of file list;
wenzelm@27814
    90
wenzelm@27814
    91
datatype slot =
wenzelm@27814
    92
  Slot |
wenzelm@27814
    93
  Value of value option |
wenzelm@32738
    94
  Assignable of value option Unsynchronized.ref;
wenzelm@27814
    95
wenzelm@27814
    96
wenzelm@5825
    97
(* datatype token *)
wenzelm@5825
    98
wenzelm@36959
    99
datatype kind =
wenzelm@27814
   100
  Command | Keyword | Ident | LongIdent | SymIdent | Var | TypeIdent | TypeVar |
wenzelm@40290
   101
  Nat | Float | String | AltString | Verbatim | Space | Comment | InternalValue |
wenzelm@40958
   102
  Error of string | Sync | EOF;
wenzelm@5825
   103
wenzelm@36959
   104
datatype T = Token of (Symbol_Pos.text * Position.range) * (kind * string) * slot;
wenzelm@5825
   105
wenzelm@5825
   106
val str_of_kind =
wenzelm@7026
   107
 fn Command => "command"
wenzelm@7026
   108
  | Keyword => "keyword"
wenzelm@5825
   109
  | Ident => "identifier"
wenzelm@5825
   110
  | LongIdent => "long identifier"
wenzelm@5825
   111
  | SymIdent => "symbolic identifier"
wenzelm@5825
   112
  | Var => "schematic variable"
wenzelm@5825
   113
  | TypeIdent => "type variable"
wenzelm@5825
   114
  | TypeVar => "schematic type variable"
wenzelm@40290
   115
  | Nat => "natural number"
wenzelm@40290
   116
  | Float => "floating-point number"
wenzelm@5825
   117
  | String => "string"
wenzelm@17164
   118
  | AltString => "back-quoted string"
wenzelm@5825
   119
  | Verbatim => "verbatim text"
wenzelm@7682
   120
  | Space => "white space"
wenzelm@7682
   121
  | Comment => "comment text"
wenzelm@27814
   122
  | InternalValue => "internal value"
wenzelm@23729
   123
  | Error _ => "bad input"
wenzelm@23788
   124
  | Sync => "sync marker"
wenzelm@48911
   125
  | EOF => "end-of-input";
wenzelm@5825
   126
wenzelm@5825
   127
wenzelm@27733
   128
(* position *)
wenzelm@5825
   129
wenzelm@27814
   130
fun position_of (Token ((_, (pos, _)), _, _)) = pos;
wenzelm@27814
   131
fun end_position_of (Token ((_, (_, pos)), _, _)) = pos;
wenzelm@27663
   132
wenzelm@5825
   133
val pos_of = Position.str_of o position_of;
wenzelm@5825
   134
wenzelm@5825
   135
wenzelm@27733
   136
(* control tokens *)
wenzelm@27733
   137
wenzelm@27814
   138
fun mk_eof pos = Token (("", (pos, Position.none)), (EOF, ""), Slot);
wenzelm@27733
   139
val eof = mk_eof Position.none;
wenzelm@27733
   140
wenzelm@27814
   141
fun is_eof (Token (_, (EOF, _), _)) = true
wenzelm@27733
   142
  | is_eof _ = false;
wenzelm@27733
   143
wenzelm@27733
   144
val not_eof = not o is_eof;
wenzelm@27733
   145
wenzelm@27814
   146
fun not_sync (Token (_, (Sync, _), _)) = false
wenzelm@27733
   147
  | not_sync _ = true;
wenzelm@27733
   148
wenzelm@27752
   149
val stopper =
wenzelm@27752
   150
  Scan.stopper (fn [] => eof | toks => mk_eof (end_position_of (List.last toks))) is_eof;
wenzelm@27733
   151
wenzelm@27733
   152
wenzelm@5825
   153
(* kind of token *)
wenzelm@5825
   154
wenzelm@27814
   155
fun kind_of (Token (_, (k, _), _)) = k;
wenzelm@27814
   156
fun is_kind k (Token (_, (k', _), _)) = k = k';
wenzelm@5825
   157
wenzelm@46811
   158
val is_command = is_kind Command;
wenzelm@48867
   159
val is_name = is_kind Ident orf is_kind SymIdent orf is_kind String orf is_kind Nat;
wenzelm@46811
   160
wenzelm@27814
   161
fun keyword_with pred (Token (_, (Keyword, x), _)) = pred x
wenzelm@7026
   162
  | keyword_with _ _ = false;
wenzelm@5825
   163
wenzelm@27814
   164
fun ident_with pred (Token (_, (Ident, x), _)) = pred x
wenzelm@16029
   165
  | ident_with _ _ = false;
wenzelm@16029
   166
wenzelm@27814
   167
fun is_proper (Token (_, (Space, _), _)) = false
wenzelm@27814
   168
  | is_proper (Token (_, (Comment, _), _)) = false
wenzelm@5825
   169
  | is_proper _ = true;
wenzelm@5825
   170
wenzelm@27814
   171
fun is_semicolon (Token (_, (Keyword, ";"), _)) = true
wenzelm@9130
   172
  | is_semicolon _ = false;
wenzelm@9130
   173
wenzelm@27814
   174
fun is_comment (Token (_, (Comment, _), _)) = true
wenzelm@17069
   175
  | is_comment _ = false;
wenzelm@17069
   176
wenzelm@27814
   177
fun is_begin_ignore (Token (_, (Comment, "<"), _)) = true
wenzelm@8580
   178
  | is_begin_ignore _ = false;
wenzelm@8580
   179
wenzelm@27814
   180
fun is_end_ignore (Token (_, (Comment, ">"), _)) = true
wenzelm@8580
   181
  | is_end_ignore _ = false;
wenzelm@8580
   182
wenzelm@48749
   183
fun is_error (Token (_, (Error _, _), _)) = true
wenzelm@48749
   184
  | is_error _ = false;
wenzelm@48749
   185
wenzelm@8651
   186
wenzelm@17069
   187
(* blanks and newlines -- space tokens obey lines *)
wenzelm@8651
   188
wenzelm@48771
   189
fun is_space (Token (_, (Space, _), _)) = true
wenzelm@48771
   190
  | is_space _ = false;
wenzelm@48771
   191
wenzelm@27814
   192
fun is_blank (Token (_, (Space, x), _)) = not (String.isSuffix "\n" x)
wenzelm@17069
   193
  | is_blank _ = false;
wenzelm@17069
   194
wenzelm@27814
   195
fun is_newline (Token (_, (Space, x), _)) = String.isSuffix "\n" x
wenzelm@8651
   196
  | is_newline _ = false;
wenzelm@8651
   197
wenzelm@5825
   198
wenzelm@14991
   199
(* token content *)
wenzelm@9155
   200
wenzelm@43731
   201
fun source_of (Token ((source, (pos, _)), (_, x), _)) =
wenzelm@43731
   202
  if YXML.detect x then x
wenzelm@45666
   203
  else
wenzelm@45666
   204
    YXML.string_of
wenzelm@45666
   205
      (XML.Elem (Isabelle_Markup.token (Position.properties_of pos), [XML.Text source]));
wenzelm@25642
   206
wenzelm@27885
   207
fun source_position_of (Token ((source, (pos, _)), _, _)) = (source, pos);
wenzelm@27873
   208
wenzelm@27814
   209
fun content_of (Token (_, (_, x), _)) = x;
wenzelm@27747
   210
wenzelm@27747
   211
wenzelm@27747
   212
(* unparse *)
wenzelm@27747
   213
wenzelm@27814
   214
fun unparse (Token (_, (kind, x), _)) =
wenzelm@14991
   215
  (case kind of
wenzelm@43773
   216
    String => Symbol_Pos.quote_string_qq x
wenzelm@43773
   217
  | AltString => Symbol_Pos.quote_string_bq x
wenzelm@43773
   218
  | Verbatim => enclose "{*" "*}" x
wenzelm@43773
   219
  | Comment => enclose "(*" "*)" x
wenzelm@23729
   220
  | Sync => ""
wenzelm@23729
   221
  | EOF => ""
wenzelm@14991
   222
  | _ => x);
wenzelm@14991
   223
wenzelm@23788
   224
fun text_of tok =
wenzelm@23788
   225
  if is_semicolon tok then ("terminator", "")
wenzelm@23729
   226
  else
wenzelm@23788
   227
    let
wenzelm@23788
   228
      val k = str_of_kind (kind_of tok);
wenzelm@40523
   229
      val s = unparse tok;
wenzelm@23788
   230
    in
wenzelm@23788
   231
      if s = "" then (k, "")
wenzelm@23788
   232
      else if size s < 40 andalso not (exists_string (fn c => c = "\n") s) then (k ^ " " ^ s, "")
wenzelm@23788
   233
      else (k, s)
wenzelm@23788
   234
    end;
wenzelm@23729
   235
wenzelm@5825
   236
wenzelm@5825
   237
wenzelm@27814
   238
(** associated values **)
wenzelm@27814
   239
wenzelm@48867
   240
(* inlined file content *)
wenzelm@48867
   241
wenzelm@48867
   242
fun get_files (Token (_, _, Value (SOME (Files files)))) = SOME files
wenzelm@48867
   243
  | get_files _ = NONE;
wenzelm@48867
   244
wenzelm@48867
   245
fun put_files files (Token (x, y, Slot)) = Token (x, y, Value (SOME (Files files)))
wenzelm@48867
   246
  | put_files _ tok =
wenzelm@48867
   247
      raise Fail ("Cannot put inlined files here" ^ Position.str_of (position_of tok));
wenzelm@48867
   248
wenzelm@48867
   249
wenzelm@27814
   250
(* access values *)
wenzelm@27814
   251
wenzelm@27814
   252
fun get_value (Token (_, _, Value v)) = v
wenzelm@27814
   253
  | get_value _ = NONE;
wenzelm@27814
   254
wenzelm@27814
   255
fun map_value f (Token (x, y, Value (SOME v))) = Token (x, y, Value (SOME (f v)))
wenzelm@27814
   256
  | map_value _ tok = tok;
wenzelm@27814
   257
wenzelm@27814
   258
wenzelm@27814
   259
(* make values *)
wenzelm@27814
   260
wenzelm@27814
   261
fun mk_value k v = Token ((k, Position.no_range), (InternalValue, k), Value (SOME v));
wenzelm@27814
   262
wenzelm@27814
   263
val mk_text = mk_value "<text>" o Text;
wenzelm@27814
   264
val mk_typ = mk_value "<typ>" o Typ;
wenzelm@27814
   265
val mk_term = mk_value "<term>" o Term;
wenzelm@27814
   266
val mk_fact = mk_value "<fact>" o Fact;
wenzelm@27814
   267
val mk_attribute = mk_value "<attribute>" o Attribute;
wenzelm@27814
   268
wenzelm@27814
   269
wenzelm@27814
   270
(* static binding *)
wenzelm@27814
   271
wenzelm@27814
   272
(*1st stage: make empty slots assignable*)
wenzelm@32738
   273
fun assignable (Token (x, y, Slot)) = Token (x, y, Assignable (Unsynchronized.ref NONE))
wenzelm@27814
   274
  | assignable tok = tok;
wenzelm@27814
   275
wenzelm@27814
   276
(*2nd stage: assign values as side-effect of scanning*)
wenzelm@27814
   277
fun assign v (Token (_, _, Assignable r)) = r := v
wenzelm@27814
   278
  | assign _ _ = ();
wenzelm@27814
   279
wenzelm@27814
   280
(*3rd stage: static closure of final values*)
wenzelm@32738
   281
fun closure (Token (x, y, Assignable (Unsynchronized.ref v))) = Token (x, y, Value v)
wenzelm@27814
   282
  | closure tok = tok;
wenzelm@27814
   283
wenzelm@27814
   284
wenzelm@27814
   285
wenzelm@5825
   286
(** scanners **)
wenzelm@5825
   287
wenzelm@30573
   288
open Basic_Symbol_Pos;
wenzelm@5825
   289
wenzelm@48764
   290
val err_prefix = "Outer lexical error: ";
wenzelm@48764
   291
wenzelm@48764
   292
fun !!! msg = Symbol_Pos.!!! (fn () => err_prefix ^ msg);
wenzelm@5825
   293
wenzelm@5825
   294
wenzelm@5825
   295
(* scan symbolic idents *)
wenzelm@5825
   296
wenzelm@40627
   297
val is_sym_char = member (op =) (raw_explode "!#$%&*+-/<=>?@^_|~");
wenzelm@5825
   298
wenzelm@8231
   299
val scan_symid =
wenzelm@40525
   300
  Scan.many1 (is_sym_char o Symbol_Pos.symbol) ||
wenzelm@40525
   301
  Scan.one (Symbol.is_symbolic o Symbol_Pos.symbol) >> single;
wenzelm@5825
   302
wenzelm@8231
   303
fun is_symid str =
wenzelm@8231
   304
  (case try Symbol.explode str of
skalberg@15531
   305
    SOME [s] => Symbol.is_symbolic s orelse is_sym_char s
skalberg@15531
   306
  | SOME ss => forall is_sym_char ss
wenzelm@8231
   307
  | _ => false);
wenzelm@8231
   308
wenzelm@27814
   309
fun ident_or_symbolic "begin" = false
wenzelm@27814
   310
  | ident_or_symbolic ":" = true
wenzelm@27814
   311
  | ident_or_symbolic "::" = true
wenzelm@42290
   312
  | ident_or_symbolic s = Lexicon.is_identifier s orelse is_symid s;
wenzelm@5825
   313
wenzelm@5825
   314
wenzelm@5825
   315
(* scan verbatim text *)
wenzelm@5825
   316
wenzelm@5825
   317
val scan_verb =
wenzelm@27769
   318
  $$$ "*" --| Scan.ahead (~$$$ "}") ||
wenzelm@27769
   319
  Scan.one (fn (s, _) => s <> "*" andalso Symbol.is_regular s) >> single;
wenzelm@5825
   320
wenzelm@5825
   321
val scan_verbatim =
wenzelm@30573
   322
  (Symbol_Pos.scan_pos --| $$$ "{" --| $$$ "*") -- !!! "missing end of verbatim text"
wenzelm@30586
   323
    (Symbol_Pos.change_prompt
wenzelm@30586
   324
      ((Scan.repeat scan_verb >> flat) -- ($$$ "*" |-- $$$ "}" |-- Symbol_Pos.scan_pos)));
wenzelm@5825
   325
wenzelm@48743
   326
val recover_verbatim =
wenzelm@48743
   327
  $$$ "{" @@@ $$$ "*" @@@ (Scan.repeat scan_verb >> flat);
wenzelm@48743
   328
wenzelm@5825
   329
wenzelm@5825
   330
(* scan space *)
wenzelm@5825
   331
wenzelm@48771
   332
fun space_symbol (s, _) = Symbol.is_blank s andalso s <> "\n";
wenzelm@5825
   333
wenzelm@5825
   334
val scan_space =
wenzelm@48771
   335
  Scan.many1 space_symbol @@@ Scan.optional ($$$ "\n") [] ||
wenzelm@48771
   336
  Scan.many space_symbol @@@ $$$ "\n";
wenzelm@5825
   337
wenzelm@5825
   338
wenzelm@27780
   339
(* scan comment *)
wenzelm@5825
   340
wenzelm@5825
   341
val scan_comment =
wenzelm@30573
   342
  Symbol_Pos.scan_pos -- (Symbol_Pos.scan_comment_body !!! -- Symbol_Pos.scan_pos);
wenzelm@5825
   343
wenzelm@5825
   344
wenzelm@27663
   345
wenzelm@27769
   346
(** token sources **)
wenzelm@5825
   347
wenzelm@27769
   348
fun source_proper src = src |> Source.filter is_proper;
wenzelm@5825
   349
wenzelm@23678
   350
local
wenzelm@23678
   351
wenzelm@27769
   352
fun token_leq ((_, syms1), (_, syms2)) = length syms1 <= length syms2;
wenzelm@27780
   353
wenzelm@27799
   354
fun token k ss =
wenzelm@43709
   355
  Token ((Symbol_Pos.implode ss, Symbol_Pos.range ss), (k, Symbol_Pos.content ss), Slot);
wenzelm@27799
   356
wenzelm@27799
   357
fun token_range k (pos1, (ss, pos2)) =
wenzelm@43709
   358
  Token (Symbol_Pos.implode_range pos1 pos2 ss, (k, Symbol_Pos.content ss), Slot);
wenzelm@23678
   359
wenzelm@27769
   360
fun scan (lex1, lex2) = !!! "bad input"
wenzelm@48764
   361
  (Symbol_Pos.scan_string_qq err_prefix >> token_range String ||
wenzelm@48764
   362
    Symbol_Pos.scan_string_bq err_prefix >> token_range AltString ||
wenzelm@27799
   363
    scan_verbatim >> token_range Verbatim ||
wenzelm@27799
   364
    scan_comment >> token_range Comment ||
wenzelm@27780
   365
    scan_space >> token Space ||
wenzelm@40525
   366
    Scan.one (Symbol.is_sync o Symbol_Pos.symbol) >> (token Sync o single) ||
wenzelm@27780
   367
    (Scan.max token_leq
wenzelm@27769
   368
      (Scan.max token_leq
wenzelm@27769
   369
        (Scan.literal lex2 >> pair Command)
wenzelm@27769
   370
        (Scan.literal lex1 >> pair Keyword))
wenzelm@42290
   371
      (Lexicon.scan_longid >> pair LongIdent ||
wenzelm@42290
   372
        Lexicon.scan_id >> pair Ident ||
wenzelm@42290
   373
        Lexicon.scan_var >> pair Var ||
wenzelm@42290
   374
        Lexicon.scan_tid >> pair TypeIdent ||
wenzelm@42290
   375
        Lexicon.scan_tvar >> pair TypeVar ||
wenzelm@42290
   376
        Lexicon.scan_float >> pair Float ||
wenzelm@42290
   377
        Lexicon.scan_nat >> pair Nat ||
wenzelm@27780
   378
        scan_symid >> pair SymIdent) >> uncurry token));
wenzelm@27769
   379
wenzelm@27769
   380
fun recover msg =
wenzelm@48743
   381
  (Symbol_Pos.recover_string_qq ||
wenzelm@48743
   382
    Symbol_Pos.recover_string_bq ||
wenzelm@48743
   383
    recover_verbatim ||
wenzelm@48743
   384
    Symbol_Pos.recover_comment ||
wenzelm@48743
   385
    Scan.one (Symbol.is_regular o Symbol_Pos.symbol) >> single)
wenzelm@27780
   386
  >> (single o token (Error msg));
wenzelm@23678
   387
wenzelm@23678
   388
in
wenzelm@5825
   389
wenzelm@27835
   390
fun source' {do_recover} get_lex =
wenzelm@30573
   391
  Source.source Symbol_Pos.stopper (Scan.bulk (fn xs => scan (get_lex ()) xs))
wenzelm@27780
   392
    (Option.map (rpair recover) do_recover);
wenzelm@27780
   393
wenzelm@5825
   394
fun source do_recover get_lex pos src =
wenzelm@30573
   395
  Symbol_Pos.source pos src
wenzelm@27780
   396
  |> source' do_recover get_lex;
wenzelm@23678
   397
wenzelm@23678
   398
end;
wenzelm@5825
   399
wenzelm@30586
   400
wenzelm@30586
   401
(* read_antiq *)
wenzelm@30586
   402
wenzelm@30586
   403
fun read_antiq lex scan (syms, pos) =
wenzelm@30586
   404
  let
wenzelm@30586
   405
    fun err msg = cat_error msg ("Malformed antiquotation" ^ Position.str_of pos ^ ":\n" ^
wenzelm@30586
   406
      "@{" ^ Symbol_Pos.content syms ^ "}");
wenzelm@30586
   407
wenzelm@30586
   408
    val res =
wenzelm@30586
   409
      Source.of_list syms
wenzelm@30586
   410
      |> source' {do_recover = NONE} (K (lex, Scan.empty_lexicon))
wenzelm@30586
   411
      |> source_proper
wenzelm@30586
   412
      |> Source.source stopper (Scan.error (Scan.bulk scan)) NONE
wenzelm@30586
   413
      |> Source.exhaust;
wenzelm@30586
   414
  in (case res of [x] => x | _ => err "") handle ERROR msg => err msg end;
wenzelm@30586
   415
wenzelm@5825
   416
end;