src/Pure/Isar/outer_lex.ML
author wenzelm
Sun Mar 01 23:36:12 2009 +0100 (2009-03-01)
changeset 30190 479806475f3c
parent 29606 fedb8be05f24
child 30573 49899f26fbd1
permissions -rw-r--r--
use long names for old-style fold combinators;
wenzelm@5825
     1
(*  Title:      Pure/Isar/outer_lex.ML
wenzelm@5825
     2
    Author:     Markus Wenzel, TU Muenchen
wenzelm@5825
     3
wenzelm@5825
     4
Outer lexical syntax for Isabelle/Isar.
wenzelm@5825
     5
*)
wenzelm@5825
     6
wenzelm@5825
     7
signature OUTER_LEX =
wenzelm@5825
     8
sig
wenzelm@5825
     9
  datatype token_kind =
wenzelm@27814
    10
    Command | Keyword | Ident | LongIdent | SymIdent | Var | TypeIdent | TypeVar |
wenzelm@27814
    11
    Nat | String | AltString | Verbatim | Space | Comment | InternalValue |
wenzelm@27814
    12
    Malformed | Error of string | Sync | EOF
wenzelm@27814
    13
  datatype value =
wenzelm@27814
    14
    Text of string | Typ of typ | Term of term | Fact of thm list |
wenzelm@27814
    15
    Attribute of morphism -> attribute
wenzelm@27814
    16
  type token
wenzelm@5825
    17
  val str_of_kind: token_kind -> string
wenzelm@5825
    18
  val position_of: token -> Position.T
wenzelm@27752
    19
  val end_position_of: token -> Position.T
wenzelm@5825
    20
  val pos_of: token -> string
wenzelm@27733
    21
  val eof: token
wenzelm@27733
    22
  val is_eof: token -> bool
wenzelm@27733
    23
  val not_eof: token -> bool
wenzelm@27733
    24
  val not_sync: token -> bool
wenzelm@27733
    25
  val stopper: token Scan.stopper
wenzelm@23721
    26
  val kind_of: token -> token_kind
wenzelm@5825
    27
  val is_kind: token_kind -> token -> bool
wenzelm@7026
    28
  val keyword_with: (string -> bool) -> token -> bool
wenzelm@16029
    29
  val ident_with: (string -> bool) -> token -> bool
wenzelm@5825
    30
  val is_proper: token -> bool
wenzelm@9130
    31
  val is_semicolon: token -> bool
wenzelm@17069
    32
  val is_comment: token -> bool
wenzelm@8580
    33
  val is_begin_ignore: token -> bool
wenzelm@8580
    34
  val is_end_ignore: token -> bool
wenzelm@17069
    35
  val is_blank: token -> bool
wenzelm@8651
    36
  val is_newline: token -> bool
wenzelm@27747
    37
  val source_of: token -> string
wenzelm@27885
    38
  val source_position_of: token -> SymbolPos.text * Position.T
wenzelm@27814
    39
  val content_of: token -> string
wenzelm@14991
    40
  val unparse: token -> string
wenzelm@23788
    41
  val text_of: token -> string * string
wenzelm@27814
    42
  val get_value: token -> value option
wenzelm@27814
    43
  val map_value: (value -> value) -> token -> token
wenzelm@27814
    44
  val mk_text: string -> token
wenzelm@27814
    45
  val mk_typ: typ -> token
wenzelm@27814
    46
  val mk_term: term -> token
wenzelm@27814
    47
  val mk_fact: thm list -> token
wenzelm@27814
    48
  val mk_attribute: (morphism -> attribute) -> token
wenzelm@27814
    49
  val assignable: token -> token
wenzelm@27814
    50
  val assign: value option -> token -> unit
wenzelm@27814
    51
  val closure: token -> token
wenzelm@27814
    52
  val ident_or_symbolic: string -> bool
wenzelm@27769
    53
  val !!! : string -> (SymbolPos.T list -> 'a) -> SymbolPos.T list -> 'a
wenzelm@27769
    54
  val scan_quoted: SymbolPos.T list -> SymbolPos.T list * SymbolPos.T list
wenzelm@27769
    55
  val source_proper: (token, 'a) Source.source -> (token, (token, 'a) Source.source) Source.source
wenzelm@27835
    56
  val source': {do_recover: bool Option.option} -> (unit -> Scan.lexicon * Scan.lexicon) ->
wenzelm@27780
    57
    (SymbolPos.T, 'a) Source.source -> (token, (SymbolPos.T, 'a) Source.source) Source.source
wenzelm@27835
    58
  val source: {do_recover: bool Option.option} -> (unit -> Scan.lexicon * Scan.lexicon) ->
wenzelm@27769
    59
    Position.T -> (Symbol.symbol, 'a) Source.source -> (token,
wenzelm@27769
    60
      (SymbolPos.T, Position.T * (Symbol.symbol, 'a) Source.source) Source.source) Source.source
wenzelm@5825
    61
end;
wenzelm@5825
    62
wenzelm@5825
    63
structure OuterLex: OUTER_LEX =
wenzelm@5825
    64
struct
wenzelm@5825
    65
wenzelm@5825
    66
(** tokens **)
wenzelm@5825
    67
wenzelm@27814
    68
(* token values *)
wenzelm@27814
    69
wenzelm@27814
    70
(*The value slot assigns an (optional) internal value to a token,
wenzelm@27814
    71
  usually as a side-effect of special scanner setup (see also
wenzelm@27814
    72
  args.ML).  Note that an assignable ref designates an intermediate
wenzelm@27814
    73
  state of internalization -- it is NOT meant to persist.*)
wenzelm@27814
    74
wenzelm@27814
    75
datatype value =
wenzelm@27814
    76
  Text of string |
wenzelm@27814
    77
  Typ of typ |
wenzelm@27814
    78
  Term of term |
wenzelm@27814
    79
  Fact of thm list |
wenzelm@27814
    80
  Attribute of morphism -> attribute;
wenzelm@27814
    81
wenzelm@27814
    82
datatype slot =
wenzelm@27814
    83
  Slot |
wenzelm@27814
    84
  Value of value option |
wenzelm@27814
    85
  Assignable of value option ref;
wenzelm@27814
    86
wenzelm@27814
    87
wenzelm@5825
    88
(* datatype token *)
wenzelm@5825
    89
wenzelm@5825
    90
datatype token_kind =
wenzelm@27814
    91
  Command | Keyword | Ident | LongIdent | SymIdent | Var | TypeIdent | TypeVar |
wenzelm@27814
    92
  Nat | String | AltString | Verbatim | Space | Comment | InternalValue |
wenzelm@27814
    93
  Malformed | Error of string | Sync | EOF;
wenzelm@5825
    94
wenzelm@27814
    95
datatype token = Token of (SymbolPos.text * Position.range) * (token_kind * string) * slot;
wenzelm@5825
    96
wenzelm@5825
    97
val str_of_kind =
wenzelm@7026
    98
 fn Command => "command"
wenzelm@7026
    99
  | Keyword => "keyword"
wenzelm@5825
   100
  | Ident => "identifier"
wenzelm@5825
   101
  | LongIdent => "long identifier"
wenzelm@5825
   102
  | SymIdent => "symbolic identifier"
wenzelm@5825
   103
  | Var => "schematic variable"
wenzelm@5825
   104
  | TypeIdent => "type variable"
wenzelm@5825
   105
  | TypeVar => "schematic type variable"
wenzelm@5825
   106
  | Nat => "number"
wenzelm@5825
   107
  | String => "string"
wenzelm@17164
   108
  | AltString => "back-quoted string"
wenzelm@5825
   109
  | Verbatim => "verbatim text"
wenzelm@7682
   110
  | Space => "white space"
wenzelm@7682
   111
  | Comment => "comment text"
wenzelm@27814
   112
  | InternalValue => "internal value"
wenzelm@23729
   113
  | Malformed => "malformed symbolic character"
wenzelm@23729
   114
  | Error _ => "bad input"
wenzelm@23788
   115
  | Sync => "sync marker"
wenzelm@5825
   116
  | EOF => "end-of-file";
wenzelm@5825
   117
wenzelm@5825
   118
wenzelm@27733
   119
(* position *)
wenzelm@5825
   120
wenzelm@27814
   121
fun position_of (Token ((_, (pos, _)), _, _)) = pos;
wenzelm@27814
   122
fun end_position_of (Token ((_, (_, pos)), _, _)) = pos;
wenzelm@27663
   123
wenzelm@5825
   124
val pos_of = Position.str_of o position_of;
wenzelm@5825
   125
wenzelm@5825
   126
wenzelm@27733
   127
(* control tokens *)
wenzelm@27733
   128
wenzelm@27814
   129
fun mk_eof pos = Token (("", (pos, Position.none)), (EOF, ""), Slot);
wenzelm@27733
   130
val eof = mk_eof Position.none;
wenzelm@27733
   131
wenzelm@27814
   132
fun is_eof (Token (_, (EOF, _), _)) = true
wenzelm@27733
   133
  | is_eof _ = false;
wenzelm@27733
   134
wenzelm@27733
   135
val not_eof = not o is_eof;
wenzelm@27733
   136
wenzelm@27814
   137
fun not_sync (Token (_, (Sync, _), _)) = false
wenzelm@27733
   138
  | not_sync _ = true;
wenzelm@27733
   139
wenzelm@27752
   140
val stopper =
wenzelm@27752
   141
  Scan.stopper (fn [] => eof | toks => mk_eof (end_position_of (List.last toks))) is_eof;
wenzelm@27733
   142
wenzelm@27733
   143
wenzelm@5825
   144
(* kind of token *)
wenzelm@5825
   145
wenzelm@27814
   146
fun kind_of (Token (_, (k, _), _)) = k;
wenzelm@27814
   147
fun is_kind k (Token (_, (k', _), _)) = k = k';
wenzelm@5825
   148
wenzelm@27814
   149
fun keyword_with pred (Token (_, (Keyword, x), _)) = pred x
wenzelm@7026
   150
  | keyword_with _ _ = false;
wenzelm@5825
   151
wenzelm@27814
   152
fun ident_with pred (Token (_, (Ident, x), _)) = pred x
wenzelm@16029
   153
  | ident_with _ _ = false;
wenzelm@16029
   154
wenzelm@27814
   155
fun is_proper (Token (_, (Space, _), _)) = false
wenzelm@27814
   156
  | is_proper (Token (_, (Comment, _), _)) = false
wenzelm@5825
   157
  | is_proper _ = true;
wenzelm@5825
   158
wenzelm@27814
   159
fun is_semicolon (Token (_, (Keyword, ";"), _)) = true
wenzelm@9130
   160
  | is_semicolon _ = false;
wenzelm@9130
   161
wenzelm@27814
   162
fun is_comment (Token (_, (Comment, _), _)) = true
wenzelm@17069
   163
  | is_comment _ = false;
wenzelm@17069
   164
wenzelm@27814
   165
fun is_begin_ignore (Token (_, (Comment, "<"), _)) = true
wenzelm@8580
   166
  | is_begin_ignore _ = false;
wenzelm@8580
   167
wenzelm@27814
   168
fun is_end_ignore (Token (_, (Comment, ">"), _)) = true
wenzelm@8580
   169
  | is_end_ignore _ = false;
wenzelm@8580
   170
wenzelm@8651
   171
wenzelm@17069
   172
(* blanks and newlines -- space tokens obey lines *)
wenzelm@8651
   173
wenzelm@27814
   174
fun is_blank (Token (_, (Space, x), _)) = not (String.isSuffix "\n" x)
wenzelm@17069
   175
  | is_blank _ = false;
wenzelm@17069
   176
wenzelm@27814
   177
fun is_newline (Token (_, (Space, x), _)) = String.isSuffix "\n" x
wenzelm@8651
   178
  | is_newline _ = false;
wenzelm@8651
   179
wenzelm@5825
   180
wenzelm@14991
   181
(* token content *)
wenzelm@9155
   182
wenzelm@27814
   183
fun source_of (Token ((source, (pos, _)), _, _)) =
wenzelm@27814
   184
  YXML.string_of (XML.Elem (Markup.tokenN, Position.properties_of pos, [XML.Text source]));
wenzelm@25642
   185
wenzelm@27885
   186
fun source_position_of (Token ((source, (pos, _)), _, _)) = (source, pos);
wenzelm@27873
   187
wenzelm@27814
   188
fun content_of (Token (_, (_, x), _)) = x;
wenzelm@27747
   189
wenzelm@27747
   190
wenzelm@27747
   191
(* unparse *)
wenzelm@27747
   192
wenzelm@18547
   193
fun escape q =
wenzelm@18547
   194
  implode o map (fn s => if s = q orelse s = "\\" then "\\" ^ s else s) o Symbol.explode;
wenzelm@18547
   195
wenzelm@27814
   196
fun unparse (Token (_, (kind, x), _)) =
wenzelm@14991
   197
  (case kind of
wenzelm@18547
   198
    String => x |> quote o escape "\""
wenzelm@18547
   199
  | AltString => x |> enclose "`" "`" o escape "`"
wenzelm@14991
   200
  | Verbatim => x |> enclose "{*" "*}"
wenzelm@14991
   201
  | Comment => x |> enclose "(*" "*)"
wenzelm@28034
   202
  | Malformed => space_implode " " (explode x)
wenzelm@23729
   203
  | Sync => ""
wenzelm@23729
   204
  | EOF => ""
wenzelm@14991
   205
  | _ => x);
wenzelm@14991
   206
wenzelm@23788
   207
fun text_of tok =
wenzelm@23788
   208
  if is_semicolon tok then ("terminator", "")
wenzelm@23729
   209
  else
wenzelm@23788
   210
    let
wenzelm@23788
   211
      val k = str_of_kind (kind_of tok);
wenzelm@25642
   212
      val s = unparse tok
wenzelm@27814
   213
        handle ERROR _ => Symbol.separate_chars (content_of tok);
wenzelm@23788
   214
    in
wenzelm@23788
   215
      if s = "" then (k, "")
wenzelm@23788
   216
      else if size s < 40 andalso not (exists_string (fn c => c = "\n") s) then (k ^ " " ^ s, "")
wenzelm@23788
   217
      else (k, s)
wenzelm@23788
   218
    end;
wenzelm@23729
   219
wenzelm@5825
   220
wenzelm@5825
   221
wenzelm@27814
   222
(** associated values **)
wenzelm@27814
   223
wenzelm@27814
   224
(* access values *)
wenzelm@27814
   225
wenzelm@27814
   226
fun get_value (Token (_, _, Value v)) = v
wenzelm@27814
   227
  | get_value _ = NONE;
wenzelm@27814
   228
wenzelm@27814
   229
fun map_value f (Token (x, y, Value (SOME v))) = Token (x, y, Value (SOME (f v)))
wenzelm@27814
   230
  | map_value _ tok = tok;
wenzelm@27814
   231
wenzelm@27814
   232
wenzelm@27814
   233
(* make values *)
wenzelm@27814
   234
wenzelm@27814
   235
fun mk_value k v = Token ((k, Position.no_range), (InternalValue, k), Value (SOME v));
wenzelm@27814
   236
wenzelm@27814
   237
val mk_text = mk_value "<text>" o Text;
wenzelm@27814
   238
val mk_typ = mk_value "<typ>" o Typ;
wenzelm@27814
   239
val mk_term = mk_value "<term>" o Term;
wenzelm@27814
   240
val mk_fact = mk_value "<fact>" o Fact;
wenzelm@27814
   241
val mk_attribute = mk_value "<attribute>" o Attribute;
wenzelm@27814
   242
wenzelm@27814
   243
wenzelm@27814
   244
(* static binding *)
wenzelm@27814
   245
wenzelm@27814
   246
(*1st stage: make empty slots assignable*)
wenzelm@27814
   247
fun assignable (Token (x, y, Slot)) = Token (x, y, Assignable (ref NONE))
wenzelm@27814
   248
  | assignable tok = tok;
wenzelm@27814
   249
wenzelm@27814
   250
(*2nd stage: assign values as side-effect of scanning*)
wenzelm@27814
   251
fun assign v (Token (_, _, Assignable r)) = r := v
wenzelm@27814
   252
  | assign _ _ = ();
wenzelm@27814
   253
wenzelm@27814
   254
(*3rd stage: static closure of final values*)
wenzelm@27814
   255
fun closure (Token (x, y, Assignable (ref v))) = Token (x, y, Value v)
wenzelm@27814
   256
  | closure tok = tok;
wenzelm@27814
   257
wenzelm@27814
   258
wenzelm@27814
   259
wenzelm@5825
   260
(** scanners **)
wenzelm@5825
   261
wenzelm@27769
   262
open BasicSymbolPos;
wenzelm@5825
   263
wenzelm@27769
   264
fun !!! msg = SymbolPos.!!! ("Outer lexical error: " ^ msg);
wenzelm@5825
   265
wenzelm@27769
   266
fun change_prompt scan = Scan.prompt "# " scan;
wenzelm@5825
   267
wenzelm@5825
   268
wenzelm@5825
   269
(* scan symbolic idents *)
wenzelm@5825
   270
wenzelm@20664
   271
val is_sym_char = member (op =) (explode "!#$%&*+-/<=>?@^_|~");
wenzelm@5825
   272
wenzelm@8231
   273
val scan_symid =
wenzelm@27769
   274
  Scan.many1 (is_sym_char o symbol) ||
wenzelm@27769
   275
  Scan.one (Symbol.is_symbolic o symbol) >> single;
wenzelm@5825
   276
wenzelm@8231
   277
fun is_symid str =
wenzelm@8231
   278
  (case try Symbol.explode str of
skalberg@15531
   279
    SOME [s] => Symbol.is_symbolic s orelse is_sym_char s
skalberg@15531
   280
  | SOME ss => forall is_sym_char ss
wenzelm@8231
   281
  | _ => false);
wenzelm@8231
   282
wenzelm@27814
   283
fun ident_or_symbolic "begin" = false
wenzelm@27814
   284
  | ident_or_symbolic ":" = true
wenzelm@27814
   285
  | ident_or_symbolic "::" = true
wenzelm@27814
   286
  | ident_or_symbolic s = Syntax.is_identifier s orelse is_symid s;
wenzelm@5825
   287
wenzelm@5825
   288
wenzelm@5825
   289
(* scan strings *)
wenzelm@5825
   290
wenzelm@17164
   291
local
wenzelm@17164
   292
wenzelm@25579
   293
val char_code =
wenzelm@27769
   294
  Scan.one (Symbol.is_ascii_digit o symbol) --
wenzelm@27769
   295
  Scan.one (Symbol.is_ascii_digit o symbol) --
wenzelm@27769
   296
  Scan.one (Symbol.is_ascii_digit o symbol) :|--
wenzelm@27769
   297
  (fn (((a, pos), (b, _)), (c, _)) =>
wenzelm@25579
   298
    let val (n, _) = Library.read_int [a, b, c]
wenzelm@27769
   299
    in if n <= 255 then Scan.succeed [(chr n, pos)] else Scan.fail end);
wenzelm@25579
   300
wenzelm@17164
   301
fun scan_str q =
wenzelm@27769
   302
  $$$ "\\" |-- !!! "bad escape character in string" ($$$ q || $$$ "\\" || char_code) ||
wenzelm@27769
   303
  Scan.one (fn (s, _) => s <> q andalso s <> "\\" andalso Symbol.is_regular s) >> single;
wenzelm@5825
   304
wenzelm@17164
   305
fun scan_strs q =
wenzelm@27780
   306
  (SymbolPos.scan_pos --| $$$ q) -- !!! "missing quote at end of string"
wenzelm@27780
   307
    (change_prompt ((Scan.repeat (scan_str q) >> flat) -- ($$$ q |-- SymbolPos.scan_pos)));
wenzelm@17164
   308
wenzelm@17164
   309
in
wenzelm@17164
   310
wenzelm@17164
   311
val scan_string = scan_strs "\"";
wenzelm@17164
   312
val scan_alt_string = scan_strs "`";
wenzelm@17164
   313
wenzelm@27769
   314
val scan_quoted = Scan.trace (scan_string || scan_alt_string) >> #2;
wenzelm@27752
   315
wenzelm@17164
   316
end;
wenzelm@5825
   317
wenzelm@5825
   318
wenzelm@5825
   319
(* scan verbatim text *)
wenzelm@5825
   320
wenzelm@5825
   321
val scan_verb =
wenzelm@27769
   322
  $$$ "*" --| Scan.ahead (~$$$ "}") ||
wenzelm@27769
   323
  Scan.one (fn (s, _) => s <> "*" andalso Symbol.is_regular s) >> single;
wenzelm@5825
   324
wenzelm@5825
   325
val scan_verbatim =
wenzelm@27780
   326
  (SymbolPos.scan_pos --| $$$ "{" --| $$$ "*") -- !!! "missing end of verbatim text"
wenzelm@27780
   327
    (change_prompt ((Scan.repeat scan_verb >> flat) -- ($$$ "*" |-- $$$ "}" |-- SymbolPos.scan_pos)));
wenzelm@5825
   328
wenzelm@5825
   329
wenzelm@5825
   330
(* scan space *)
wenzelm@5825
   331
wenzelm@19305
   332
fun is_space s = Symbol.is_blank s andalso s <> "\n";
wenzelm@5825
   333
wenzelm@5825
   334
val scan_space =
wenzelm@27769
   335
  Scan.many1 (is_space o symbol) @@@ Scan.optional ($$$ "\n") [] ||
wenzelm@27769
   336
  Scan.many (is_space o symbol) @@@ $$$ "\n";
wenzelm@5825
   337
wenzelm@5825
   338
wenzelm@27780
   339
(* scan comment *)
wenzelm@5825
   340
wenzelm@5825
   341
val scan_comment =
wenzelm@27780
   342
  SymbolPos.scan_pos -- (SymbolPos.scan_comment_body !!! -- SymbolPos.scan_pos);
wenzelm@5825
   343
wenzelm@5825
   344
wenzelm@23678
   345
(* scan malformed symbols *)
wenzelm@23678
   346
wenzelm@23678
   347
val scan_malformed =
wenzelm@27769
   348
  $$$ Symbol.malformed |--
wenzelm@27769
   349
    change_prompt (Scan.many (Symbol.is_regular o symbol))
wenzelm@27769
   350
  --| Scan.option ($$$ Symbol.end_malformed);
wenzelm@27752
   351
wenzelm@27752
   352
wenzelm@27663
   353
wenzelm@27769
   354
(** token sources **)
wenzelm@5825
   355
wenzelm@27769
   356
fun source_proper src = src |> Source.filter is_proper;
wenzelm@5825
   357
wenzelm@23678
   358
local
wenzelm@23678
   359
wenzelm@27769
   360
fun token_leq ((_, syms1), (_, syms2)) = length syms1 <= length syms2;
wenzelm@27780
   361
wenzelm@27799
   362
fun token k ss =
wenzelm@27856
   363
  Token ((SymbolPos.implode ss, SymbolPos.range ss), (k, SymbolPos.untabify_content ss), Slot);
wenzelm@27799
   364
wenzelm@27799
   365
fun token_range k (pos1, (ss, pos2)) =
wenzelm@27856
   366
  Token (SymbolPos.implode_range pos1 pos2 ss, (k, SymbolPos.untabify_content ss), Slot);
wenzelm@23678
   367
wenzelm@27769
   368
fun scan (lex1, lex2) = !!! "bad input"
wenzelm@27799
   369
  (scan_string >> token_range String ||
wenzelm@27799
   370
    scan_alt_string >> token_range AltString ||
wenzelm@27799
   371
    scan_verbatim >> token_range Verbatim ||
wenzelm@27799
   372
    scan_comment >> token_range Comment ||
wenzelm@27780
   373
    scan_space >> token Space ||
wenzelm@27780
   374
    scan_malformed >> token Malformed ||
wenzelm@27780
   375
    Scan.one (Symbol.is_sync o symbol) >> (token Sync o single) ||
wenzelm@27780
   376
    (Scan.max token_leq
wenzelm@27769
   377
      (Scan.max token_leq
wenzelm@27769
   378
        (Scan.literal lex2 >> pair Command)
wenzelm@27769
   379
        (Scan.literal lex1 >> pair Keyword))
wenzelm@27769
   380
      (Syntax.scan_longid >> pair LongIdent ||
wenzelm@27769
   381
        Syntax.scan_id >> pair Ident ||
wenzelm@27769
   382
        Syntax.scan_var >> pair Var ||
wenzelm@27769
   383
        Syntax.scan_tid >> pair TypeIdent ||
wenzelm@27769
   384
        Syntax.scan_tvar >> pair TypeVar ||
wenzelm@27769
   385
        Syntax.scan_nat >> pair Nat ||
wenzelm@27780
   386
        scan_symid >> pair SymIdent) >> uncurry token));
wenzelm@27769
   387
wenzelm@27769
   388
fun recover msg =
wenzelm@27769
   389
  Scan.many ((Symbol.is_regular andf (not o Symbol.is_blank)) o symbol)
wenzelm@27780
   390
  >> (single o token (Error msg));
wenzelm@23678
   391
wenzelm@23678
   392
in
wenzelm@5825
   393
wenzelm@27835
   394
fun source' {do_recover} get_lex =
wenzelm@27780
   395
  Source.source SymbolPos.stopper (Scan.bulk (fn xs => scan (get_lex ()) xs))
wenzelm@27780
   396
    (Option.map (rpair recover) do_recover);
wenzelm@27780
   397
wenzelm@5825
   398
fun source do_recover get_lex pos src =
wenzelm@27769
   399
  SymbolPos.source pos src
wenzelm@27780
   400
  |> source' do_recover get_lex;
wenzelm@23678
   401
wenzelm@23678
   402
end;
wenzelm@5825
   403
wenzelm@5825
   404
end;