src/Pure/Isar/token.ML
author wenzelm
Fri Apr 01 17:37:46 2016 +0200 (2016-04-01)
changeset 62797 e08c44eed27f
parent 62782 057e8dbe4326
child 62799 46e6f91c4da1
permissions -rw-r--r--
tuned signature;
     1 (*  Title:      Pure/Isar/token.ML
     2     Author:     Markus Wenzel, TU Muenchen
     3 
     4 Outer token syntax for Isabelle/Isar.
     5 *)
     6 
     7 signature TOKEN =
     8 sig
     9   datatype kind =
    10     (*immediate source*)
    11     Command | Keyword | Ident | Long_Ident | Sym_Ident | Var | Type_Ident | Type_Var | Nat |
    12     Float | Space |
    13     (*delimited content*)
    14     String | Alt_String | Verbatim | Cartouche | Comment |
    15     (*special content*)
    16     Error of string | EOF
    17   val str_of_kind: kind -> string
    18   type file = {src_path: Path.T, lines: string list, digest: SHA1.digest, pos: Position.T}
    19   type T
    20   type src = T list
    21   type name_value = {name: string, kind: string, print: Proof.context -> Markup.T * xstring}
    22   datatype value =
    23     Source of src |
    24     Literal of bool * Markup.T |
    25     Name of name_value * morphism |
    26     Typ of typ |
    27     Term of term |
    28     Fact of string option * thm list |
    29     Attribute of morphism -> attribute |
    30     Declaration of declaration |
    31     Files of file Exn.result list
    32   val pos_of: T -> Position.T
    33   val range_of: T list -> Position.range
    34   val eof: T
    35   val is_eof: T -> bool
    36   val not_eof: T -> bool
    37   val stopper: T Scan.stopper
    38   val kind_of: T -> kind
    39   val is_kind: kind -> T -> bool
    40   val is_command: T -> bool
    41   val is_name: T -> bool
    42   val keyword_with: (string -> bool) -> T -> bool
    43   val is_command_modifier: T -> bool
    44   val ident_with: (string -> bool) -> T -> bool
    45   val is_proper: T -> bool
    46   val is_improper: T -> bool
    47   val is_comment: T -> bool
    48   val is_begin_ignore: T -> bool
    49   val is_end_ignore: T -> bool
    50   val is_error: T -> bool
    51   val is_space: T -> bool
    52   val is_blank: T -> bool
    53   val is_newline: T -> bool
    54   val content_of: T -> string
    55   val input_of: T -> Input.source
    56   val inner_syntax_of: T -> string
    57   val keyword_markup: bool * Markup.T -> string -> Markup.T
    58   val completion_report: T -> Position.report_text list
    59   val reports: Keyword.keywords -> T -> Position.report_text list
    60   val markups: Keyword.keywords -> T -> Markup.T list
    61   val unparse: T -> string
    62   val print: T -> string
    63   val text_of: T -> string * string
    64   val get_files: T -> file Exn.result list
    65   val put_files: file Exn.result list -> T -> T
    66   val get_value: T -> value option
    67   val reports_of_value: T -> Position.report list
    68   val name_value: name_value -> value
    69   val get_name: T -> name_value option
    70   val declare_maxidx: T -> Proof.context -> Proof.context
    71   val map_facts: (string option -> thm list -> thm list) -> T -> T
    72   val transform: morphism -> T -> T
    73   val init_assignable: T -> T
    74   val assign: value option -> T -> T
    75   val evaluate: ('a -> value) -> (T -> 'a) -> T -> 'a
    76   val closure: T -> T
    77   val pretty_value: Proof.context -> T -> Pretty.T
    78   val name_of_src: src -> string * Position.T
    79   val args_of_src: src -> T list
    80   val checked_src: src -> bool
    81   val check_src: Proof.context -> (Proof.context -> 'a Name_Space.table) -> src -> src * 'a
    82   val pretty_src: Proof.context -> src -> Pretty.T
    83   val ident_or_symbolic: string -> bool
    84   val source': bool -> Keyword.keywords -> (Symbol_Pos.T, 'a) Source.source ->
    85     (T, (Symbol_Pos.T, 'a) Source.source) Source.source
    86   val source_proper: (T, 'a) Source.source -> (T, (T, 'a) Source.source) Source.source
    87   val source: Keyword.keywords ->
    88     Position.T -> (Symbol.symbol, 'a) Source.source -> (T,
    89       (Symbol_Pos.T, Position.T * (Symbol.symbol, 'a) Source.source) Source.source) Source.source
    90   val source_strict: Keyword.keywords ->
    91     Position.T -> (Symbol.symbol, 'a) Source.source -> (T,
    92       (Symbol_Pos.T, Position.T * (Symbol.symbol, 'a) Source.source) Source.source) Source.source
    93   val read_cartouche: Symbol_Pos.T list -> T
    94   val explode: Keyword.keywords -> Position.T -> string -> T list
    95   val make: (int * int) * string -> Position.T -> T * Position.T
    96   val make_string: string * Position.T -> T
    97   val make_src: string * Position.T -> T list -> src
    98   type 'a parser = T list -> 'a * T list
    99   type 'a context_parser = Context.generic * T list -> 'a * (Context.generic * T list)
   100   val read_no_commands: Keyword.keywords -> 'a parser -> Symbol_Pos.T list -> 'a list
   101   val read_antiq: Keyword.keywords -> 'a parser -> Symbol_Pos.T list * Position.T -> 'a
   102   val syntax_generic: 'a context_parser -> src -> Context.generic -> 'a * Context.generic
   103   val syntax: 'a context_parser -> src -> Proof.context -> 'a * Proof.context
   104 end;
   105 
   106 structure Token: TOKEN =
   107 struct
   108 
   109 (** tokens **)
   110 
   111 (* token kind *)
   112 
   113 datatype kind =
   114   (*immediate source*)
   115   Command | Keyword | Ident | Long_Ident | Sym_Ident | Var | Type_Ident | Type_Var | Nat |
   116   Float | Space |
   117   (*delimited content*)
   118   String | Alt_String | Verbatim | Cartouche | Comment |
   119   (*special content*)
   120   Error of string | EOF;
   121 
   122 val str_of_kind =
   123  fn Command => "command"
   124   | Keyword => "keyword"
   125   | Ident => "identifier"
   126   | Long_Ident => "long identifier"
   127   | Sym_Ident => "symbolic identifier"
   128   | Var => "schematic variable"
   129   | Type_Ident => "type variable"
   130   | Type_Var => "schematic type variable"
   131   | Nat => "natural number"
   132   | Float => "floating-point number"
   133   | Space => "white space"
   134   | String => "quoted string"
   135   | Alt_String => "back-quoted string"
   136   | Verbatim => "verbatim text"
   137   | Cartouche => "text cartouche"
   138   | Comment => "comment text"
   139   | Error _ => "bad input"
   140   | EOF => "end-of-input";
   141 
   142 val immediate_kinds =
   143   Vector.fromList
   144     [Command, Keyword, Ident, Long_Ident, Sym_Ident, Var, Type_Ident, Type_Var, Nat, Float, Space];
   145 
   146 val delimited_kind = member (op =) [String, Alt_String, Verbatim, Cartouche, Comment];
   147 
   148 
   149 (* datatype token *)
   150 
   151 (*The value slot assigns an (optional) internal value to a token,
   152   usually as a side-effect of special scanner setup (see also
   153   args.ML).  Note that an assignable ref designates an intermediate
   154   state of internalization -- it is NOT meant to persist.*)
   155 
   156 type file = {src_path: Path.T, lines: string list, digest: SHA1.digest, pos: Position.T};
   157 
   158 type name_value = {name: string, kind: string, print: Proof.context -> Markup.T * xstring};
   159 
   160 datatype T = Token of (Symbol_Pos.text * Position.range) * (kind * string) * slot
   161 
   162 and slot =
   163   Slot |
   164   Value of value option |
   165   Assignable of value option Unsynchronized.ref
   166 
   167 and value =
   168   Source of T list |
   169   Literal of bool * Markup.T |
   170   Name of name_value * morphism |
   171   Typ of typ |
   172   Term of term |
   173   Fact of string option * thm list |  (*optional name for dynamic fact, i.e. fact "variable"*)
   174   Attribute of morphism -> attribute |
   175   Declaration of declaration |
   176   Files of file Exn.result list;
   177 
   178 type src = T list;
   179 
   180 
   181 (* position *)
   182 
   183 fun pos_of (Token ((_, (pos, _)), _, _)) = pos;
   184 fun end_pos_of (Token ((_, (_, pos)), _, _)) = pos;
   185 
   186 fun reset_range pos (Token ((x, _), y, z)) =
   187   let val pos' = Position.reset_range pos
   188   in Token ((x, (pos', pos')), y, z) end;
   189 
   190 fun range_of (toks as tok :: _) =
   191       let val pos' = end_pos_of (List.last toks)
   192       in Position.range (pos_of tok, pos') end
   193   | range_of [] = Position.no_range;
   194 
   195 
   196 (* stopper *)
   197 
   198 fun mk_eof pos = Token (("", (pos, Position.none)), (EOF, ""), Slot);
   199 val eof = mk_eof Position.none;
   200 
   201 fun is_eof (Token (_, (EOF, _), _)) = true
   202   | is_eof _ = false;
   203 
   204 val not_eof = not o is_eof;
   205 
   206 val stopper =
   207   Scan.stopper (fn [] => eof | toks => mk_eof (end_pos_of (List.last toks))) is_eof;
   208 
   209 
   210 (* kind of token *)
   211 
   212 fun kind_of (Token (_, (k, _), _)) = k;
   213 fun is_kind k (Token (_, (k', _), _)) = k = k';
   214 
   215 val is_command = is_kind Command;
   216 
   217 val is_name = is_kind Ident orf is_kind Sym_Ident orf is_kind String orf is_kind Nat;
   218 
   219 fun keyword_with pred (Token (_, (Keyword, x), _)) = pred x
   220   | keyword_with _ _ = false;
   221 
   222 val is_command_modifier = keyword_with (fn x => x = "private" orelse x = "qualified");
   223 
   224 fun ident_with pred (Token (_, (Ident, x), _)) = pred x
   225   | ident_with _ _ = false;
   226 
   227 fun is_proper (Token (_, (Space, _), _)) = false
   228   | is_proper (Token (_, (Comment, _), _)) = false
   229   | is_proper _ = true;
   230 
   231 val is_improper = not o is_proper;
   232 
   233 fun is_comment (Token (_, (Comment, _), _)) = true
   234   | is_comment _ = false;
   235 
   236 fun is_begin_ignore (Token (_, (Comment, "<"), _)) = true
   237   | is_begin_ignore _ = false;
   238 
   239 fun is_end_ignore (Token (_, (Comment, ">"), _)) = true
   240   | is_end_ignore _ = false;
   241 
   242 fun is_error (Token (_, (Error _, _), _)) = true
   243   | is_error _ = false;
   244 
   245 
   246 (* blanks and newlines -- space tokens obey lines *)
   247 
   248 fun is_space (Token (_, (Space, _), _)) = true
   249   | is_space _ = false;
   250 
   251 fun is_blank (Token (_, (Space, x), _)) = not (String.isSuffix "\n" x)
   252   | is_blank _ = false;
   253 
   254 fun is_newline (Token (_, (Space, x), _)) = String.isSuffix "\n" x
   255   | is_newline _ = false;
   256 
   257 
   258 (* token content *)
   259 
   260 fun content_of (Token (_, (_, x), _)) = x;
   261 
   262 fun input_of (Token ((source, range), (kind, _), _)) =
   263   Input.source (delimited_kind kind) source range;
   264 
   265 fun inner_syntax_of tok =
   266   let val x = content_of tok
   267   in if YXML.detect x then x else Syntax.implode_input (input_of tok) end;
   268 
   269 
   270 (* markup reports *)
   271 
   272 local
   273 
   274 val token_kind_markup =
   275  fn Var => (Markup.var, "")
   276   | Type_Ident => (Markup.tfree, "")
   277   | Type_Var => (Markup.tvar, "")
   278   | String => (Markup.string, "")
   279   | Alt_String => (Markup.alt_string, "")
   280   | Verbatim => (Markup.verbatim, "")
   281   | Cartouche => (Markup.cartouche, "")
   282   | Comment => (Markup.comment, "")
   283   | Error msg => (Markup.bad, msg)
   284   | _ => (Markup.empty, "");
   285 
   286 fun keyword_reports tok = map (fn markup => ((pos_of tok, markup), ""));
   287 
   288 fun command_markups keywords x =
   289   if Keyword.is_theory_end keywords x then [Markup.keyword2]
   290   else if Keyword.is_proof_asm keywords x then [Markup.keyword3]
   291   else if Keyword.is_improper keywords x then [Markup.keyword1, Markup.improper]
   292   else [Markup.keyword1];
   293 
   294 in
   295 
   296 fun keyword_markup (important, keyword) x =
   297   if important orelse Symbol.is_ascii_identifier x then keyword else Markup.delimiter;
   298 
   299 fun completion_report tok =
   300   if is_kind Keyword tok
   301   then map (fn m => ((pos_of tok, m), "")) (Completion.suppress_abbrevs (content_of tok))
   302   else [];
   303 
   304 fun reports keywords tok =
   305   if is_command tok then
   306     keyword_reports tok (command_markups keywords (content_of tok))
   307   else if is_kind Keyword tok then
   308     keyword_reports tok [keyword_markup (false, Markup.keyword2) (content_of tok)]
   309   else
   310     let val (m, text) = token_kind_markup (kind_of tok)
   311     in [((pos_of tok, m), text)] end;
   312 
   313 fun markups keywords = map (#2 o #1) o reports keywords;
   314 
   315 end;
   316 
   317 
   318 (* unparse *)
   319 
   320 fun unparse (Token (_, (kind, x), _)) =
   321   (case kind of
   322     String => Symbol_Pos.quote_string_qq x
   323   | Alt_String => Symbol_Pos.quote_string_bq x
   324   | Verbatim => enclose "{*" "*}" x
   325   | Cartouche => cartouche x
   326   | Comment => enclose "(*" "*)" x
   327   | EOF => ""
   328   | _ => x);
   329 
   330 fun print tok = Markup.markups (markups Keyword.empty_keywords tok) (unparse tok);
   331 
   332 fun text_of tok =
   333   let
   334     val k = str_of_kind (kind_of tok);
   335     val ms = markups Keyword.empty_keywords tok;
   336     val s = unparse tok;
   337   in
   338     if s = "" then (k, "")
   339     else if size s < 40 andalso not (exists_string (fn c => c = "\n") s)
   340     then (k ^ " " ^ Markup.markups ms s, "")
   341     else (k, Markup.markups ms s)
   342   end;
   343 
   344 
   345 
   346 (** associated values **)
   347 
   348 (* inlined file content *)
   349 
   350 fun get_files (Token (_, _, Value (SOME (Files files)))) = files
   351   | get_files _ = [];
   352 
   353 fun put_files [] tok = tok
   354   | put_files files (Token (x, y, Slot)) = Token (x, y, Value (SOME (Files files)))
   355   | put_files _ tok = raise Fail ("Cannot put inlined files here" ^ Position.here (pos_of tok));
   356 
   357 
   358 (* access values *)
   359 
   360 fun get_value (Token (_, _, Value v)) = v
   361   | get_value _ = NONE;
   362 
   363 fun map_value f (Token (x, y, Value (SOME v))) = Token (x, y, Value (SOME (f v)))
   364   | map_value _ tok = tok;
   365 
   366 
   367 (* reports of value *)
   368 
   369 fun get_assignable_value (Token (_, _, Assignable r)) = ! r
   370   | get_assignable_value (Token (_, _, Value v)) = v
   371   | get_assignable_value _ = NONE;
   372 
   373 fun reports_of_value tok =
   374   (case get_assignable_value tok of
   375     SOME (Literal markup) =>
   376       let
   377         val pos = pos_of tok;
   378         val x = content_of tok;
   379       in
   380         if Position.is_reported pos then
   381           map (pair pos) (keyword_markup markup x :: Completion.suppress_abbrevs x)
   382         else []
   383       end
   384   | _ => []);
   385 
   386 
   387 (* name value *)
   388 
   389 fun name_value a = Name (a, Morphism.identity);
   390 
   391 fun get_name tok =
   392   (case get_assignable_value tok of
   393     SOME (Name (a, _)) => SOME a
   394   | _ => NONE);
   395 
   396 
   397 (* maxidx *)
   398 
   399 fun declare_maxidx tok =
   400   (case get_value tok of
   401     SOME (Source src) => fold declare_maxidx src
   402   | SOME (Typ T) => Variable.declare_maxidx (Term.maxidx_of_typ T)
   403   | SOME (Term t) => Variable.declare_maxidx (Term.maxidx_of_term t)
   404   | SOME (Fact (_, ths)) => fold (Variable.declare_maxidx o Thm.maxidx_of) ths
   405   | SOME (Attribute _) => I  (* FIXME !? *)
   406   | SOME (Declaration decl) =>
   407       (fn ctxt =>
   408         let val ctxt' = Context.proof_map (Morphism.form decl) ctxt
   409         in Variable.declare_maxidx (Variable.maxidx_of ctxt') ctxt end)
   410   | _ => I);
   411 
   412 
   413 (* fact values *)
   414 
   415 fun map_facts f =
   416   map_value (fn v =>
   417     (case v of
   418       Source src => Source (map (map_facts f) src)
   419     | Fact (a, ths) => Fact (a, f a ths)
   420     | _ => v));
   421 
   422 
   423 (* transform *)
   424 
   425 fun transform phi =
   426   map_value (fn v =>
   427     (case v of
   428       Source src => Source (map (transform phi) src)
   429     | Literal _ => v
   430     | Name (a, psi) => Name (a, psi $> phi)
   431     | Typ T => Typ (Morphism.typ phi T)
   432     | Term t => Term (Morphism.term phi t)
   433     | Fact (a, ths) => Fact (a, Morphism.fact phi ths)
   434     | Attribute att => Attribute (Morphism.transform phi att)
   435     | Declaration decl => Declaration (Morphism.transform phi decl)
   436     | Files _ => v));
   437 
   438 
   439 (* static binding *)
   440 
   441 (*1st stage: initialize assignable slots*)
   442 fun init_assignable tok =
   443   (case tok of
   444     Token (x, y, Slot) => Token (x, y, Assignable (Unsynchronized.ref NONE))
   445   | Token (_, _, Value _) => tok
   446   | Token (_, _, Assignable r) => (r := NONE; tok));
   447 
   448 (*2nd stage: assign values as side-effect of scanning*)
   449 fun assign v tok =
   450   (case tok of
   451     Token (x, y, Slot) => Token (x, y, Value v)
   452   | Token (_, _, Value _) => tok
   453   | Token (_, _, Assignable r) => (r := v; tok));
   454 
   455 fun evaluate mk eval arg =
   456   let val x = eval arg in (assign (SOME (mk x)) arg; x) end;
   457 
   458 (*3rd stage: static closure of final values*)
   459 fun closure (Token (x, y, Assignable (Unsynchronized.ref v))) = Token (x, y, Value v)
   460   | closure tok = tok;
   461 
   462 
   463 (* pretty *)
   464 
   465 fun pretty_value ctxt tok =
   466   (case get_value tok of
   467     SOME (Literal markup) =>
   468       let val x = content_of tok
   469       in Pretty.mark_str (keyword_markup markup x, x) end
   470   | SOME (Name ({print, ...}, _)) => Pretty.quote (Pretty.mark_str (print ctxt))
   471   | SOME (Typ T) => Syntax.pretty_typ ctxt T
   472   | SOME (Term t) => Syntax.pretty_term ctxt t
   473   | SOME (Fact (_, ths)) =>
   474       Pretty.enclose "(" ")" (Pretty.breaks (map (Pretty.cartouche o Thm.pretty_thm ctxt) ths))
   475   | _ => Pretty.marks_str (markups Keyword.empty_keywords tok, unparse tok));
   476 
   477 
   478 (* src *)
   479 
   480 fun dest_src ([]: src) = raise Fail "Empty token source"
   481   | dest_src (head :: args) = (head, args);
   482 
   483 fun name_of_src src =
   484   let
   485     val head = #1 (dest_src src);
   486     val name =
   487       (case get_name head of
   488         SOME {name, ...} => name
   489       | NONE => content_of head);
   490   in (name, pos_of head) end;
   491 
   492 val args_of_src = #2 o dest_src;
   493 
   494 fun pretty_src ctxt src =
   495   let
   496     val (head, args) = dest_src src;
   497     val prt_name =
   498       (case get_name head of
   499         SOME {print, ...} => Pretty.mark_str (print ctxt)
   500       | NONE => Pretty.str (content_of head));
   501   in Pretty.block (Pretty.breaks (Pretty.quote prt_name :: map (pretty_value ctxt) args)) end;
   502 
   503 fun checked_src (head :: _) = is_some (get_name head)
   504   | checked_src [] = true;
   505 
   506 fun check_src ctxt get_table src =
   507   let
   508     val (head, args) = dest_src src;
   509     val table = get_table ctxt;
   510   in
   511     (case get_name head of
   512       SOME {name, ...} => (src, Name_Space.get table name)
   513     | NONE =>
   514         let
   515           val (name, x) =
   516             Name_Space.check (Context.Proof ctxt) table (content_of head, pos_of head);
   517           val kind = Name_Space.kind_of (Name_Space.space_of_table table);
   518           fun print ctxt' =
   519             Name_Space.markup_extern ctxt' (Name_Space.space_of_table (get_table ctxt')) name;
   520           val value = name_value {name = name, kind = kind, print = print};
   521           val head' = closure (assign (SOME value) head);
   522         in (head' :: args, x) end)
   523   end;
   524 
   525 
   526 
   527 (** scanners **)
   528 
   529 open Basic_Symbol_Pos;
   530 
   531 val err_prefix = "Outer lexical error: ";
   532 
   533 fun !!! msg = Symbol_Pos.!!! (fn () => err_prefix ^ msg);
   534 
   535 
   536 (* scan symbolic idents *)
   537 
   538 val scan_symid =
   539   Scan.many1 (Symbol.is_symbolic_char o Symbol_Pos.symbol) ||
   540   Scan.one (Symbol.is_symbolic o Symbol_Pos.symbol) >> single;
   541 
   542 fun is_symid str =
   543   (case try Symbol.explode str of
   544     SOME [s] => Symbol.is_symbolic s orelse Symbol.is_symbolic_char s
   545   | SOME ss => forall Symbol.is_symbolic_char ss
   546   | _ => false);
   547 
   548 fun ident_or_symbolic "begin" = false
   549   | ident_or_symbolic ":" = true
   550   | ident_or_symbolic "::" = true
   551   | ident_or_symbolic s = Symbol_Pos.is_identifier s orelse is_symid s;
   552 
   553 
   554 (* scan verbatim text *)
   555 
   556 val scan_verb =
   557   $$$ "*" --| Scan.ahead (~$$ "}") ||
   558   Scan.one (fn (s, _) => s <> "*" andalso Symbol.not_eof s) >> single;
   559 
   560 val scan_verbatim =
   561   Scan.ahead ($$ "{" -- $$ "*") |--
   562     !!! "unclosed verbatim text"
   563       ((Symbol_Pos.scan_pos --| $$ "{" --| $$ "*") --
   564         (Scan.repeats scan_verb -- ($$ "*" |-- $$ "}" |-- Symbol_Pos.scan_pos)));
   565 
   566 val recover_verbatim =
   567   $$$ "{" @@@ $$$ "*" @@@ Scan.repeats scan_verb;
   568 
   569 
   570 (* scan cartouche *)
   571 
   572 val scan_cartouche =
   573   Symbol_Pos.scan_pos --
   574     ((Symbol_Pos.scan_cartouche err_prefix >> Symbol_Pos.cartouche_content) -- Symbol_Pos.scan_pos);
   575 
   576 
   577 (* scan space *)
   578 
   579 fun space_symbol (s, _) = Symbol.is_blank s andalso s <> "\n";
   580 
   581 val scan_space =
   582   Scan.many1 space_symbol @@@ Scan.optional ($$$ "\n") [] ||
   583   Scan.many space_symbol @@@ $$$ "\n";
   584 
   585 
   586 (* scan comment *)
   587 
   588 val scan_comment =
   589   Symbol_Pos.scan_pos -- (Symbol_Pos.scan_comment_body err_prefix -- Symbol_Pos.scan_pos);
   590 
   591 
   592 
   593 (** token sources **)
   594 
   595 fun source_proper src = src |> Source.filter is_proper;
   596 
   597 local
   598 
   599 fun token_leq ((_, syms1), (_, syms2)) = length syms1 <= length syms2;
   600 
   601 fun token k ss =
   602   Token ((Symbol_Pos.implode ss, Symbol_Pos.range ss), (k, Symbol_Pos.content ss), Slot);
   603 
   604 fun token_range k (pos1, (ss, pos2)) =
   605   Token (Symbol_Pos.implode_range (pos1, pos2) ss, (k, Symbol_Pos.content ss), Slot);
   606 
   607 fun scan_token keywords = !!! "bad input"
   608   (Symbol_Pos.scan_string_qq err_prefix >> token_range String ||
   609     Symbol_Pos.scan_string_bq err_prefix >> token_range Alt_String ||
   610     scan_verbatim >> token_range Verbatim ||
   611     scan_cartouche >> token_range Cartouche ||
   612     scan_comment >> token_range Comment ||
   613     scan_space >> token Space ||
   614     (Scan.max token_leq
   615       (Scan.max token_leq
   616         (Scan.literal (Keyword.major_keywords keywords) >> pair Command)
   617         (Scan.literal (Keyword.minor_keywords keywords) >> pair Keyword))
   618       (Lexicon.scan_longid >> pair Long_Ident ||
   619         Lexicon.scan_id >> pair Ident ||
   620         Lexicon.scan_var >> pair Var ||
   621         Lexicon.scan_tid >> pair Type_Ident ||
   622         Lexicon.scan_tvar >> pair Type_Var ||
   623         Symbol_Pos.scan_float >> pair Float ||
   624         Symbol_Pos.scan_nat >> pair Nat ||
   625         scan_symid >> pair Sym_Ident) >> uncurry token));
   626 
   627 fun recover msg =
   628   (Symbol_Pos.recover_string_qq ||
   629     Symbol_Pos.recover_string_bq ||
   630     recover_verbatim ||
   631     Symbol_Pos.recover_cartouche ||
   632     Symbol_Pos.recover_comment ||
   633     Scan.one (Symbol.not_eof o Symbol_Pos.symbol) >> single)
   634   >> (single o token (Error msg));
   635 
   636 in
   637 
   638 fun source' strict keywords =
   639   let
   640     val scan_strict = Scan.bulk (scan_token keywords);
   641     val scan = if strict then scan_strict else Scan.recover scan_strict recover;
   642   in Source.source Symbol_Pos.stopper scan end;
   643 
   644 fun source keywords pos src = Symbol_Pos.source pos src |> source' false keywords;
   645 fun source_strict keywords pos src = Symbol_Pos.source pos src |> source' true keywords;
   646 
   647 fun read_cartouche syms =
   648   (case Scan.read Symbol_Pos.stopper (scan_cartouche >> token_range Cartouche) syms of
   649     SOME tok => tok
   650   | NONE => error ("Single cartouche expected" ^ Position.here (#1 (Symbol_Pos.range syms))));
   651 
   652 end;
   653 
   654 
   655 (* explode *)
   656 
   657 fun explode keywords pos =
   658   Source.of_string #>
   659   Symbol.source #>
   660   source keywords pos #>
   661   Source.exhaust;
   662 
   663 
   664 (* make *)
   665 
   666 fun make ((k, n), s) pos =
   667   let
   668     val pos' = Position.advance_offset n pos;
   669     val range = Position.range (pos, pos');
   670     val tok =
   671       if 0 <= k andalso k < Vector.length immediate_kinds then
   672         Token ((s, range), (Vector.sub (immediate_kinds, k), s), Slot)
   673       else
   674         (case explode Keyword.empty_keywords pos s of
   675           [tok] => tok
   676         | _ => Token ((s, range), (Error (err_prefix ^ "exactly one token expected"), s), Slot))
   677   in (tok, pos') end;
   678 
   679 fun make_string (s, pos) =
   680   #1 (make ((~1, 0), Symbol_Pos.quote_string_qq s) Position.none)
   681   |> reset_range pos;
   682 
   683 fun make_src a args = make_string a :: args;
   684 
   685 
   686 
   687 (** parsers **)
   688 
   689 type 'a parser = T list -> 'a * T list;
   690 type 'a context_parser = Context.generic * T list -> 'a * (Context.generic * T list);
   691 
   692 
   693 (* read antiquotation source *)
   694 
   695 fun read_no_commands keywords scan syms =
   696   Source.of_list syms
   697   |> source' true (Keyword.no_command_keywords keywords)
   698   |> source_proper
   699   |> Source.source stopper (Scan.error (Scan.bulk scan))
   700   |> Source.exhaust;
   701 
   702 fun read_antiq keywords scan (syms, pos) =
   703   let
   704     fun err msg =
   705       cat_error msg ("Malformed antiquotation" ^ Position.here pos ^ ":\n" ^
   706         "@{" ^ Symbol_Pos.content syms ^ "}");
   707     val res = read_no_commands keywords scan syms handle ERROR msg => err msg;
   708   in (case res of [x] => x | _ => err "") end;
   709 
   710 
   711 (* wrapped syntax *)
   712 
   713 fun syntax_generic scan src context =
   714   let
   715     val (name, pos) = name_of_src src;
   716     val args1 = map init_assignable (args_of_src src);
   717     fun reported_text () =
   718       if Context_Position.is_visible_generic context then
   719         ((pos, Markup.operator) :: maps (reports_of_value o closure) args1)
   720         |> map (fn (p, m) => Position.reported_text p m "")
   721       else [];
   722   in
   723     (case Scan.error (Scan.finite' stopper (Scan.option scan)) (context, args1) of
   724       (SOME x, (context', [])) =>
   725         let val _ = Output.report (reported_text ())
   726         in (x, context') end
   727     | (_, (context', args2)) =>
   728         let
   729           val print_name =
   730             (case get_name (hd src) of
   731               NONE => quote name
   732             | SOME {kind, print, ...} =>
   733                 let
   734                   val ctxt' = Context.proof_of context';
   735                   val (markup, xname) = print ctxt';
   736                 in plain_words kind ^ " " ^ quote (Markup.markup markup xname) end);
   737           val print_args =
   738             if null args2 then "" else ":\n  " ^ space_implode " " (map print args2);
   739         in
   740           error ("Bad arguments for " ^ print_name ^ Position.here pos ^ print_args ^
   741             Markup.markup_report (implode (reported_text ())))
   742         end)
   743   end;
   744 
   745 fun syntax scan src = apsnd Context.the_proof o syntax_generic scan src o Context.Proof;
   746 
   747 end;
   748 
   749 type 'a parser = 'a Token.parser;
   750 type 'a context_parser = 'a Token.context_parser;