src/Pure/Isar/outer_lex.ML

 
 signature OUTER_LEX =
 sig
   datatype token_kind =
     Keyword | Ident | LongIdent | SymIdent | Var | TextVar | TypeIdent | TypeVar | Nat |
-    String | Verbatim | Ignore | EOF
+    String | Verbatim | Ignore | Sync | EOF
   type token
   val str_of_kind: token_kind -> string
   val stopper: token * (token -> bool)
+  val not_sync: token -> bool
   val not_eof: token -> bool
   val position_of: token -> Position.T
   val pos_of: token -> string
   val is_kind: token_kind -> token -> bool
   val keyword_pred: (string -> bool) -> token -> bool

 
 (* datatype token *)
 
 datatype token_kind =
   Keyword | Ident | LongIdent | SymIdent | Var | TextVar | TypeIdent | TypeVar | Nat |
-  String | Verbatim | Ignore | EOF;
+  String | Verbatim | Ignore | Sync | EOF;
 
 datatype token = Token of Position.T * (token_kind * string);
 
 val str_of_kind =
  fn Keyword => "keyword"

   | TypeVar => "schematic type variable"
   | Nat => "number"
   | String => "string"
   | Verbatim => "verbatim text"
   | Ignore => "ignored text"
+  | Sync => "sync marker"
   | EOF => "end-of-file";
+
+
+(* sync token *)
+
+fun not_sync (Token (_, (Sync, _))) = false
+  | not_sync _ = true;
 
 
 (* eof token *)
 
 val eof = Token (Position.none, (EOF, ""));

 
 (* scan strings *)
 
 val scan_str =
   scan_blank >> K Symbol.space ||
-  keep_line ($$ "\\" |-- Scan.one Symbol.not_eof) ||
-  keep_line (Scan.one (not_equal "\\" andf not_equal "\"" andf Symbol.not_eof));
+  keep_line ($$ "\\" |-- Scan.one (Symbol.not_sync andf Symbol.not_eof)) ||
+  keep_line (Scan.one (not_equal "\\" andf not_equal "\"" andf
+    Symbol.not_sync andf Symbol.not_eof));
 
 val scan_string =
   keep_line ($$ "\"") |--
     !! (lex_err (K "missing quote at end of string"))
       (change_prompt ((Scan.repeat scan_str >> implode) --| keep_line ($$ "\"")));

 (* scan verbatim text *)
 
 val scan_verb =
   scan_blank ||
   keep_line ($$ "*" --| Scan.ahead (Scan.one (not_equal "}"))) ||
-  keep_line (Scan.one (not_equal "*" andf Symbol.not_eof));
+  keep_line (Scan.one (not_equal "*" andf Symbol.not_sync andf Symbol.not_eof));
 
 val scan_verbatim =
   keep_line ($$ "{" -- $$ "*") |--
     !! (lex_err (K "missing end of verbatim text"))
       (change_prompt ((Scan.repeat scan_verb >> implode) --| keep_line ($$ "*" -- $$ "}")));

 val scan_cmt =
   Scan.lift scan_blank ||
   Scan.depend (fn d => keep_line ($$ "(" ^^ $$ "*") >> pair (d + 1)) ||
   Scan.depend (fn 0 => Scan.fail | d => keep_line ($$ "*" ^^ $$ ")") >> pair (d - 1)) ||
   Scan.lift (keep_line ($$ "*" --| Scan.ahead (Scan.one (not_equal ")")))) ||
-  Scan.lift (keep_line (Scan.one (not_equal "*" andf Symbol.not_eof)));
+  Scan.lift (keep_line (Scan.one (not_equal "*" andf Symbol.not_sync andf Symbol.not_eof)));
 
 val scan_comment =
   keep_line ($$ "(" -- $$ "*") |--
     !! (lex_err (K "missing end of comment"))
       (change_prompt

 
 fun scan lex (pos, cs) =
   let
     fun token k x = Token (pos, (k, x));
     fun ignore _ = token Ignore "";
+    fun sync _ = token Sync Symbol.sync;
 
     val scanner =
       scan_string >> token String ||
       scan_verbatim >> token Verbatim ||
       scan_space >> ignore ||
       scan_comment >> ignore ||
+      keep_line (Scan.one Symbol.is_sync >> sync) ||
       keep_line (Scan.max token_leq
         (Scan.literal lex >> (token Keyword o implode))
         (Syntax.scan_longid >> token LongIdent ||
           Syntax.scan_id >> token Ident ||
           Syntax.scan_var >> token Var ||
           $$ "?" ^^ $$ "?" ^^ Syntax.scan_id >> token TextVar ||
           Syntax.scan_tid >> token TypeIdent ||
           Syntax.scan_tvar >> token TypeVar ||
           Syntax.scan_nat >> token Nat ||
           scan_symid >> token SymIdent));
-  in
-    !! (lex_err (fn cs => "bad input " ^ quote (Symbol.beginning cs))) scanner (pos, cs)
-  end;
+  in !! (lex_err (fn cs => "bad input " ^ quote (Symbol.beginning cs))) scanner (pos, cs) end;
 
 
 (* source of (proper) tokens *)
 
-fun recover xs = keep_line (Scan.any1 ((not o Symbol.is_blank) andf Symbol.not_eof)) xs;
+val is_junk = (not o Symbol.is_blank) andf Symbol.not_sync andf Symbol.not_eof;
+fun recover xs = keep_line (Scan.any1 is_junk) xs;
 
 fun source do_recover get_lex pos src =
   Source.source' pos Symbol.stopper (Scan.bulk (fn xs => scan (get_lex ()) xs))
     (if do_recover then Some recover else None) src
   |> Source.filter is_proper;