src/Pure/ML/ml_lex.ML
changeset 40523 1050315f6ee2
parent 40337 d25bbb5f1c9e
child 40525 14a2e686bdac
equal deleted inserted replaced
40522:37b79d789d91 40523:1050315f6ee2
   263 
   263 
   264 fun source src =
   264 fun source src =
   265   Symbol_Pos.source (Position.line 1) src
   265   Symbol_Pos.source (Position.line 1) src
   266   |> Source.source Symbol_Pos.stopper (Scan.bulk (!!! "bad input" scan_ml)) (SOME (false, recover));
   266   |> Source.source Symbol_Pos.stopper (Scan.bulk (!!! "bad input" scan_ml)) (SOME (false, recover));
   267 
   267 
   268 val tokenize =
   268 val tokenize = Source.of_string #> Symbol.source #> source #> Source.exhaust;
   269   Source.of_string #>
       
   270   Symbol.source {do_recover = true} #>
       
   271   source #>
       
   272   Source.exhaust;
       
   273 
   269 
   274 fun read pos txt =
   270 fun read pos txt =
   275   let
   271   let
   276     val _ = Position.report pos Markup.ML_source;
   272     val _ = Position.report pos Markup.ML_source;
   277     val syms = Symbol_Pos.explode (txt, pos);
   273     val syms = Symbol_Pos.explode (txt, pos);