src/Pure/ML/ml_lex.ML
changeset 30573 49899f26fbd1
parent 29606 fedb8be05f24
child 30591 6c9c00ea4ae6
equal deleted inserted replaced
30572:8fbc355100f2 30573:49899f26fbd1
    16   val pos_of: token -> string
    16   val pos_of: token -> string
    17   val kind_of: token -> token_kind
    17   val kind_of: token -> token_kind
    18   val content_of: token -> string
    18   val content_of: token -> string
    19   val keywords: string list
    19   val keywords: string list
    20   val source: (Symbol.symbol, 'a) Source.source ->
    20   val source: (Symbol.symbol, 'a) Source.source ->
    21     (token, (SymbolPos.T, Position.T * (Symbol.symbol, 'a) Source.source)
    21     (token, (Symbol_Pos.T, Position.T * (Symbol.symbol, 'a) Source.source)
    22       Source.source) Source.source
    22       Source.source) Source.source
    23 end;
    23 end;
    24 
    24 
    25 structure ML_Lex: ML_LEX =
    25 structure ML_Lex: ML_LEX =
    26 struct
    26 struct
    73 
    73 
    74 
    74 
    75 
    75 
    76 (** scanners **)
    76 (** scanners **)
    77 
    77 
    78 open BasicSymbolPos;
    78 open Basic_Symbol_Pos;
    79 
    79 
    80 fun !!! msg = SymbolPos.!!! ("SML lexical error: " ^ msg);
    80 fun !!! msg = Symbol_Pos.!!! ("SML lexical error: " ^ msg);
    81 
    81 
    82 
    82 
    83 (* blanks *)
    83 (* blanks *)
    84 
    84 
    85 val scan_blank = Scan.one (Symbol.is_ascii_blank o symbol);
    85 val scan_blank = Scan.one (Symbol.is_ascii_blank o symbol);
   181 
   181 
   182 (* token source *)
   182 (* token source *)
   183 
   183 
   184 local
   184 local
   185 
   185 
   186 fun token k ss = Token (SymbolPos.range ss, (k, SymbolPos.implode ss));
   186 fun token k ss = Token (Symbol_Pos.range ss, (k, Symbol_Pos.implode ss));
   187 
   187 
   188 val scan = !!! "bad input"
   188 val scan = !!! "bad input"
   189  (scan_char >> token Char ||
   189  (scan_char >> token Char ||
   190   scan_string >> token String ||
   190   scan_string >> token String ||
   191   scan_blanks1 >> token Space ||
   191   scan_blanks1 >> token Space ||
   192   SymbolPos.scan_comment !!! >> token Comment ||
   192   Symbol_Pos.scan_comment !!! >> token Comment ||
   193   Scan.max token_leq
   193   Scan.max token_leq
   194    (scan_keyword >> token Keyword)
   194    (scan_keyword >> token Keyword)
   195    (scan_word >> token Word ||
   195    (scan_word >> token Word ||
   196     scan_real >> token Real ||
   196     scan_real >> token Real ||
   197     scan_int >> token Int ||
   197     scan_int >> token Int ||
   204   >> (fn cs => [token (Error msg) cs]);
   204   >> (fn cs => [token (Error msg) cs]);
   205 
   205 
   206 in
   206 in
   207 
   207 
   208 fun source src =
   208 fun source src =
   209   SymbolPos.source (Position.line 1) src
   209   Symbol_Pos.source (Position.line 1) src
   210   |> Source.source SymbolPos.stopper (Scan.bulk scan) (SOME (false, recover));
   210   |> Source.source Symbol_Pos.stopper (Scan.bulk scan) (SOME (false, recover));
   211 
   211 
   212 end;
   212 end;
   213 
   213 
   214 end;
   214 end;
   215 
   215