src/Pure/Thy/thy_scan.ML
changeset 15531 08c8dad8e399
parent 14981 e73f8140af78
child 15570 8d8c70b41bab
equal deleted inserted replaced
15530:6f43714517ee 15531:08c8dad8e399
    37 
    37 
    38 (** scanners **)
    38 (** scanners **)
    39 
    39 
    40 (* diagnostics *)
    40 (* diagnostics *)
    41 
    41 
    42 fun lex_err None msg = "Outer lexical error: " ^ msg
    42 fun lex_err NONE msg = "Outer lexical error: " ^ msg
    43   | lex_err (Some n) msg = "Outer lexical error on line " ^ string_of_int n ^ ": " ^ msg;
    43   | lex_err (SOME n) msg = "Outer lexical error on line " ^ string_of_int n ^ ": " ^ msg;
    44 
    44 
    45 
    45 
    46 (* line numbering *)
    46 (* line numbering *)
    47 
    47 
    48 val incr = apsome (fn n:int => n + 1);
    48 val incr = apsome (fn n:int => n + 1);
   107     (Scan.pass 0 (Scan.repeat scan_cmt) |-- keep_line ($$ "*" -- $$ ")") >> K "");
   107     (Scan.pass 0 (Scan.repeat scan_cmt) |-- keep_line ($$ "*" -- $$ ")") >> K "");
   108 
   108 
   109 
   109 
   110 (* scan token *)
   110 (* scan token *)
   111 
   111 
   112 fun token k None x = (k, x, 0)
   112 fun token k NONE x = (k, x, 0)
   113   | token k (Some n) x = (k, x, n);
   113   | token k (SOME n) x = (k, x, n);
   114 
   114 
   115 fun scan_tok lex (n, cs) =
   115 fun scan_tok lex (n, cs) =
   116  (scan_string >> token String n ||
   116  (scan_string >> token String n ||
   117   scan_verbatim >> token Verbatim n ||
   117   scan_verbatim >> token Verbatim n ||
   118   Scan.repeat1 scan_blank >> (token Ignore n o implode) ||
   118   Scan.repeat1 scan_blank >> (token Ignore n o implode) ||
   127 
   127 
   128 val scan_rest = Scan.any Symbol.not_eof >> implode;
   128 val scan_rest = Scan.any Symbol.not_eof >> implode;
   129 
   129 
   130 fun scan_token lex x =
   130 fun scan_token lex x =
   131   (case scan_tok lex x of
   131   (case scan_tok lex x of
   132     ((Keyword, "ML", n), x') => (keep_line scan_rest >> token Verbatim (Some n)) x'
   132     ((Keyword, "ML", n), x') => (keep_line scan_rest >> token Verbatim (SOME n)) x'
   133   | tok_x' => tok_x');
   133   | tok_x' => tok_x');
   134 
   134 
   135 
   135 
   136 (* tokenize *)
   136 (* tokenize *)
   137 
   137 
   138 fun tokenize lex chs =
   138 fun tokenize lex chs =
   139   let
   139   let
   140     val scan_toks = Scan.repeat (scan_token lex);
   140     val scan_toks = Scan.repeat (scan_token lex);
   141     val ignore = fn (Ignore, _, _) => true | _ => false;
   141     val ignore = fn (Ignore, _, _) => true | _ => false;
   142   in
   142   in
   143     (case Scan.error (Scan.finite' Symbol.stopper scan_toks) (Some 1, chs) of
   143     (case Scan.error (Scan.finite' Symbol.stopper scan_toks) (SOME 1, chs) of
   144       (toks, (n, [])) => filter_out ignore toks @ [token EOF n "end-of-file"]
   144       (toks, (n, [])) => filter_out ignore toks @ [token EOF n "end-of-file"]
   145     | (_, (n, cs)) => error (lex_err n ("Bad input " ^ quote (Symbol.beginning 10 cs))))
   145     | (_, (n, cs)) => error (lex_err n ("Bad input " ^ quote (Symbol.beginning 10 cs))))
   146   end;
   146   end;
   147 
   147 
   148 
   148