src/Pure/Syntax/lexicon.ML
changeset 4938 c8bbbf3c59fa
parent 4921 74bc10921f7d
child 5112 9e74cf11e4a4
equal deleted inserted replaced
4937:e3132cf1d68e 4938:c8bbbf3c59fa
   234 
   234 
   235 
   235 
   236 fun implode_xstr cs = enclose "''" "''" (implode (map (fn "'" => "\\'" | c => c) cs));
   236 fun implode_xstr cs = enclose "''" "''" (implode (map (fn "'" => "\\'" | c => c) cs));
   237 
   237 
   238 fun explode_xstr str =
   238 fun explode_xstr str =
   239   #1 (Scan.error (Scan.finite Symbol.eof scan_str) (Symbol.explode str));
   239   #1 (Scan.error (Scan.finite Symbol.stopper scan_str) (Symbol.explode str));
   240 
   240 
   241 
   241 
   242 
   242 
   243 (** tokenize **)
   243 (** tokenize **)
   244 
   244 
   261     val scan_token =
   261     val scan_token =
   262       Scan.max (op <= o pairself snd) scan_lit scan_val >> (fn (tk, s) => Some (tk s)) ||
   262       Scan.max (op <= o pairself snd) scan_lit scan_val >> (fn (tk, s) => Some (tk s)) ||
   263       scan_str >> (Some o StrSy o implode_xstr) ||
   263       scan_str >> (Some o StrSy o implode_xstr) ||
   264       Scan.one Symbol.is_blank >> K None;
   264       Scan.one Symbol.is_blank >> K None;
   265   in
   265   in
   266     (case Scan.error (Scan.finite Symbol.eof (Scan.repeat scan_token)) chs of
   266     (case Scan.error (Scan.finite Symbol.stopper (Scan.repeat scan_token)) chs of
   267       (toks, []) => mapfilter I toks @ [EndToken]
   267       (toks, []) => mapfilter I toks @ [EndToken]
   268     | (_, cs) => error ("Inner lexical error at: " ^ quote (implode cs)))
   268     | (_, cs) => error ("Inner lexical error at: " ^ quote (implode cs)))
   269   end;
   269   end;
   270 
   270 
   271 
   271 
   295 
   295 
   296 
   296 
   297 (* indexname *)
   297 (* indexname *)
   298 
   298 
   299 fun indexname cs =
   299 fun indexname cs =
   300   (case Scan.error (Scan.finite Symbol.eof (Scan.option scan_vname)) cs of
   300   (case Scan.error (Scan.finite Symbol.stopper (Scan.option scan_vname)) cs of
   301     (Some xi, []) => xi
   301     (Some xi, []) => xi
   302   | _ => error ("Lexical error in variable name: " ^ quote (implode cs)));
   302   | _ => error ("Lexical error in variable name: " ^ quote (implode cs)));
   303 
   303 
   304 
   304 
   305 (* read_var *)
   305 (* read_var *)
   315     val scan =
   315     val scan =
   316       $$ "?" |-- $$ "'" |-- scan_vname >> tvar ||
   316       $$ "?" |-- $$ "'" |-- scan_vname >> tvar ||
   317       $$ "?" |-- scan_vname >> var ||
   317       $$ "?" |-- scan_vname >> var ||
   318       Scan.any Symbol.not_eof >> (free o implode);
   318       Scan.any Symbol.not_eof >> (free o implode);
   319   in
   319   in
   320     #1 (Scan.error (Scan.finite Symbol.eof scan) (Symbol.explode str))
   320     #1 (Scan.error (Scan.finite Symbol.stopper scan) (Symbol.explode str))
   321   end;
   321   end;
   322 
   322 
   323 
   323 
   324 end;
   324 end;