src/Pure/Syntax/lexicon.ML
changeset 67361 f834d6f21c55
parent 67352 5f7f339f3d7e
child 67367 2b11c071d016
equal deleted inserted replaced
67360:fbc31dea36fa 67361:f834d6f21c55
   244 
   244 
   245 fun implode_str ss = enclose "''" "''" (implode (map (fn "'" => "\\'" | s => s) ss));
   245 fun implode_str ss = enclose "''" "''" (implode (map (fn "'" => "\\'" | s => s) ss));
   246 val explode_str = explode_literal scan_str_body;
   246 val explode_str = explode_literal scan_str_body;
   247 
   247 
   248 
   248 
   249 (* comment cartouche *)
       
   250 
       
   251 val scan_comment_cartouche =
       
   252   $$$ "\<comment>" @@@ Scan.many (Symbol.is_blank o Symbol_Pos.symbol) @@@
       
   253     !!! "cartouche expected after \"\<comment>\"" (Symbol_Pos.scan_cartouche err_prefix);
       
   254 
       
   255 
   249 
   256 (** tokenize **)
   250 (** tokenize **)
   257 
   251 
   258 fun token_leq (Token (_, s1, _), Token (_, s2, _)) = s1 <= s2;
   252 fun token_leq (Token (_, s1, _), Token (_, s2, _)) = s1 <= s2;
   259 fun token kind ss = Token (kind, Symbol_Pos.content ss, Symbol_Pos.range ss);
   253 fun token kind ss = Token (kind, Symbol_Pos.content ss, Symbol_Pos.range ss);
   276     val scan_lit = Scan.literal lex >> token Literal;
   270     val scan_lit = Scan.literal lex >> token Literal;
   277 
   271 
   278     val scan_token =
   272     val scan_token =
   279       Symbol_Pos.scan_cartouche err_prefix >> token Cartouche ||
   273       Symbol_Pos.scan_cartouche err_prefix >> token Cartouche ||
   280       Symbol_Pos.scan_comment err_prefix >> token Comment ||
   274       Symbol_Pos.scan_comment err_prefix >> token Comment ||
   281       scan_comment_cartouche >> token Comment_Cartouche ||
   275       Symbol_Pos.scan_comment_cartouche err_prefix >> token Comment_Cartouche ||
   282       Scan.max token_leq scan_lit scan_val ||
   276       Scan.max token_leq scan_lit scan_val ||
   283       scan_string >> token StringSy ||
   277       scan_string >> token StringSy ||
   284       scan_str >> token StrSy ||
   278       scan_str >> token StrSy ||
   285       Scan.many1 (Symbol.is_blank o Symbol_Pos.symbol) >> token Space;
   279       Scan.many1 (Symbol.is_blank o Symbol_Pos.symbol) >> token Space;
   286   in
   280   in