src/Pure/ML/ml_lex.ML
changeset 78687 5fe4c11b5ecb
parent 74373 6e4093927dbb
child 78701 c7b2697094ac
equal deleted inserted replaced
78686:c37f2eb8d038 78687:5fe4c11b5ecb
    53   "let", "local", "nonfix", "of", "op", "open", "orelse", "raise",
    53   "let", "local", "nonfix", "of", "op", "open", "orelse", "raise",
    54   "rec", "sharing", "sig", "signature", "struct", "structure", "then",
    54   "rec", "sharing", "sig", "signature", "struct", "structure", "then",
    55   "type", "val", "where", "while", "with", "withtype"];
    55   "type", "val", "where", "while", "with", "withtype"];
    56 
    56 
    57 val keywords2 =
    57 val keywords2 =
    58  ["and", "case", "do", "else", "end", "if", "in", "let", "local", "of",
    58  Symset.make ["and", "case", "do", "else", "end", "if", "in", "let", "local", "of",
    59   "sig", "struct", "then", "while", "with"];
    59   "sig", "struct", "then", "while", "with"];
    60 
    60 
    61 val keywords3 =
    61 val keywords3 =
    62  ["handle", "open", "raise"];
    62  Symset.make ["handle", "open", "raise"];
    63 
    63 
    64 val lexicon = Scan.make_lexicon (map raw_explode keywords);
    64 val lexicon = Scan.make_lexicon (map raw_explode keywords);
    65 
    65 
    66 
    66 
    67 
    67 
   166 fun token_report (tok as Token ((pos, _), (kind, x))) =
   166 fun token_report (tok as Token ((pos, _), (kind, x))) =
   167   let
   167   let
   168     val (markup, txt) =
   168     val (markup, txt) =
   169       if not (is_keyword tok) then token_kind_markup kind
   169       if not (is_keyword tok) then token_kind_markup kind
   170       else if is_delimiter tok then (Markup.ML_delimiter, "")
   170       else if is_delimiter tok then (Markup.ML_delimiter, "")
   171       else if member (op =) keywords2 x then (Markup.ML_keyword2 |> Markup.keyword_properties, "")
   171       else if Symset.member keywords2 x then (Markup.ML_keyword2 |> Markup.keyword_properties, "")
   172       else if member (op =) keywords3 x then (Markup.ML_keyword3 |> Markup.keyword_properties, "")
   172       else if Symset.member keywords3 x then (Markup.ML_keyword3 |> Markup.keyword_properties, "")
   173       else (Markup.ML_keyword1 |> Markup.keyword_properties, "");
   173       else (Markup.ML_keyword1 |> Markup.keyword_properties, "");
   174   in ((pos, markup), txt) end;
   174   in ((pos, markup), txt) end;
   175 
   175 
   176 end;
   176 end;
   177 
   177 
   194   Scan.many (Symbol.is_ascii_letdig o Symbol_Pos.symbol);
   194   Scan.many (Symbol.is_ascii_letdig o Symbol_Pos.symbol);
   195 
   195 
   196 val scan_alphanumeric =
   196 val scan_alphanumeric =
   197   Scan.one (Symbol.is_ascii_letter o Symbol_Pos.symbol) ::: scan_letdigs;
   197   Scan.one (Symbol.is_ascii_letter o Symbol_Pos.symbol) ::: scan_letdigs;
   198 
   198 
   199 val scan_symbolic =
   199 val symbolic = Symset.make (raw_explode "!#$%&*+-/:<=>?@\\^`|~");
   200   Scan.many1 (member (op =) (raw_explode "!#$%&*+-/:<=>?@\\^`|~") o Symbol_Pos.symbol);
   200 
       
   201 val scan_symbolic = Scan.many1 (Symset.member symbolic o Symbol_Pos.symbol);
   201 
   202 
   202 in
   203 in
   203 
   204 
   204 val scan_ident = scan_alphanumeric || scan_symbolic;
   205 val scan_ident = scan_alphanumeric || scan_symbolic;
   205 
   206 
   242 
   243 
   243 val scan_blanks1 = Scan.many1 (Symbol.is_ascii_blank o Symbol_Pos.symbol);
   244 val scan_blanks1 = Scan.many1 (Symbol.is_ascii_blank o Symbol_Pos.symbol);
   244 
   245 
   245 local
   246 local
   246 
   247 
       
   248 val escape = Symset.make (raw_explode "\"\\abtnvfr");
       
   249 
   247 val scan_escape =
   250 val scan_escape =
   248   Scan.one (member (op =) (raw_explode "\"\\abtnvfr") o Symbol_Pos.symbol) >> single ||
   251   Scan.one (Symset.member escape o Symbol_Pos.symbol) >> single ||
   249   $$$ "^" @@@
   252   $$$ "^" @@@
   250     (Scan.one (fn (s, _) => Char.ord #"@" <= ord s andalso ord s <= Char.ord #"_") >> single) ||
   253     (Scan.one (fn (s, _) => Char.ord #"@" <= ord s andalso ord s <= Char.ord #"_") >> single) ||
   251   Scan.one (Symbol.is_ascii_digit o Symbol_Pos.symbol) --
   254   Scan.one (Symbol.is_ascii_digit o Symbol_Pos.symbol) --
   252     Scan.one (Symbol.is_ascii_digit o Symbol_Pos.symbol) --
   255     Scan.one (Symbol.is_ascii_digit o Symbol_Pos.symbol) --
   253     Scan.one (Symbol.is_ascii_digit o Symbol_Pos.symbol) >> (fn ((a, b), c) => [a, b, c]);
   256     Scan.one (Symbol.is_ascii_digit o Symbol_Pos.symbol) >> (fn ((a, b), c) => [a, b, c]);