src/Pure/Syntax/lexicon.ML
changeset 50238 98d35a7368bd
parent 50201 c26369c9eda6
child 50239 fb579401dc26
equal deleted inserted replaced
50237:e356f86729bc 50238:98d35a7368bd
    11     val const: string -> term
    11     val const: string -> term
    12     val free: string -> term
    12     val free: string -> term
    13     val var: indexname -> term
    13     val var: indexname -> term
    14   end
    14   end
    15   val is_identifier: string -> bool
    15   val is_identifier: string -> bool
    16   val is_ascii_identifier: string -> bool
       
    17   val is_xid: string -> bool
    16   val is_xid: string -> bool
    18   val is_tid: string -> bool
    17   val is_tid: string -> bool
    19   val scan_id: Symbol_Pos.T list -> Symbol_Pos.T list * Symbol_Pos.T list
    18   val scan_id: Symbol_Pos.T list -> Symbol_Pos.T list * Symbol_Pos.T list
    20   val scan_longid: Symbol_Pos.T list -> Symbol_Pos.T list * Symbol_Pos.T list
    19   val scan_longid: Symbol_Pos.T list -> Symbol_Pos.T list * Symbol_Pos.T list
    21   val scan_tid: Symbol_Pos.T list -> Symbol_Pos.T list * Symbol_Pos.T list
    20   val scan_tid: Symbol_Pos.T list -> Symbol_Pos.T list * Symbol_Pos.T list
    93 
    92 
    94 (** is_identifier etc. **)
    93 (** is_identifier etc. **)
    95 
    94 
    96 val is_identifier = Symbol.is_ident o Symbol.explode;
    95 val is_identifier = Symbol.is_ident o Symbol.explode;
    97 
    96 
    98 fun is_ascii_identifier s =
       
    99   let val cs = Symbol.explode s
       
   100   in forall Symbol.is_ascii cs andalso Symbol.is_ident cs end;
       
   101 
       
   102 fun is_xid s =
    97 fun is_xid s =
   103   (case Symbol.explode s of
    98   (case Symbol.explode s of
   104     "_" :: cs => Symbol.is_ident cs
    99     "_" :: cs => Symbol.is_ident cs
   105   | cs => Symbol.is_ident cs);
   100   | cs => Symbol.is_ident cs);
   106 
   101 
   188 
   183 
   189 
   184 
   190 (* markup *)
   185 (* markup *)
   191 
   186 
   192 fun literal_markup s =
   187 fun literal_markup s =
   193   if is_ascii_identifier s then Markup.literal else Markup.delimiter;
   188   if Symbol.is_ascii_identifier s then Markup.literal else Markup.delimiter;
   194 
   189 
   195 val token_kind_markup =
   190 val token_kind_markup =
   196  fn VarSy   => Markup.var
   191  fn VarSy   => Markup.var
   197   | TFreeSy => Markup.tfree
   192   | TFreeSy => Markup.tfree
   198   | TVarSy  => Markup.tvar
   193   | TVarSy  => Markup.tvar