src/Pure/ML/ml_lex.scala
changeset 67438 fdb7b995974d
parent 67366 e2575ccc0f5c
child 67509 524dc5c2a031
equal deleted inserted replaced
67437:a6bf7167c5e1 67438:fdb7b995974d
    48     val INT = Value("integer")
    48     val INT = Value("integer")
    49     val REAL = Value("real")
    49     val REAL = Value("real")
    50     val CHAR = Value("character")
    50     val CHAR = Value("character")
    51     val STRING = Value("quoted string")
    51     val STRING = Value("quoted string")
    52     val SPACE = Value("white space")
    52     val SPACE = Value("white space")
    53     val COMMENT = Value("comment text")
    53     val INFORMAL_COMMENT = Value("informal comment")
    54     val COMMENT_CARTOUCHE = Value("comment cartouche")
    54     val FORMAL_COMMENT = Value("formal comment")
    55     val CONTROL = Value("control symbol antiquotation")
    55     val CONTROL = Value("control symbol antiquotation")
    56     val ANTIQ = Value("antiquotation")
    56     val ANTIQ = Value("antiquotation")
    57     val ANTIQ_START = Value("antiquotation: start")
    57     val ANTIQ_START = Value("antiquotation: start")
    58     val ANTIQ_STOP = Value("antiquotation: stop")
    58     val ANTIQ_STOP = Value("antiquotation: stop")
    59     val ANTIQ_OTHER = Value("antiquotation: other")
    59     val ANTIQ_OTHER = Value("antiquotation: other")
    66   sealed case class Token(kind: Kind.Value, source: String)
    66   sealed case class Token(kind: Kind.Value, source: String)
    67   {
    67   {
    68     def is_keyword: Boolean = kind == Kind.KEYWORD
    68     def is_keyword: Boolean = kind == Kind.KEYWORD
    69     def is_delimiter: Boolean = is_keyword && !Symbol.is_ascii_identifier(source)
    69     def is_delimiter: Boolean = is_keyword && !Symbol.is_ascii_identifier(source)
    70     def is_space: Boolean = kind == Kind.SPACE
    70     def is_space: Boolean = kind == Kind.SPACE
    71     def is_comment: Boolean = kind == Kind.COMMENT || kind == Kind.COMMENT_CARTOUCHE
    71     def is_comment: Boolean = kind == Kind.INFORMAL_COMMENT || kind == Kind.FORMAL_COMMENT
    72   }
    72   }
    73 
    73 
    74 
    74 
    75 
    75 
    76   /** parsers **/
    76   /** parsers **/
    77 
    77 
    78   case object ML_String extends Scan.Line_Context
    78   case object ML_String extends Scan.Line_Context
    79   case class Antiq(ctxt: Scan.Line_Context) extends Scan.Line_Context
    79   case class Antiq(ctxt: Scan.Line_Context) extends Scan.Line_Context
    80 
    80 
    81   private object Parsers extends Scan.Parsers with Antiquote.Parsers
    81   private object Parsers extends Scan.Parsers with Antiquote.Parsers with Comment.Parsers
    82   {
    82   {
    83     /* string material */
    83     /* string material */
    84 
    84 
    85     private val blanks = many(character(Symbol.is_ascii_blank))
    85     private val blanks = many(character(Symbol.is_ascii_blank))
    86     private val blanks1 = many1(character(Symbol.is_ascii_blank))
    86     private val blanks1 = many1(character(Symbol.is_ascii_blank))
   138 
   138 
   139 
   139 
   140     /* ML comment */
   140     /* ML comment */
   141 
   141 
   142     private val ml_comment: Parser[Token] =
   142     private val ml_comment: Parser[Token] =
   143       comment ^^ (x => Token(Kind.COMMENT, x))
   143       comment ^^ (x => Token(Kind.INFORMAL_COMMENT, x))
   144 
   144 
   145     private def ml_comment_line(ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] =
   145     private def ml_comment_line(ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] =
   146       comment_line(ctxt) ^^ { case (x, c) => (Token(Kind.COMMENT, x), c) }
   146       comment_line(ctxt) ^^ { case (x, c) => (Token(Kind.INFORMAL_COMMENT, x), c) }
   147 
   147 
   148     private val ml_comment_cartouche: Parser[Token] =
   148     private val ml_formal_comment: Parser[Token] =
   149       comment_cartouche ^^ (x => Token(Kind.COMMENT_CARTOUCHE, x))
   149       comment_cartouche ^^ (x => Token(Kind.FORMAL_COMMENT, x))
   150 
   150 
   151     private def ml_comment_cartouche_line(ctxt: Scan.Line_Context)
   151     private def ml_formal_comment_line(ctxt: Scan.Line_Context)
   152         : Parser[(Token, Scan.Line_Context)] =
   152         : Parser[(Token, Scan.Line_Context)] =
   153       comment_cartouche_line(ctxt) ^^ { case (x, c) => (Token(Kind.COMMENT_CARTOUCHE, x), c) }
   153       comment_cartouche_line(ctxt) ^^ { case (x, c) => (Token(Kind.FORMAL_COMMENT, x), c) }
   154 
   154 
   155 
   155 
   156     /* antiquotations (line-oriented) */
   156     /* antiquotations (line-oriented) */
   157 
   157 
   158     def ml_cartouche_line(ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] =
   158     def ml_cartouche_line(ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] =
   250       space | (ml_control | (recover | (ml_antiq |
   250       space | (ml_control | (recover | (ml_antiq |
   251         (((word | (real | (int | (long_ident | (ident | type_var))))) ||| keyword) | bad))))
   251         (((word | (real | (int | (long_ident | (ident | type_var))))) ||| keyword) | bad))))
   252     }
   252     }
   253 
   253 
   254     def token: Parser[Token] =
   254     def token: Parser[Token] =
   255       ml_char | (ml_string | (ml_comment | (ml_comment_cartouche | other_token)))
   255       ml_char | (ml_string | (ml_comment | (ml_formal_comment | other_token)))
   256 
   256 
   257     def token_line(SML: Boolean, ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] =
   257     def token_line(SML: Boolean, ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] =
   258     {
   258     {
   259       val other = (ml_char | other_token) ^^ (x => (x, Scan.Finished))
   259       val other = (ml_char | other_token) ^^ (x => (x, Scan.Finished))
   260 
   260 
   261       if (SML) ml_string_line(ctxt) | (ml_comment_line(ctxt) | other)
   261       if (SML) ml_string_line(ctxt) | (ml_comment_line(ctxt) | other)
   262       else {
   262       else {
   263         ml_string_line(ctxt) |
   263         ml_string_line(ctxt) |
   264           (ml_comment_line(ctxt) |
   264           (ml_comment_line(ctxt) |
   265             (ml_comment_cartouche_line(ctxt) |
   265             (ml_formal_comment_line(ctxt) |
   266               (ml_cartouche_line(ctxt) |
   266               (ml_cartouche_line(ctxt) |
   267                 (ml_antiq_start(ctxt) | (ml_antiq_stop(ctxt) | (ml_antiq_body(ctxt) | other))))))
   267                 (ml_antiq_start(ctxt) | (ml_antiq_stop(ctxt) | (ml_antiq_body(ctxt) | other))))))
   268       }
   268       }
   269     }
   269     }
   270   }
   270   }