src/Pure/Isar/token.scala
changeset 58900 1435cc20b022
parent 58899 0a793c580685
child 59081 2ceb05ee0331
equal deleted inserted replaced
58899:0a793c580685 58900:1435cc20b022
    49       val cmt = comment ^^ (x => Token(Token.Kind.COMMENT, x))
    49       val cmt = comment ^^ (x => Token(Token.Kind.COMMENT, x))
    50 
    50 
    51       string | (alt_string | (verb | (cart | cmt)))
    51       string | (alt_string | (verb | (cart | cmt)))
    52     }
    52     }
    53 
    53 
    54     private def other_token(minor: Scan.Lexicon, major: Scan.Lexicon): Parser[Token] =
    54     private def other_token(keywords: Keyword.Keywords): Parser[Token] =
    55     {
    55     {
    56       val letdigs1 = many1(Symbol.is_letdig)
    56       val letdigs1 = many1(Symbol.is_letdig)
    57       val sub = one(s => s == Symbol.sub_decoded || s == "\\<^sub>")
    57       val sub = one(s => s == Symbol.sub_decoded || s == "\\<^sub>")
    58       val id =
    58       val id =
    59         one(Symbol.is_letter) ~
    59         one(Symbol.is_letter) ~
    78       val sym_ident =
    78       val sym_ident =
    79         (many1(Symbol.is_symbolic_char) | one(sym => Symbol.is_symbolic(sym))) ^^
    79         (many1(Symbol.is_symbolic_char) | one(sym => Symbol.is_symbolic(sym))) ^^
    80         (x => Token(Token.Kind.SYM_IDENT, x))
    80         (x => Token(Token.Kind.SYM_IDENT, x))
    81 
    81 
    82       val keyword =
    82       val keyword =
    83         literal(minor) ^^ (x => Token(Token.Kind.KEYWORD, x)) |||
    83         literal(keywords.minor) ^^ (x => Token(Token.Kind.KEYWORD, x)) |||
    84         literal(major) ^^ (x => Token(Token.Kind.COMMAND, x))
    84         literal(keywords.major) ^^ (x => Token(Token.Kind.COMMAND, x))
    85 
    85 
    86       val space = many1(Symbol.is_blank) ^^ (x => Token(Token.Kind.SPACE, x))
    86       val space = many1(Symbol.is_blank) ^^ (x => Token(Token.Kind.SPACE, x))
    87 
    87 
    88       val recover_delimited =
    88       val recover_delimited =
    89         (recover_quoted("\"") |
    89         (recover_quoted("\"") |
    96       space | (recover_delimited |
    96       space | (recover_delimited |
    97         (((ident | (var_ | (type_ident | (type_var | (float | (nat_ | sym_ident)))))) |||
    97         (((ident | (var_ | (type_ident | (type_var | (float | (nat_ | sym_ident)))))) |||
    98           keyword) | bad))
    98           keyword) | bad))
    99     }
    99     }
   100 
   100 
   101     def token(minor: Scan.Lexicon, major: Scan.Lexicon): Parser[Token] =
   101     def token(keywords: Keyword.Keywords): Parser[Token] =
   102       delimited_token | other_token(minor, major)
   102       delimited_token | other_token(keywords)
   103 
   103 
   104     def token_line(minor: Scan.Lexicon, major: Scan.Lexicon, ctxt: Scan.Line_Context)
   104     def token_line(keywords: Keyword.Keywords, ctxt: Scan.Line_Context)
   105       : Parser[(Token, Scan.Line_Context)] =
   105       : Parser[(Token, Scan.Line_Context)] =
   106     {
   106     {
   107       val string =
   107       val string =
   108         quoted_line("\"", ctxt) ^^ { case (x, c) => (Token(Token.Kind.STRING, x), c) }
   108         quoted_line("\"", ctxt) ^^ { case (x, c) => (Token(Token.Kind.STRING, x), c) }
   109       val alt_string =
   109       val alt_string =
   110         quoted_line("`", ctxt) ^^ { case (x, c) => (Token(Token.Kind.ALT_STRING, x), c) }
   110         quoted_line("`", ctxt) ^^ { case (x, c) => (Token(Token.Kind.ALT_STRING, x), c) }
   111       val verb = verbatim_line(ctxt) ^^ { case (x, c) => (Token(Token.Kind.VERBATIM, x), c) }
   111       val verb = verbatim_line(ctxt) ^^ { case (x, c) => (Token(Token.Kind.VERBATIM, x), c) }
   112       val cart = cartouche_line(ctxt) ^^ { case (x, c) => (Token(Token.Kind.CARTOUCHE, x), c) }
   112       val cart = cartouche_line(ctxt) ^^ { case (x, c) => (Token(Token.Kind.CARTOUCHE, x), c) }
   113       val cmt = comment_line(ctxt) ^^ { case (x, c) => (Token(Token.Kind.COMMENT, x), c) }
   113       val cmt = comment_line(ctxt) ^^ { case (x, c) => (Token(Token.Kind.COMMENT, x), c) }
   114       val other = other_token(minor, major) ^^ { case x => (x, Scan.Finished) }
   114       val other = other_token(keywords) ^^ { case x => (x, Scan.Finished) }
   115 
   115 
   116       string | (alt_string | (verb | (cart | (cmt | other))))
   116       string | (alt_string | (verb | (cart | (cmt | other))))
   117     }
   117     }
   118   }
   118   }
   119 
   119