49 val cmt = comment ^^ (x => Token(Token.Kind.COMMENT, x)) |
49 val cmt = comment ^^ (x => Token(Token.Kind.COMMENT, x)) |
50 |
50 |
51 string | (alt_string | (verb | (cart | cmt))) |
51 string | (alt_string | (verb | (cart | cmt))) |
52 } |
52 } |
53 |
53 |
54 private def other_token(minor: Scan.Lexicon, major: Scan.Lexicon): Parser[Token] = |
54 private def other_token(keywords: Keyword.Keywords): Parser[Token] = |
55 { |
55 { |
56 val letdigs1 = many1(Symbol.is_letdig) |
56 val letdigs1 = many1(Symbol.is_letdig) |
57 val sub = one(s => s == Symbol.sub_decoded || s == "\\<^sub>") |
57 val sub = one(s => s == Symbol.sub_decoded || s == "\\<^sub>") |
58 val id = |
58 val id = |
59 one(Symbol.is_letter) ~ |
59 one(Symbol.is_letter) ~ |
78 val sym_ident = |
78 val sym_ident = |
79 (many1(Symbol.is_symbolic_char) | one(sym => Symbol.is_symbolic(sym))) ^^ |
79 (many1(Symbol.is_symbolic_char) | one(sym => Symbol.is_symbolic(sym))) ^^ |
80 (x => Token(Token.Kind.SYM_IDENT, x)) |
80 (x => Token(Token.Kind.SYM_IDENT, x)) |
81 |
81 |
82 val keyword = |
82 val keyword = |
83 literal(minor) ^^ (x => Token(Token.Kind.KEYWORD, x)) ||| |
83 literal(keywords.minor) ^^ (x => Token(Token.Kind.KEYWORD, x)) ||| |
84 literal(major) ^^ (x => Token(Token.Kind.COMMAND, x)) |
84 literal(keywords.major) ^^ (x => Token(Token.Kind.COMMAND, x)) |
85 |
85 |
86 val space = many1(Symbol.is_blank) ^^ (x => Token(Token.Kind.SPACE, x)) |
86 val space = many1(Symbol.is_blank) ^^ (x => Token(Token.Kind.SPACE, x)) |
87 |
87 |
88 val recover_delimited = |
88 val recover_delimited = |
89 (recover_quoted("\"") | |
89 (recover_quoted("\"") | |
96 space | (recover_delimited | |
96 space | (recover_delimited | |
97 (((ident | (var_ | (type_ident | (type_var | (float | (nat_ | sym_ident)))))) ||| |
97 (((ident | (var_ | (type_ident | (type_var | (float | (nat_ | sym_ident)))))) ||| |
98 keyword) | bad)) |
98 keyword) | bad)) |
99 } |
99 } |
100 |
100 |
101 def token(minor: Scan.Lexicon, major: Scan.Lexicon): Parser[Token] = |
101 def token(keywords: Keyword.Keywords): Parser[Token] = |
102 delimited_token | other_token(minor, major) |
102 delimited_token | other_token(keywords) |
103 |
103 |
104 def token_line(minor: Scan.Lexicon, major: Scan.Lexicon, ctxt: Scan.Line_Context) |
104 def token_line(keywords: Keyword.Keywords, ctxt: Scan.Line_Context) |
105 : Parser[(Token, Scan.Line_Context)] = |
105 : Parser[(Token, Scan.Line_Context)] = |
106 { |
106 { |
107 val string = |
107 val string = |
108 quoted_line("\"", ctxt) ^^ { case (x, c) => (Token(Token.Kind.STRING, x), c) } |
108 quoted_line("\"", ctxt) ^^ { case (x, c) => (Token(Token.Kind.STRING, x), c) } |
109 val alt_string = |
109 val alt_string = |
110 quoted_line("`", ctxt) ^^ { case (x, c) => (Token(Token.Kind.ALT_STRING, x), c) } |
110 quoted_line("`", ctxt) ^^ { case (x, c) => (Token(Token.Kind.ALT_STRING, x), c) } |
111 val verb = verbatim_line(ctxt) ^^ { case (x, c) => (Token(Token.Kind.VERBATIM, x), c) } |
111 val verb = verbatim_line(ctxt) ^^ { case (x, c) => (Token(Token.Kind.VERBATIM, x), c) } |
112 val cart = cartouche_line(ctxt) ^^ { case (x, c) => (Token(Token.Kind.CARTOUCHE, x), c) } |
112 val cart = cartouche_line(ctxt) ^^ { case (x, c) => (Token(Token.Kind.CARTOUCHE, x), c) } |
113 val cmt = comment_line(ctxt) ^^ { case (x, c) => (Token(Token.Kind.COMMENT, x), c) } |
113 val cmt = comment_line(ctxt) ^^ { case (x, c) => (Token(Token.Kind.COMMENT, x), c) } |
114 val other = other_token(minor, major) ^^ { case x => (x, Scan.Finished) } |
114 val other = other_token(keywords) ^^ { case x => (x, Scan.Finished) } |
115 |
115 |
116 string | (alt_string | (verb | (cart | (cmt | other)))) |
116 string | (alt_string | (verb | (cart | (cmt | other)))) |
117 } |
117 } |
118 } |
118 } |
119 |
119 |