48 val INT = Value("integer") |
48 val INT = Value("integer") |
49 val REAL = Value("real") |
49 val REAL = Value("real") |
50 val CHAR = Value("character") |
50 val CHAR = Value("character") |
51 val STRING = Value("quoted string") |
51 val STRING = Value("quoted string") |
52 val SPACE = Value("white space") |
52 val SPACE = Value("white space") |
53 val COMMENT = Value("comment text") |
53 val INFORMAL_COMMENT = Value("informal comment") |
54 val COMMENT_CARTOUCHE = Value("comment cartouche") |
54 val FORMAL_COMMENT = Value("formal comment") |
55 val CONTROL = Value("control symbol antiquotation") |
55 val CONTROL = Value("control symbol antiquotation") |
56 val ANTIQ = Value("antiquotation") |
56 val ANTIQ = Value("antiquotation") |
57 val ANTIQ_START = Value("antiquotation: start") |
57 val ANTIQ_START = Value("antiquotation: start") |
58 val ANTIQ_STOP = Value("antiquotation: stop") |
58 val ANTIQ_STOP = Value("antiquotation: stop") |
59 val ANTIQ_OTHER = Value("antiquotation: other") |
59 val ANTIQ_OTHER = Value("antiquotation: other") |
66 sealed case class Token(kind: Kind.Value, source: String) |
66 sealed case class Token(kind: Kind.Value, source: String) |
67 { |
67 { |
68 def is_keyword: Boolean = kind == Kind.KEYWORD |
68 def is_keyword: Boolean = kind == Kind.KEYWORD |
69 def is_delimiter: Boolean = is_keyword && !Symbol.is_ascii_identifier(source) |
69 def is_delimiter: Boolean = is_keyword && !Symbol.is_ascii_identifier(source) |
70 def is_space: Boolean = kind == Kind.SPACE |
70 def is_space: Boolean = kind == Kind.SPACE |
71 def is_comment: Boolean = kind == Kind.COMMENT || kind == Kind.COMMENT_CARTOUCHE |
71 def is_comment: Boolean = kind == Kind.INFORMAL_COMMENT || kind == Kind.FORMAL_COMMENT |
72 } |
72 } |
73 |
73 |
74 |
74 |
75 |
75 |
76 /** parsers **/ |
76 /** parsers **/ |
77 |
77 |
78 case object ML_String extends Scan.Line_Context |
78 case object ML_String extends Scan.Line_Context |
79 case class Antiq(ctxt: Scan.Line_Context) extends Scan.Line_Context |
79 case class Antiq(ctxt: Scan.Line_Context) extends Scan.Line_Context |
80 |
80 |
81 private object Parsers extends Scan.Parsers with Antiquote.Parsers |
81 private object Parsers extends Scan.Parsers with Antiquote.Parsers with Comment.Parsers |
82 { |
82 { |
83 /* string material */ |
83 /* string material */ |
84 |
84 |
85 private val blanks = many(character(Symbol.is_ascii_blank)) |
85 private val blanks = many(character(Symbol.is_ascii_blank)) |
86 private val blanks1 = many1(character(Symbol.is_ascii_blank)) |
86 private val blanks1 = many1(character(Symbol.is_ascii_blank)) |
138 |
138 |
139 |
139 |
140 /* ML comment */ |
140 /* ML comment */ |
141 |
141 |
142 private val ml_comment: Parser[Token] = |
142 private val ml_comment: Parser[Token] = |
143 comment ^^ (x => Token(Kind.COMMENT, x)) |
143 comment ^^ (x => Token(Kind.INFORMAL_COMMENT, x)) |
144 |
144 |
145 private def ml_comment_line(ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] = |
145 private def ml_comment_line(ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] = |
146 comment_line(ctxt) ^^ { case (x, c) => (Token(Kind.COMMENT, x), c) } |
146 comment_line(ctxt) ^^ { case (x, c) => (Token(Kind.INFORMAL_COMMENT, x), c) } |
147 |
147 |
148 private val ml_comment_cartouche: Parser[Token] = |
148 private val ml_formal_comment: Parser[Token] = |
149 comment_cartouche ^^ (x => Token(Kind.COMMENT_CARTOUCHE, x)) |
149 comment_cartouche ^^ (x => Token(Kind.FORMAL_COMMENT, x)) |
150 |
150 |
151 private def ml_comment_cartouche_line(ctxt: Scan.Line_Context) |
151 private def ml_formal_comment_line(ctxt: Scan.Line_Context) |
152 : Parser[(Token, Scan.Line_Context)] = |
152 : Parser[(Token, Scan.Line_Context)] = |
153 comment_cartouche_line(ctxt) ^^ { case (x, c) => (Token(Kind.COMMENT_CARTOUCHE, x), c) } |
153 comment_cartouche_line(ctxt) ^^ { case (x, c) => (Token(Kind.FORMAL_COMMENT, x), c) } |
154 |
154 |
155 |
155 |
156 /* antiquotations (line-oriented) */ |
156 /* antiquotations (line-oriented) */ |
157 |
157 |
158 def ml_cartouche_line(ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] = |
158 def ml_cartouche_line(ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] = |
250 space | (ml_control | (recover | (ml_antiq | |
250 space | (ml_control | (recover | (ml_antiq | |
251 (((word | (real | (int | (long_ident | (ident | type_var))))) ||| keyword) | bad)))) |
251 (((word | (real | (int | (long_ident | (ident | type_var))))) ||| keyword) | bad)))) |
252 } |
252 } |
253 |
253 |
254 def token: Parser[Token] = |
254 def token: Parser[Token] = |
255 ml_char | (ml_string | (ml_comment | (ml_comment_cartouche | other_token))) |
255 ml_char | (ml_string | (ml_comment | (ml_formal_comment | other_token))) |
256 |
256 |
257 def token_line(SML: Boolean, ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] = |
257 def token_line(SML: Boolean, ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] = |
258 { |
258 { |
259 val other = (ml_char | other_token) ^^ (x => (x, Scan.Finished)) |
259 val other = (ml_char | other_token) ^^ (x => (x, Scan.Finished)) |
260 |
260 |
261 if (SML) ml_string_line(ctxt) | (ml_comment_line(ctxt) | other) |
261 if (SML) ml_string_line(ctxt) | (ml_comment_line(ctxt) | other) |
262 else { |
262 else { |
263 ml_string_line(ctxt) | |
263 ml_string_line(ctxt) | |
264 (ml_comment_line(ctxt) | |
264 (ml_comment_line(ctxt) | |
265 (ml_comment_cartouche_line(ctxt) | |
265 (ml_formal_comment_line(ctxt) | |
266 (ml_cartouche_line(ctxt) | |
266 (ml_cartouche_line(ctxt) | |
267 (ml_antiq_start(ctxt) | (ml_antiq_stop(ctxt) | (ml_antiq_body(ctxt) | other)))))) |
267 (ml_antiq_start(ctxt) | (ml_antiq_stop(ctxt) | (ml_antiq_body(ctxt) | other)))))) |
268 } |
268 } |
269 } |
269 } |
270 } |
270 } |