237 |
237 |
238 /* token */ |
238 /* token */ |
239 |
239 |
240 def token: Parser[Token] = delimited_token | other_token |
240 def token: Parser[Token] = delimited_token | other_token |
241 |
241 |
242 def token_line(ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] = |
242 def token_line(SML: Boolean, ctxt: Scan.Line_Context): Parser[(Token, Scan.Line_Context)] = |
243 { |
243 { |
244 val other = (ml_char | other_token) ^^ (x => (x, Scan.Finished)) |
244 val other = (ml_char | other_token) ^^ (x => (x, Scan.Finished)) |
245 |
245 |
246 ml_string_line(ctxt) | |
246 if (SML) ml_string_line(ctxt) | (ml_comment_line(ctxt) | other) |
247 (ml_comment_line(ctxt) | |
247 else |
248 (ml_antiq_start(ctxt) | (ml_antiq_stop(ctxt) | (ml_antiq_body(ctxt) | other)))) |
248 ml_string_line(ctxt) | |
|
249 (ml_comment_line(ctxt) | |
|
250 (ml_antiq_start(ctxt) | (ml_antiq_stop(ctxt) | (ml_antiq_body(ctxt) | other)))) |
249 } |
251 } |
250 } |
252 } |
251 |
253 |
252 |
254 |
253 /* tokenize */ |
255 /* tokenize */ |
258 case Parsers.Success(tokens, _) => tokens |
260 case Parsers.Success(tokens, _) => tokens |
259 case _ => error("Unexpected failure of tokenizing input:\n" + input.toString) |
261 case _ => error("Unexpected failure of tokenizing input:\n" + input.toString) |
260 } |
262 } |
261 } |
263 } |
262 |
264 |
263 def tokenize_line(input: CharSequence, context: Scan.Line_Context) |
265 def tokenize_line(SML: Boolean, input: CharSequence, context: Scan.Line_Context) |
264 : (List[Token], Scan.Line_Context) = |
266 : (List[Token], Scan.Line_Context) = |
265 { |
267 { |
266 var in: Reader[Char] = new CharSequenceReader(input) |
268 var in: Reader[Char] = new CharSequenceReader(input) |
267 val toks = new mutable.ListBuffer[Token] |
269 val toks = new mutable.ListBuffer[Token] |
268 var ctxt = context |
270 var ctxt = context |
269 while (!in.atEnd) { |
271 while (!in.atEnd) { |
270 Parsers.parse(Parsers.token_line(ctxt), in) match { |
272 Parsers.parse(Parsers.token_line(SML, ctxt), in) match { |
271 case Parsers.Success((x, c), rest) => { toks += x; ctxt = c; in = rest } |
273 case Parsers.Success((x, c), rest) => { toks += x; ctxt = c; in = rest } |
272 case Parsers.NoSuccess(_, rest) => |
274 case Parsers.NoSuccess(_, rest) => |
273 error("Unexpected failure of tokenizing input:\n" + rest.source.toString) |
275 error("Unexpected failure of tokenizing input:\n" + rest.source.toString) |
274 } |
276 } |
275 } |
277 } |