equal
deleted
inserted
replaced
122 heading_level(command.name) |
122 heading_level(command.name) |
123 |
123 |
124 |
124 |
125 /* token language */ |
125 /* token language */ |
126 |
126 |
127 def scan(input: Reader[Char]): List[Token] = |
127 def scan(input: CharSequence): List[Token] = |
128 { |
128 { |
|
129 var in: Reader[Char] = new CharSequenceReader(input) |
129 Token.Parsers.parseAll( |
130 Token.Parsers.parseAll( |
130 Token.Parsers.rep(Token.Parsers.token(lexicon, is_command)), input) match { |
131 Token.Parsers.rep(Token.Parsers.token(lexicon, is_command)), in) match { |
131 case Token.Parsers.Success(tokens, _) => tokens |
132 case Token.Parsers.Success(tokens, _) => tokens |
132 case _ => error("Unexpected failure of tokenizing input:\n" + input.source.toString) |
133 case _ => error("Unexpected failure of tokenizing input:\n" + input.toString) |
133 } |
134 } |
134 } |
135 } |
135 |
|
136 def scan(input: CharSequence): List[Token] = |
|
137 scan(new CharSequenceReader(input)) |
|
138 |
136 |
139 def scan_line(input: CharSequence, context: Scan.Line_Context): (List[Token], Scan.Line_Context) = |
137 def scan_line(input: CharSequence, context: Scan.Line_Context): (List[Token], Scan.Line_Context) = |
140 { |
138 { |
141 var in: Reader[Char] = new CharSequenceReader(input) |
139 var in: Reader[Char] = new CharSequenceReader(input) |
142 val toks = new mutable.ListBuffer[Token] |
140 val toks = new mutable.ListBuffer[Token] |