src/Pure/General/scan.scala
changeset 34316 f879b649ac4c
parent 34301 78c10aea025d
child 36011 3ff725ac13a4
equal deleted inserted replaced
34315:c47a2228fead 34316:f879b649ac4c
   177     /* quoted strings */
   177     /* quoted strings */
   178 
   178 
   179     private def quoted(quote: String): Parser[String] =
   179     private def quoted(quote: String): Parser[String] =
   180     {
   180     {
   181       quote ~
   181       quote ~
   182         rep(many1(sym => sym != quote && sym != "\\" && Symbol.is_closed(sym)) |
   182         rep(many1(sym => sym != quote && sym != "\\" && Symbol.is_wellformed(sym)) |
   183           "\\" + quote | "\\\\" |
   183           "\\" + quote | "\\\\" |
   184           (("""\\\d\d\d""".r) ^? { case x if x.substring(1, 4).toInt <= 255 => x })) ~
   184           (("""\\\d\d\d""".r) ^? { case x if x.substring(1, 4).toInt <= 255 => x })) ~
   185       quote ^^ { case x ~ ys ~ z => x + ys.mkString + z }
   185       quote ^^ { case x ~ ys ~ z => x + ys.mkString + z }
   186     }.named("quoted")
   186     }.named("quoted")
   187 
   187 
   189     {
   189     {
   190       require(parseAll(quoted(quote), source).successful)
   190       require(parseAll(quoted(quote), source).successful)
   191       val body = source.substring(1, source.length - 1)
   191       val body = source.substring(1, source.length - 1)
   192       if (body.exists(_ == '\\')) {
   192       if (body.exists(_ == '\\')) {
   193         val content =
   193         val content =
   194           rep(many1(sym => sym != quote && sym != "\\" && Symbol.is_closed(sym)) |
   194           rep(many1(sym => sym != quote && sym != "\\" && Symbol.is_wellformed(sym)) |
   195               "\\" ~> (quote | "\\" | """\d\d\d""".r ^^ { case x => x.toInt.toChar.toString }))
   195               "\\" ~> (quote | "\\" | """\d\d\d""".r ^^ { case x => x.toInt.toChar.toString }))
   196         parseAll(content ^^ (_.mkString), body).get
   196         parseAll(content ^^ (_.mkString), body).get
   197       }
   197       }
   198       else body
   198       else body
   199     }
   199     }
   201 
   201 
   202     /* verbatim text */
   202     /* verbatim text */
   203 
   203 
   204     private def verbatim: Parser[String] =
   204     private def verbatim: Parser[String] =
   205     {
   205     {
   206       "{*" ~ rep(many1(sym => sym != "*" && Symbol.is_closed(sym)) | """\*(?!\})""".r) ~ "*}" ^^
   206       "{*" ~ rep(many1(sym => sym != "*" && Symbol.is_wellformed(sym)) | """\*(?!\})""".r) ~ "*}" ^^
   207         { case x ~ ys ~ z => x + ys.mkString + z }
   207         { case x ~ ys ~ z => x + ys.mkString + z }
   208     }.named("verbatim")
   208     }.named("verbatim")
   209 
   209 
   210     def verbatim_content(source: String): String =
   210     def verbatim_content(source: String): String =
   211     {
   211     {
   217     /* nested comments */
   217     /* nested comments */
   218 
   218 
   219     def comment: Parser[String] = new Parser[String]
   219     def comment: Parser[String] = new Parser[String]
   220     {
   220     {
   221       val comment_text =
   221       val comment_text =
   222         rep(many1(sym => sym != "*" && sym != "(" && Symbol.is_closed(sym)) |
   222         rep(many1(sym => sym != "*" && sym != "(" && Symbol.is_wellformed(sym)) |
   223           """\*(?!\))|\((?!\*)""".r)
   223           """\*(?!\))|\((?!\*)""".r)
   224       val comment_open = "(*" ~ comment_text ^^^ ()
   224       val comment_open = "(*" ~ comment_text ^^^ ()
   225       val comment_close = comment_text ~ "*)" ^^^ ()
   225       val comment_close = comment_text ~ "*)" ^^^ ()
   226 
   226 
   227       def apply(in: Input) =
   227       def apply(in: Input) =
   274       val type_var = "?'" ~ id_nat ^^ { case x ~ y => Token(Token_Kind.TYPE_VAR, x + y) }
   274       val type_var = "?'" ~ id_nat ^^ { case x ~ y => Token(Token_Kind.TYPE_VAR, x + y) }
   275       val nat_ = nat ^^ (x => Token(Token_Kind.NAT, x))
   275       val nat_ = nat ^^ (x => Token(Token_Kind.NAT, x))
   276 
   276 
   277       val sym_ident =
   277       val sym_ident =
   278         (many1(symbols.is_symbolic_char) |
   278         (many1(symbols.is_symbolic_char) |
   279           one(sym => symbols.is_symbolic(sym) & Symbol.is_closed(sym))) ^^
   279           one(sym => symbols.is_symbolic(sym) & Symbol.is_wellformed(sym))) ^^
   280         (x => Token(Token_Kind.SYM_IDENT, x))
   280         (x => Token(Token_Kind.SYM_IDENT, x))
   281 
   281 
   282       val space = many1(symbols.is_blank) ^^ (x => Token(Token_Kind.SPACE, x))
   282       val space = many1(symbols.is_blank) ^^ (x => Token(Token_Kind.SPACE, x))
   283 
   283 
   284       val string = quoted("\"") ^^ (x => Token(Token_Kind.STRING, x))
   284       val string = quoted("\"") ^^ (x => Token(Token_Kind.STRING, x))