src/Pure/Isar/outer_syntax.scala
author wenzelm
Mon Dec 01 15:21:49 2014 +0100 (2014-12-01)
changeset 59073 dcecfcc56dce
parent 58938 0c45680b7d9d
child 59077 7e0d3da6e6d8
permissions -rw-r--r--
more merge operations;
     1 /*  Title:      Pure/Isar/outer_syntax.scala
     2     Author:     Makarius
     3 
     4 Isabelle/Isar outer syntax.
     5 */
     6 
     7 package isabelle
     8 
     9 
    10 import scala.util.parsing.input.{Reader, CharSequenceReader}
    11 import scala.collection.mutable
    12 import scala.annotation.tailrec
    13 
    14 
    15 object Outer_Syntax
    16 {
    17   /* syntax */
    18 
    19   val empty: Outer_Syntax = new Outer_Syntax()
    20 
    21   def init(): Outer_Syntax = new Outer_Syntax(completion = Completion.init())
    22 
    23 
    24   /* string literals */
    25 
    26   def quote_string(str: String): String =
    27   {
    28     val result = new StringBuilder(str.length + 10)
    29     result += '"'
    30     for (s <- Symbol.iterator(str)) {
    31       if (s.length == 1) {
    32         val c = s(0)
    33         if (c < 32 && c != YXML.X && c != YXML.Y || c == '\\' || c == '"') {
    34           result += '\\'
    35           if (c < 10) result += '0'
    36           if (c < 100) result += '0'
    37           result ++= (c.asInstanceOf[Int].toString)
    38         }
    39         else result += c
    40       }
    41       else result ++= s
    42     }
    43     result += '"'
    44     result.toString
    45   }
    46 
    47 
    48   /* line-oriented structure */
    49 
    50   object Line_Structure
    51   {
    52     val init = Line_Structure()
    53   }
    54 
    55   sealed case class Line_Structure(
    56     improper: Boolean = true,
    57     command: Boolean = false,
    58     depth: Int = 0,
    59     span_depth: Int = 0,
    60     after_span_depth: Int = 0)
    61 
    62 
    63   /* overall document structure */
    64 
    65   sealed abstract class Document { def length: Int }
    66   case class Document_Block(name: String, text: String, body: List[Document]) extends Document
    67   {
    68     val length: Int = (0 /: body)(_ + _.length)
    69   }
    70   case class Document_Atom(command: Command) extends Document
    71   {
    72     def length: Int = command.length
    73   }
    74 }
    75 
    76 final class Outer_Syntax private(
    77   val keywords: Keyword.Keywords = Keyword.Keywords.empty,
    78   val completion: Completion = Completion.empty,
    79   val language_context: Completion.Language_Context = Completion.Language_Context.outer,
    80   val has_tokens: Boolean = true) extends Prover.Syntax
    81 {
    82   /** syntax content **/
    83 
    84   override def toString: String = keywords.toString
    85 
    86 
    87   /* add keywords */
    88 
    89   def + (name: String): Outer_Syntax = this + (name, None, None)
    90   def + (name: String, kind: String): Outer_Syntax = this + (name, Some((kind, Nil)), None)
    91   def + (name: String, opt_kind: Option[(String, List[String])], replace: Option[String])
    92     : Outer_Syntax =
    93   {
    94     val keywords1 =
    95       opt_kind match {
    96         case None => keywords + name
    97         case Some(kind) => keywords + (name, kind)
    98       }
    99     val completion1 =
   100       if (replace == Some("")) completion
   101       else completion + (name, replace getOrElse name)
   102     new Outer_Syntax(keywords1, completion1, language_context, true)
   103   }
   104 
   105   def add_keywords(keywords: Thy_Header.Keywords): Outer_Syntax =
   106     (this /: keywords) {
   107       case (syntax, (name, opt_spec, replace)) =>
   108         val opt_kind = opt_spec.map(_._1)
   109         syntax +
   110           (Symbol.decode(name), opt_kind, replace) +
   111           (Symbol.encode(name), opt_kind, replace)
   112     }
   113 
   114 
   115   /* merge */
   116 
   117   def ++ (other: Outer_Syntax): Outer_Syntax =
   118     if (this eq other) this
   119     else {
   120       val keywords1 = keywords ++ other.keywords
   121       val completion1 = completion ++ other.completion
   122       new Outer_Syntax(keywords1, completion1, language_context, has_tokens)
   123     }
   124 
   125 
   126   /* load commands */
   127 
   128   def load_command(name: String): Option[List[String]] = keywords.load_command(name)
   129   def load_commands_in(text: String): Boolean = keywords.load_commands_in(text)
   130 
   131 
   132   /* language context */
   133 
   134   def set_language_context(context: Completion.Language_Context): Outer_Syntax =
   135     new Outer_Syntax(keywords, completion, context, has_tokens)
   136 
   137   def no_tokens: Outer_Syntax =
   138   {
   139     require(keywords.is_empty)
   140     new Outer_Syntax(
   141       completion = completion,
   142       language_context = language_context,
   143       has_tokens = false)
   144   }
   145 
   146 
   147 
   148   /** parsing **/
   149 
   150   /* line-oriented structure */
   151 
   152   def line_structure(tokens: List[Token], struct: Outer_Syntax.Line_Structure)
   153     : Outer_Syntax.Line_Structure =
   154   {
   155     val improper1 = tokens.forall(_.is_improper)
   156     val command1 = tokens.exists(_.is_command)
   157 
   158     val depth1 =
   159       if (tokens.exists(tok => keywords.is_command_kind(tok, Keyword.theory))) 0
   160       else if (command1) struct.after_span_depth
   161       else struct.span_depth
   162 
   163     val (span_depth1, after_span_depth1) =
   164       ((struct.span_depth, struct.after_span_depth) /: tokens) {
   165         case ((x, y), tok) =>
   166           if (tok.is_command) {
   167             if (keywords.is_command_kind(tok, Keyword.theory_goal))
   168               (2, 1)
   169             else if (keywords.is_command_kind(tok, Keyword.theory))
   170               (1, 0)
   171             else if (keywords.is_command_kind(tok, Keyword.proof_goal) || tok.is_begin_block)
   172               (y + 2, y + 1)
   173             else if (keywords.is_command_kind(tok, Keyword.qed) || tok.is_end_block)
   174               (y + 1, y - 1)
   175             else if (keywords.is_command_kind(tok, Keyword.qed_global))
   176               (1, 0)
   177             else (x, y)
   178           }
   179           else (x, y)
   180       }
   181 
   182     Outer_Syntax.Line_Structure(improper1, command1, depth1, span_depth1, after_span_depth1)
   183   }
   184 
   185 
   186   /* token language */
   187 
   188   def scan(input: CharSequence): List[Token] =
   189   {
   190     val in: Reader[Char] = new CharSequenceReader(input)
   191     Token.Parsers.parseAll(Token.Parsers.rep(Token.Parsers.token(keywords)), in) match {
   192       case Token.Parsers.Success(tokens, _) => tokens
   193       case _ => error("Unexpected failure of tokenizing input:\n" + input.toString)
   194     }
   195   }
   196 
   197   def scan_line(input: CharSequence, context: Scan.Line_Context): (List[Token], Scan.Line_Context) =
   198   {
   199     var in: Reader[Char] = new CharSequenceReader(input)
   200     val toks = new mutable.ListBuffer[Token]
   201     var ctxt = context
   202     while (!in.atEnd) {
   203       Token.Parsers.parse(Token.Parsers.token_line(keywords, ctxt), in) match {
   204         case Token.Parsers.Success((x, c), rest) => { toks += x; ctxt = c; in = rest }
   205         case Token.Parsers.NoSuccess(_, rest) =>
   206           error("Unexpected failure of tokenizing input:\n" + rest.source.toString)
   207       }
   208     }
   209     (toks.toList, ctxt)
   210   }
   211 
   212 
   213   /* command spans */
   214 
   215   def parse_spans(toks: List[Token]): List[Command_Span.Span] =
   216   {
   217     val result = new mutable.ListBuffer[Command_Span.Span]
   218     val content = new mutable.ListBuffer[Token]
   219     val improper = new mutable.ListBuffer[Token]
   220 
   221     def ship(span: List[Token])
   222     {
   223       val kind =
   224         if (!span.isEmpty && span.head.is_command && !span.exists(_.is_error)) {
   225           val name = span.head.source
   226           val pos = Position.Range(Text.Range(0, Symbol.iterator(name).length) + 1)
   227           Command_Span.Command_Span(name, pos)
   228         }
   229         else if (span.forall(_.is_improper)) Command_Span.Ignored_Span
   230         else Command_Span.Malformed_Span
   231       result += Command_Span.Span(kind, span)
   232     }
   233 
   234     def flush()
   235     {
   236       if (!content.isEmpty) { ship(content.toList); content.clear }
   237       if (!improper.isEmpty) { ship(improper.toList); improper.clear }
   238     }
   239 
   240     for (tok <- toks) {
   241       if (tok.is_command) { flush(); content += tok }
   242       else if (tok.is_improper) improper += tok
   243       else { content ++= improper; improper.clear; content += tok }
   244     }
   245     flush()
   246 
   247     result.toList
   248   }
   249 
   250   def parse_spans(input: CharSequence): List[Command_Span.Span] =
   251     parse_spans(scan(input))
   252 
   253 
   254   /* overall document structure */
   255 
   256   def heading_level(command: Command): Option[Int] =
   257   {
   258     command.name match {
   259       case "chapter" => Some(0)
   260       case "section" | "header" => Some(1)
   261       case "subsection" => Some(2)
   262       case "subsubsection" => Some(3)
   263       case _ =>
   264         keywords.command_kind(command.name) match {
   265           case Some(kind) if Keyword.theory(kind) && kind != Keyword.THY_END => Some(4)
   266           case _ => None
   267         }
   268     }
   269   }
   270 
   271   def parse_document(node_name: Document.Node.Name, text: CharSequence):
   272     List[Outer_Syntax.Document] =
   273   {
   274     /* stack operations */
   275 
   276     def buffer(): mutable.ListBuffer[Outer_Syntax.Document] =
   277       new mutable.ListBuffer[Outer_Syntax.Document]
   278 
   279     var stack: List[(Int, Command, mutable.ListBuffer[Outer_Syntax.Document])] =
   280       List((0, Command.empty, buffer()))
   281 
   282     @tailrec def close(level: Int => Boolean)
   283     {
   284       stack match {
   285         case (lev, command, body) :: (_, _, body2) :: rest if level(lev) =>
   286           body2 += Outer_Syntax.Document_Block(command.name, command.source, body.toList)
   287           stack = stack.tail
   288           close(level)
   289         case _ =>
   290       }
   291     }
   292 
   293     def result(): List[Outer_Syntax.Document] =
   294     {
   295       close(_ => true)
   296       stack.head._3.toList
   297     }
   298 
   299     def add(command: Command)
   300     {
   301       heading_level(command) match {
   302         case Some(i) =>
   303           close(_ > i)
   304           stack = (i + 1, command, buffer()) :: stack
   305         case None =>
   306       }
   307       stack.head._3 += Outer_Syntax.Document_Atom(command)
   308     }
   309 
   310 
   311     /* result structure */
   312 
   313     val spans = parse_spans(text)
   314     spans.foreach(span => add(Command(Document_ID.none, node_name, Nil, span)))
   315     result()
   316   }
   317 }