src/Pure/Isar/outer_syntax.scala
author wenzelm
Tue Oct 21 15:21:44 2014 +0200 (2014-10-21)
changeset 58748 8f92f17d8781
parent 58747 c680f181b32e
child 58753 960bf499ca5d
permissions -rw-r--r--
support for structure matching;
misc tuning;
     1 /*  Title:      Pure/Isar/outer_syntax.scala
     2     Author:     Makarius
     3 
     4 Isabelle/Isar outer syntax.
     5 */
     6 
     7 package isabelle
     8 
     9 
    10 import scala.util.parsing.input.{Reader, CharSequenceReader}
    11 import scala.collection.mutable
    12 import scala.annotation.tailrec
    13 
    14 
    15 object Outer_Syntax
    16 {
    17   /* syntax */
    18 
    19   val empty: Outer_Syntax = new Outer_Syntax()
    20 
    21   def init(): Outer_Syntax = new Outer_Syntax(completion = Completion.init())
    22 
    23 
    24   /* string literals */
    25 
    26   def quote_string(str: String): String =
    27   {
    28     val result = new StringBuilder(str.length + 10)
    29     result += '"'
    30     for (s <- Symbol.iterator(str)) {
    31       if (s.length == 1) {
    32         val c = s(0)
    33         if (c < 32 && c != YXML.X && c != YXML.Y || c == '\\' || c == '"') {
    34           result += '\\'
    35           if (c < 10) result += '0'
    36           if (c < 100) result += '0'
    37           result ++= (c.asInstanceOf[Int].toString)
    38         }
    39         else result += c
    40       }
    41       else result ++= s
    42     }
    43     result += '"'
    44     result.toString
    45   }
    46 
    47 
    48   /* line-oriented structure */
    49 
    50   object Line_Structure
    51   {
    52     val init = Line_Structure()
    53   }
    54 
    55   sealed case class Line_Structure(
    56     improper: Boolean = true,
    57     command: Boolean = false,
    58     depth: Int = 0,
    59     span_depth: Int = 0,
    60     after_span_depth: Int = 0)
    61 
    62 
    63   /* overall document structure */
    64 
    65   sealed abstract class Document { def length: Int }
    66   case class Document_Block(name: String, text: String, body: List[Document]) extends Document
    67   {
    68     val length: Int = (0 /: body)(_ + _.length)
    69   }
    70   case class Document_Atom(command: Command) extends Document
    71   {
    72     def length: Int = command.length
    73   }
    74 }
    75 
    76 final class Outer_Syntax private(
    77   keywords: Map[String, (String, List[String])] = Map.empty,
    78   lexicon: Scan.Lexicon = Scan.Lexicon.empty,
    79   val completion: Completion = Completion.empty,
    80   val language_context: Completion.Language_Context = Completion.Language_Context.outer,
    81   val has_tokens: Boolean = true) extends Prover.Syntax
    82 {
    83   /** syntax content **/
    84 
    85   override def toString: String =
    86     (for ((name, (kind, files)) <- keywords) yield {
    87       if (kind == Keyword.MINOR) quote(name)
    88       else
    89         quote(name) + " :: " + quote(kind) +
    90         (if (files.isEmpty) "" else " (" + commas_quote(files) + ")")
    91     }).toList.sorted.mkString("keywords\n  ", " and\n  ", "")
    92 
    93 
    94   /* keyword kind */
    95 
    96   def keyword_kind_files(name: String): Option[(String, List[String])] = keywords.get(name)
    97   def keyword_kind(name: String): Option[String] = keyword_kind_files(name).map(_._1)
    98 
    99   def is_command(name: String): Boolean =
   100     keyword_kind(name) match {
   101       case Some(kind) => kind != Keyword.MINOR
   102       case None => false
   103     }
   104 
   105   def command_kind(token: Token, pred: String => Boolean): Boolean =
   106     token.is_command && is_command(token.source) &&
   107       pred(keyword_kind(token.source).get)
   108 
   109 
   110   /* load commands */
   111 
   112   def load_command(name: String): Option[List[String]] =
   113     keywords.get(name) match {
   114       case Some((Keyword.THY_LOAD, exts)) => Some(exts)
   115       case _ => None
   116     }
   117 
   118   val load_commands: List[(String, List[String])] =
   119     (for ((name, (Keyword.THY_LOAD, files)) <- keywords.iterator) yield (name, files)).toList
   120 
   121   def load_commands_in(text: String): Boolean =
   122     load_commands.exists({ case (cmd, _) => text.containsSlice(cmd) })
   123 
   124 
   125   /* add keywords */
   126 
   127   def + (name: String, kind: (String, List[String]), replace: Option[String]): Outer_Syntax =
   128   {
   129     val keywords1 = keywords + (name -> kind)
   130     val lexicon1 = lexicon + name
   131     val completion1 =
   132       if (Keyword.control(kind._1) || replace == Some("")) completion
   133       else completion + (name, replace getOrElse name)
   134     new Outer_Syntax(keywords1, lexicon1, completion1, language_context, true)
   135   }
   136 
   137   def + (name: String, kind: (String, List[String])): Outer_Syntax =
   138     this + (name, kind, Some(name))
   139   def + (name: String, kind: String): Outer_Syntax =
   140     this + (name, (kind, Nil), Some(name))
   141   def + (name: String, replace: Option[String]): Outer_Syntax =
   142     this + (name, (Keyword.MINOR, Nil), replace)
   143   def + (name: String): Outer_Syntax = this + (name, None)
   144 
   145   def add_keywords(keywords: Thy_Header.Keywords): Outer_Syntax =
   146     (this /: keywords) {
   147       case (syntax, (name, Some((kind, _)), replace)) =>
   148         syntax +
   149           (Symbol.decode(name), kind, replace) +
   150           (Symbol.encode(name), kind, replace)
   151       case (syntax, (name, None, replace)) =>
   152         syntax +
   153           (Symbol.decode(name), replace) +
   154           (Symbol.encode(name), replace)
   155     }
   156 
   157 
   158   /* language context */
   159 
   160   def set_language_context(context: Completion.Language_Context): Outer_Syntax =
   161     new Outer_Syntax(keywords, lexicon, completion, context, has_tokens)
   162 
   163   def no_tokens: Outer_Syntax =
   164   {
   165     require(keywords.isEmpty && lexicon.isEmpty)
   166     new Outer_Syntax(
   167       completion = completion,
   168       language_context = language_context,
   169       has_tokens = false)
   170   }
   171 
   172 
   173 
   174   /** parsing **/
   175 
   176   /* line-oriented structure */
   177 
   178   def line_structure(tokens: List[Token], struct: Outer_Syntax.Line_Structure)
   179     : Outer_Syntax.Line_Structure =
   180   {
   181     val improper1 = tokens.forall(_.is_improper)
   182     val command1 = tokens.exists(_.is_command)
   183 
   184     val depth1 =
   185       if (tokens.exists(tok => command_kind(tok, Keyword.theory))) 0
   186       else if (command1) struct.after_span_depth
   187       else struct.span_depth
   188 
   189     val (span_depth1, after_span_depth1) =
   190       ((struct.span_depth, struct.after_span_depth) /: tokens) {
   191         case ((x, y), tok) =>
   192           if (tok.is_command) {
   193             if (command_kind(tok, Keyword.theory_goal)) (2, 1)
   194             else if (command_kind(tok, Keyword.theory)) (1, 0)
   195             else if (command_kind(tok, Keyword.proof_goal) || tok.source == "{") (y + 2, y + 1)
   196             else if (command_kind(tok, Keyword.qed) || tok.source == "}") (y + 1, y - 1)
   197             else if (command_kind(tok, Keyword.qed_global)) (1, 0)
   198             else (x, y)
   199           }
   200           else (x, y)
   201       }
   202 
   203     Outer_Syntax.Line_Structure(improper1, command1, depth1, span_depth1, after_span_depth1)
   204   }
   205 
   206 
   207   /* token language */
   208 
   209   def scan(input: CharSequence): List[Token] =
   210   {
   211     val in: Reader[Char] = new CharSequenceReader(input)
   212     Token.Parsers.parseAll(
   213         Token.Parsers.rep(Token.Parsers.token(lexicon, is_command)), in) match {
   214       case Token.Parsers.Success(tokens, _) => tokens
   215       case _ => error("Unexpected failure of tokenizing input:\n" + input.toString)
   216     }
   217   }
   218 
   219   def scan_line(input: CharSequence, context: Scan.Line_Context): (List[Token], Scan.Line_Context) =
   220   {
   221     var in: Reader[Char] = new CharSequenceReader(input)
   222     val toks = new mutable.ListBuffer[Token]
   223     var ctxt = context
   224     while (!in.atEnd) {
   225       Token.Parsers.parse(Token.Parsers.token_line(lexicon, is_command, ctxt), in) match {
   226         case Token.Parsers.Success((x, c), rest) => { toks += x; ctxt = c; in = rest }
   227         case Token.Parsers.NoSuccess(_, rest) =>
   228           error("Unexpected failure of tokenizing input:\n" + rest.source.toString)
   229       }
   230     }
   231     (toks.toList, ctxt)
   232   }
   233 
   234 
   235   /* command spans */
   236 
   237   def parse_spans(toks: List[Token]): List[Command_Span.Span] =
   238   {
   239     val result = new mutable.ListBuffer[Command_Span.Span]
   240     val content = new mutable.ListBuffer[Token]
   241     val improper = new mutable.ListBuffer[Token]
   242 
   243     def ship(span: List[Token])
   244     {
   245       val kind =
   246         if (!span.isEmpty && span.head.is_command && !span.exists(_.is_error)) {
   247           val name = span.head.source
   248           val pos = Position.Range(Text.Range(0, Symbol.iterator(name).length) + 1)
   249           Command_Span.Command_Span(name, pos)
   250         }
   251         else if (span.forall(_.is_improper)) Command_Span.Ignored_Span
   252         else Command_Span.Malformed_Span
   253       result += Command_Span.Span(kind, span)
   254     }
   255 
   256     def flush()
   257     {
   258       if (!content.isEmpty) { ship(content.toList); content.clear }
   259       if (!improper.isEmpty) { ship(improper.toList); improper.clear }
   260     }
   261 
   262     for (tok <- toks) {
   263       if (tok.is_command) { flush(); content += tok }
   264       else if (tok.is_improper) improper += tok
   265       else { content ++= improper; improper.clear; content += tok }
   266     }
   267     flush()
   268 
   269     result.toList
   270   }
   271 
   272   def parse_spans(input: CharSequence): List[Command_Span.Span] =
   273     parse_spans(scan(input))
   274 
   275 
   276   /* overall document structure */
   277 
   278   def heading_level(command: Command): Option[Int] =
   279   {
   280     keyword_kind(command.name) match {
   281       case _ if command.name == "header" => Some(0)
   282       case Some(Keyword.THY_HEADING1) => Some(1)
   283       case Some(Keyword.THY_HEADING2) | Some(Keyword.PRF_HEADING2) => Some(2)
   284       case Some(Keyword.THY_HEADING3) | Some(Keyword.PRF_HEADING3) => Some(3)
   285       case Some(Keyword.THY_HEADING4) | Some(Keyword.PRF_HEADING4) => Some(4)
   286       case Some(kind) if Keyword.theory(kind) => Some(5)
   287       case _ => None
   288     }
   289   }
   290 
   291   def parse_document(node_name: Document.Node.Name, text: CharSequence):
   292     List[Outer_Syntax.Document] =
   293   {
   294     /* stack operations */
   295 
   296     def buffer(): mutable.ListBuffer[Outer_Syntax.Document] =
   297       new mutable.ListBuffer[Outer_Syntax.Document]
   298 
   299     var stack: List[(Int, Command, mutable.ListBuffer[Outer_Syntax.Document])] =
   300       List((0, Command.empty, buffer()))
   301 
   302     @tailrec def close(level: Int => Boolean)
   303     {
   304       stack match {
   305         case (lev, command, body) :: (_, _, body2) :: rest if level(lev) =>
   306           body2 += Outer_Syntax.Document_Block(command.name, command.source, body.toList)
   307           stack = stack.tail
   308           close(level)
   309         case _ =>
   310       }
   311     }
   312 
   313     def result(): List[Outer_Syntax.Document] =
   314     {
   315       close(_ => true)
   316       stack.head._3.toList
   317     }
   318 
   319     def add(command: Command)
   320     {
   321       heading_level(command) match {
   322         case Some(i) =>
   323           close(_ > i)
   324           stack = (i + 1, command, buffer()) :: stack
   325         case None =>
   326       }
   327       stack.head._3 += Outer_Syntax.Document_Atom(command)
   328     }
   329 
   330 
   331     /* result structure */
   332 
   333     val spans = parse_spans(text)
   334     spans.foreach(span => add(Command(Document_ID.none, node_name, Nil, span)))
   335     result()
   336   }
   337 }