src/Pure/Isar/outer_syntax.scala
author wenzelm
Wed Aug 03 11:45:09 2016 +0200 (2016-08-03)
changeset 63592 64db21931bcb
parent 63587 881e8e2cfec2
child 63603 9d9ea2c6bc38
permissions -rw-r--r--
include 'begin' and 'end' structure in text folds;
wenzelm@34166
     1
/*  Title:      Pure/Isar/outer_syntax.scala
wenzelm@34166
     2
    Author:     Makarius
wenzelm@34166
     3
wenzelm@34166
     4
Isabelle/Isar outer syntax.
wenzelm@34166
     5
*/
wenzelm@34166
     6
wenzelm@34166
     7
package isabelle
wenzelm@34166
     8
wenzelm@34166
     9
wenzelm@43411
    10
import scala.collection.mutable
wenzelm@58706
    11
import scala.annotation.tailrec
wenzelm@34166
    12
wenzelm@34166
    13
wenzelm@43774
    14
object Outer_Syntax
wenzelm@43774
    15
{
wenzelm@58706
    16
  /* syntax */
wenzelm@58706
    17
wenzelm@58706
    18
  val empty: Outer_Syntax = new Outer_Syntax()
wenzelm@58706
    19
wenzelm@58706
    20
  def init(): Outer_Syntax = new Outer_Syntax(completion = Completion.init())
wenzelm@58706
    21
wenzelm@58706
    22
wenzelm@58706
    23
  /* string literals */
wenzelm@58706
    24
wenzelm@43774
    25
  def quote_string(str: String): String =
wenzelm@43774
    26
  {
wenzelm@43774
    27
    val result = new StringBuilder(str.length + 10)
wenzelm@43774
    28
    result += '"'
wenzelm@43774
    29
    for (s <- Symbol.iterator(str)) {
wenzelm@43774
    30
      if (s.length == 1) {
wenzelm@43774
    31
        val c = s(0)
wenzelm@43774
    32
        if (c < 32 && c != YXML.X && c != YXML.Y || c == '\\' || c == '"') {
wenzelm@43774
    33
          result += '\\'
wenzelm@43774
    34
          if (c < 10) result += '0'
wenzelm@43774
    35
          if (c < 100) result += '0'
wenzelm@60215
    36
          result ++= c.asInstanceOf[Int].toString
wenzelm@43774
    37
        }
wenzelm@43774
    38
        else result += c
wenzelm@43774
    39
      }
wenzelm@43774
    40
      else result ++= s
wenzelm@43774
    41
    }
wenzelm@43774
    42
    result += '"'
wenzelm@43774
    43
    result.toString
wenzelm@43774
    44
  }
wenzelm@46626
    45
wenzelm@58696
    46
wenzelm@58697
    47
  /* line-oriented structure */
wenzelm@58696
    48
wenzelm@58697
    49
  object Line_Structure
wenzelm@58696
    50
  {
wenzelm@58700
    51
    val init = Line_Structure()
wenzelm@58696
    52
  }
wenzelm@58696
    53
wenzelm@58700
    54
  sealed case class Line_Structure(
wenzelm@58700
    55
    improper: Boolean = true,
wenzelm@58700
    56
    command: Boolean = false,
wenzelm@58700
    57
    depth: Int = 0,
wenzelm@58700
    58
    span_depth: Int = 0,
wenzelm@63592
    59
    after_span_depth: Int = 0,
wenzelm@63592
    60
    element_depth: Int = 0)
wenzelm@58706
    61
wenzelm@58706
    62
wenzelm@58706
    63
  /* overall document structure */
wenzelm@58706
    64
wenzelm@58706
    65
  sealed abstract class Document { def length: Int }
wenzelm@58747
    66
  case class Document_Block(name: String, text: String, body: List[Document]) extends Document
wenzelm@58706
    67
  {
wenzelm@58706
    68
    val length: Int = (0 /: body)(_ + _.length)
wenzelm@58706
    69
  }
wenzelm@58747
    70
  case class Document_Atom(command: Command) extends Document
wenzelm@58706
    71
  {
wenzelm@58706
    72
    def length: Int = command.length
wenzelm@58706
    73
  }
wenzelm@43774
    74
}
wenzelm@43774
    75
wenzelm@46712
    76
final class Outer_Syntax private(
wenzelm@58900
    77
  val keywords: Keyword.Keywords = Keyword.Keywords.empty,
wenzelm@53280
    78
  val completion: Completion = Completion.empty,
wenzelm@55749
    79
  val language_context: Completion.Language_Context = Completion.Language_Context.outer,
wenzelm@63584
    80
  val has_tokens: Boolean = true)
wenzelm@34166
    81
{
wenzelm@58706
    82
  /** syntax content **/
wenzelm@58706
    83
wenzelm@58900
    84
  override def toString: String = keywords.toString
wenzelm@56393
    85
wenzelm@58695
    86
wenzelm@58695
    87
  /* add keywords */
wenzelm@58695
    88
wenzelm@63579
    89
  def + (name: String, kind: String = "", tags: List[String] = Nil): Outer_Syntax =
wenzelm@53280
    90
  {
wenzelm@63429
    91
    val keywords1 = keywords + (name, kind, tags)
wenzelm@53280
    92
    val completion1 =
wenzelm@63587
    93
      completion.add_keyword(name).
wenzelm@63587
    94
        add_abbrevs(
wenzelm@63587
    95
          (if (Keyword.theory_block.contains(kind)) List((name, name + "\nbegin\n\u0007\nend"))
wenzelm@63587
    96
           else Nil) :::
wenzelm@63587
    97
          (if (Completion.Word_Parsers.is_word(name)) List((name, name)) else Nil))
wenzelm@58900
    98
    new Outer_Syntax(keywords1, completion1, language_context, true)
wenzelm@53280
    99
  }
wenzelm@48706
   100
wenzelm@48873
   101
  def add_keywords(keywords: Thy_Header.Keywords): Outer_Syntax =
wenzelm@48873
   102
    (this /: keywords) {
wenzelm@63579
   103
      case (syntax, (name, ((kind, tags), _))) =>
wenzelm@63579
   104
        syntax + (Symbol.decode(name), kind, tags) + (Symbol.encode(name), kind, tags)
wenzelm@63579
   105
    }
wenzelm@63579
   106
wenzelm@63579
   107
  def add_abbrevs(abbrevs: Thy_Header.Abbrevs): Outer_Syntax =
wenzelm@63579
   108
    if (abbrevs.isEmpty) this
wenzelm@63579
   109
    else {
wenzelm@63579
   110
      val completion1 =
wenzelm@63579
   111
        completion.add_abbrevs(
wenzelm@63579
   112
          (for ((a, b) <- abbrevs) yield {
wenzelm@63579
   113
            val a1 = Symbol.decode(a)
wenzelm@63579
   114
            val a2 = Symbol.encode(a)
wenzelm@63579
   115
            val b1 = Symbol.decode(b)
wenzelm@63579
   116
            List((a1, b1), (a2, b1))
wenzelm@63579
   117
          }).flatten)
wenzelm@63579
   118
      new Outer_Syntax(keywords, completion1, language_context, has_tokens)
wenzelm@46940
   119
    }
wenzelm@34166
   120
wenzelm@58695
   121
wenzelm@59073
   122
  /* merge */
wenzelm@59073
   123
wenzelm@63584
   124
  def ++ (other: Outer_Syntax): Outer_Syntax =
wenzelm@59073
   125
    if (this eq other) this
wenzelm@59073
   126
    else {
wenzelm@59077
   127
      val keywords1 = keywords ++ other.asInstanceOf[Outer_Syntax].keywords
wenzelm@59077
   128
      val completion1 = completion ++ other.asInstanceOf[Outer_Syntax].completion
wenzelm@59077
   129
      if ((keywords eq keywords1) && (completion eq completion1)) this
wenzelm@59077
   130
      else new Outer_Syntax(keywords1, completion1, language_context, has_tokens)
wenzelm@59073
   131
    }
wenzelm@59073
   132
wenzelm@59073
   133
wenzelm@59735
   134
  /* load commands */
wenzelm@58900
   135
wenzelm@63441
   136
  def load_command(name: String): Option[List[String]] = keywords.load_commands.get(name)
wenzelm@58900
   137
  def load_commands_in(text: String): Boolean = keywords.load_commands_in(text)
wenzelm@58900
   138
wenzelm@58900
   139
wenzelm@58706
   140
  /* language context */
wenzelm@34166
   141
wenzelm@58706
   142
  def set_language_context(context: Completion.Language_Context): Outer_Syntax =
wenzelm@58900
   143
    new Outer_Syntax(keywords, completion, context, has_tokens)
wenzelm@58706
   144
wenzelm@58706
   145
  def no_tokens: Outer_Syntax =
wenzelm@46969
   146
  {
wenzelm@58900
   147
    require(keywords.is_empty)
wenzelm@58706
   148
    new Outer_Syntax(
wenzelm@58706
   149
      completion = completion,
wenzelm@58706
   150
      language_context = language_context,
wenzelm@58706
   151
      has_tokens = false)
wenzelm@46969
   152
  }
wenzelm@40454
   153
wenzelm@58706
   154
wenzelm@40454
   155
wenzelm@58706
   156
  /** parsing **/
wenzelm@34166
   157
wenzelm@58697
   158
  /* line-oriented structure */
wenzelm@58696
   159
wenzelm@63460
   160
  private val close_structure =
wenzelm@63592
   161
    Set(Keyword.NEXT_BLOCK, Keyword.QED_BLOCK, Keyword.PRF_CLOSE, Keyword.THY_END)
wenzelm@63460
   162
wenzelm@59924
   163
  def line_structure(tokens: List[Token], structure: Outer_Syntax.Line_Structure)
wenzelm@58700
   164
    : Outer_Syntax.Line_Structure =
wenzelm@58696
   165
  {
wenzelm@58700
   166
    val improper1 = tokens.forall(_.is_improper)
wenzelm@63592
   167
    val command1 = tokens.exists(_.is_begin_or_command)
wenzelm@58700
   168
wenzelm@63458
   169
    val command_depth =
wenzelm@63458
   170
      tokens.iterator.filter(_.is_proper).toStream.headOption match {
wenzelm@63458
   171
        case Some(tok) =>
wenzelm@63460
   172
          if (keywords.is_command(tok, close_structure))
wenzelm@63458
   173
            Some(structure.after_span_depth - 1)
wenzelm@63458
   174
          else None
wenzelm@63458
   175
        case None => None
wenzelm@63458
   176
      }
wenzelm@63458
   177
wenzelm@63592
   178
    val depth0 = structure.element_depth
wenzelm@58696
   179
    val depth1 =
wenzelm@63459
   180
      if (tokens.exists(tok =>
wenzelm@63592
   181
            keywords.is_before_command(tok) ||
wenzelm@63592
   182
            !tok.is_end && keywords.is_command(tok, Keyword.theory))) depth0
wenzelm@63458
   183
      else if (command_depth.isDefined) command_depth.get
wenzelm@59924
   184
      else if (command1) structure.after_span_depth
wenzelm@59924
   185
      else structure.span_depth
wenzelm@58700
   186
wenzelm@63592
   187
    val (span_depth1, after_span_depth1, element_depth1) =
wenzelm@63592
   188
      ((structure.span_depth, structure.after_span_depth, structure.element_depth) /: tokens) {
wenzelm@63592
   189
        case (depths @ (x, y, z), tok) =>
wenzelm@63592
   190
          if (tok.is_begin) (z + 2, z + 1, z + 1)
wenzelm@63592
   191
          else if (tok.is_end) (z + 1, z - 1, z - 1)
wenzelm@63592
   192
          else if (tok.is_command) {
wenzelm@63592
   193
            if (keywords.is_command(tok, Keyword.theory_goal)) (depth0 + 2, depth0 + 1, z)
wenzelm@63592
   194
            else if (keywords.is_command(tok, Keyword.theory)) (depth0 + 1, depth0, z)
wenzelm@63592
   195
            else if (keywords.is_command(tok, Keyword.proof_open)) (y + 2, y + 1, z)
wenzelm@63592
   196
            else if (keywords.is_command(tok, Set(Keyword.PRF_BLOCK))) (y + 2, y + 1, z)
wenzelm@63592
   197
            else if (keywords.is_command(tok, Set(Keyword.QED_BLOCK))) (y - 1, y - 2, z)
wenzelm@63592
   198
            else if (keywords.is_command(tok, Set(Keyword.PRF_CLOSE))) (y, y - 1, z)
wenzelm@63592
   199
            else if (keywords.is_command(tok, Keyword.proof_close)) (y + 1, y - 1, z)
wenzelm@63592
   200
            else if (keywords.is_command(tok, Keyword.qed_global)) (depth0 + 1, depth0, z)
wenzelm@63592
   201
            else depths
wenzelm@58703
   202
          }
wenzelm@63592
   203
          else depths
wenzelm@58696
   204
      }
wenzelm@58700
   205
wenzelm@63592
   206
    Outer_Syntax.Line_Structure(
wenzelm@63592
   207
      improper1, command1, depth1, span_depth1, after_span_depth1, element_depth1)
wenzelm@58696
   208
  }
wenzelm@58696
   209
wenzelm@58696
   210
wenzelm@58706
   211
  /* command spans */
wenzelm@57905
   212
wenzelm@57905
   213
  def parse_spans(toks: List[Token]): List[Command_Span.Span] =
wenzelm@57905
   214
  {
wenzelm@57905
   215
    val result = new mutable.ListBuffer[Command_Span.Span]
wenzelm@57905
   216
    val content = new mutable.ListBuffer[Token]
wenzelm@57905
   217
    val improper = new mutable.ListBuffer[Token]
wenzelm@57905
   218
wenzelm@57905
   219
    def ship(span: List[Token])
wenzelm@57905
   220
    {
wenzelm@57905
   221
      val kind =
wenzelm@59924
   222
        if (span.forall(_.is_improper)) Command_Span.Ignored_Span
wenzelm@59924
   223
        else if (span.exists(_.is_error)) Command_Span.Malformed_Span
wenzelm@59924
   224
        else
wenzelm@59924
   225
          span.find(_.is_command) match {
wenzelm@59924
   226
            case None => Command_Span.Malformed_Span
wenzelm@59924
   227
            case Some(cmd) =>
wenzelm@59924
   228
              val name = cmd.source
wenzelm@59924
   229
              val offset =
wenzelm@59924
   230
                (0 /: span.takeWhile(_ != cmd)) {
wenzelm@59924
   231
                  case (i, tok) => i + Symbol.iterator(tok.source).length }
wenzelm@59924
   232
              val end_offset = offset + Symbol.iterator(name).length
wenzelm@59924
   233
              val pos = Position.Range(Text.Range(offset, end_offset) + 1)
wenzelm@59924
   234
              Command_Span.Command_Span(name, pos)
wenzelm@59924
   235
          }
wenzelm@57905
   236
      result += Command_Span.Span(kind, span)
wenzelm@57905
   237
    }
wenzelm@57905
   238
wenzelm@57905
   239
    def flush()
wenzelm@57905
   240
    {
wenzelm@59319
   241
      if (content.nonEmpty) { ship(content.toList); content.clear }
wenzelm@59319
   242
      if (improper.nonEmpty) { ship(improper.toList); improper.clear }
wenzelm@57905
   243
    }
wenzelm@57905
   244
wenzelm@57905
   245
    for (tok <- toks) {
wenzelm@59924
   246
      if (tok.is_improper) improper += tok
wenzelm@63441
   247
      else if (keywords.is_before_command(tok) ||
wenzelm@63441
   248
        tok.is_command &&
wenzelm@63441
   249
          (!content.exists(keywords.is_before_command(_)) || content.exists(_.is_command)))
wenzelm@59924
   250
      { flush(); content += tok }
wenzelm@57905
   251
      else { content ++= improper; improper.clear; content += tok }
wenzelm@57905
   252
    }
wenzelm@57905
   253
    flush()
wenzelm@57905
   254
wenzelm@57905
   255
    result.toList
wenzelm@57905
   256
  }
wenzelm@57905
   257
wenzelm@57906
   258
  def parse_spans(input: CharSequence): List[Command_Span.Span] =
wenzelm@59083
   259
    parse_spans(Token.explode(keywords, input))
wenzelm@57906
   260
wenzelm@57905
   261
wenzelm@58706
   262
  /* overall document structure */
wenzelm@55616
   263
wenzelm@58706
   264
  def heading_level(command: Command): Option[Int] =
wenzelm@58706
   265
  {
wenzelm@59735
   266
    val name = command.span.name
wenzelm@59735
   267
    name match {
wenzelm@59735
   268
      case Thy_Header.CHAPTER => Some(0)
wenzelm@62453
   269
      case Thy_Header.SECTION => Some(1)
wenzelm@59735
   270
      case Thy_Header.SUBSECTION => Some(2)
wenzelm@59735
   271
      case Thy_Header.SUBSUBSECTION => Some(3)
wenzelm@61463
   272
      case Thy_Header.PARAGRAPH => Some(4)
wenzelm@61463
   273
      case Thy_Header.SUBPARAGRAPH => Some(5)
wenzelm@58868
   274
      case _ =>
wenzelm@63441
   275
        keywords.kinds.get(name) match {
wenzelm@61463
   276
          case Some(kind) if Keyword.theory(kind) && !Keyword.theory_end(kind) => Some(6)
wenzelm@58868
   277
          case _ => None
wenzelm@58868
   278
        }
wenzelm@58706
   279
    }
wenzelm@58706
   280
  }
wenzelm@58706
   281
wenzelm@58743
   282
  def parse_document(node_name: Document.Node.Name, text: CharSequence):
wenzelm@58743
   283
    List[Outer_Syntax.Document] =
wenzelm@58706
   284
  {
wenzelm@58706
   285
    /* stack operations */
wenzelm@58706
   286
wenzelm@58706
   287
    def buffer(): mutable.ListBuffer[Outer_Syntax.Document] =
wenzelm@58706
   288
      new mutable.ListBuffer[Outer_Syntax.Document]
wenzelm@58706
   289
wenzelm@58747
   290
    var stack: List[(Int, Command, mutable.ListBuffer[Outer_Syntax.Document])] =
wenzelm@58747
   291
      List((0, Command.empty, buffer()))
wenzelm@55616
   292
wenzelm@58706
   293
    @tailrec def close(level: Int => Boolean)
wenzelm@58706
   294
    {
wenzelm@58706
   295
      stack match {
wenzelm@58747
   296
        case (lev, command, body) :: (_, _, body2) :: rest if level(lev) =>
wenzelm@59735
   297
          body2 += Outer_Syntax.Document_Block(command.span.name, command.source, body.toList)
wenzelm@58706
   298
          stack = stack.tail
wenzelm@58706
   299
          close(level)
wenzelm@58706
   300
        case _ =>
wenzelm@58706
   301
      }
wenzelm@58706
   302
    }
wenzelm@58706
   303
wenzelm@58743
   304
    def result(): List[Outer_Syntax.Document] =
wenzelm@58706
   305
    {
wenzelm@58706
   306
      close(_ => true)
wenzelm@58743
   307
      stack.head._3.toList
wenzelm@58706
   308
    }
wenzelm@58706
   309
wenzelm@58706
   310
    def add(command: Command)
wenzelm@58706
   311
    {
wenzelm@58706
   312
      heading_level(command) match {
wenzelm@58706
   313
        case Some(i) =>
wenzelm@58706
   314
          close(_ > i)
wenzelm@58747
   315
          stack = (i + 1, command, buffer()) :: stack
wenzelm@58706
   316
        case None =>
wenzelm@58706
   317
      }
wenzelm@58706
   318
      stack.head._3 += Outer_Syntax.Document_Atom(command)
wenzelm@58706
   319
    }
wenzelm@58706
   320
wenzelm@58706
   321
wenzelm@58706
   322
    /* result structure */
wenzelm@58706
   323
wenzelm@58706
   324
    val spans = parse_spans(text)
wenzelm@59702
   325
    spans.foreach(span => add(Command(Document_ID.none, node_name, Command.no_blobs, span)))
wenzelm@58706
   326
    result()
wenzelm@55616
   327
  }
wenzelm@34166
   328
}