src/Pure/Isar/outer_syntax.scala
author wenzelm
Tue Dec 02 14:16:56 2014 +0100 (2014-12-02)
changeset 59077 7e0d3da6e6d8
parent 59073 dcecfcc56dce
child 59083 88b0b1f28adc
permissions -rw-r--r--
node-specific syntax, with base_syntax as default;
clarified Document_Model.init: convergence of editor events towards buffer-specific token marker;
wenzelm@34166
     1
/*  Title:      Pure/Isar/outer_syntax.scala
wenzelm@34166
     2
    Author:     Makarius
wenzelm@34166
     3
wenzelm@34166
     4
Isabelle/Isar outer syntax.
wenzelm@34166
     5
*/
wenzelm@34166
     6
wenzelm@34166
     7
package isabelle
wenzelm@34166
     8
wenzelm@34166
     9
wenzelm@34166
    10
import scala.util.parsing.input.{Reader, CharSequenceReader}
wenzelm@43411
    11
import scala.collection.mutable
wenzelm@58706
    12
import scala.annotation.tailrec
wenzelm@34166
    13
wenzelm@34166
    14
wenzelm@43774
    15
object Outer_Syntax
wenzelm@43774
    16
{
wenzelm@58706
    17
  /* syntax */
wenzelm@58706
    18
wenzelm@58706
    19
  val empty: Outer_Syntax = new Outer_Syntax()
wenzelm@58706
    20
wenzelm@58706
    21
  def init(): Outer_Syntax = new Outer_Syntax(completion = Completion.init())
wenzelm@58706
    22
wenzelm@58706
    23
wenzelm@58706
    24
  /* string literals */
wenzelm@58706
    25
wenzelm@43774
    26
  def quote_string(str: String): String =
wenzelm@43774
    27
  {
wenzelm@43774
    28
    val result = new StringBuilder(str.length + 10)
wenzelm@43774
    29
    result += '"'
wenzelm@43774
    30
    for (s <- Symbol.iterator(str)) {
wenzelm@43774
    31
      if (s.length == 1) {
wenzelm@43774
    32
        val c = s(0)
wenzelm@43774
    33
        if (c < 32 && c != YXML.X && c != YXML.Y || c == '\\' || c == '"') {
wenzelm@43774
    34
          result += '\\'
wenzelm@43774
    35
          if (c < 10) result += '0'
wenzelm@43774
    36
          if (c < 100) result += '0'
wenzelm@43774
    37
          result ++= (c.asInstanceOf[Int].toString)
wenzelm@43774
    38
        }
wenzelm@43774
    39
        else result += c
wenzelm@43774
    40
      }
wenzelm@43774
    41
      else result ++= s
wenzelm@43774
    42
    }
wenzelm@43774
    43
    result += '"'
wenzelm@43774
    44
    result.toString
wenzelm@43774
    45
  }
wenzelm@46626
    46
wenzelm@58696
    47
wenzelm@58697
    48
  /* line-oriented structure */
wenzelm@58696
    49
wenzelm@58697
    50
  object Line_Structure
wenzelm@58696
    51
  {
wenzelm@58700
    52
    val init = Line_Structure()
wenzelm@58696
    53
  }
wenzelm@58696
    54
wenzelm@58700
    55
  sealed case class Line_Structure(
wenzelm@58700
    56
    improper: Boolean = true,
wenzelm@58700
    57
    command: Boolean = false,
wenzelm@58700
    58
    depth: Int = 0,
wenzelm@58700
    59
    span_depth: Int = 0,
wenzelm@58700
    60
    after_span_depth: Int = 0)
wenzelm@58706
    61
wenzelm@58706
    62
wenzelm@58706
    63
  /* overall document structure */
wenzelm@58706
    64
wenzelm@58706
    65
  sealed abstract class Document { def length: Int }
wenzelm@58747
    66
  case class Document_Block(name: String, text: String, body: List[Document]) extends Document
wenzelm@58706
    67
  {
wenzelm@58706
    68
    val length: Int = (0 /: body)(_ + _.length)
wenzelm@58706
    69
  }
wenzelm@58747
    70
  case class Document_Atom(command: Command) extends Document
wenzelm@58706
    71
  {
wenzelm@58706
    72
    def length: Int = command.length
wenzelm@58706
    73
  }
wenzelm@43774
    74
}
wenzelm@43774
    75
wenzelm@46712
    76
final class Outer_Syntax private(
wenzelm@58900
    77
  val keywords: Keyword.Keywords = Keyword.Keywords.empty,
wenzelm@53280
    78
  val completion: Completion = Completion.empty,
wenzelm@55749
    79
  val language_context: Completion.Language_Context = Completion.Language_Context.outer,
wenzelm@56393
    80
  val has_tokens: Boolean = true) extends Prover.Syntax
wenzelm@34166
    81
{
wenzelm@58706
    82
  /** syntax content **/
wenzelm@58706
    83
wenzelm@58900
    84
  override def toString: String = keywords.toString
wenzelm@56393
    85
wenzelm@58695
    86
wenzelm@58695
    87
  /* add keywords */
wenzelm@58695
    88
wenzelm@58907
    89
  def + (name: String): Outer_Syntax = this + (name, None, None)
wenzelm@58907
    90
  def + (name: String, kind: String): Outer_Syntax = this + (name, Some((kind, Nil)), None)
wenzelm@58901
    91
  def + (name: String, opt_kind: Option[(String, List[String])], replace: Option[String])
wenzelm@58901
    92
    : Outer_Syntax =
wenzelm@53280
    93
  {
wenzelm@58901
    94
    val keywords1 =
wenzelm@58901
    95
      opt_kind match {
wenzelm@58901
    96
        case None => keywords + name
wenzelm@58901
    97
        case Some(kind) => keywords + (name, kind)
wenzelm@58901
    98
      }
wenzelm@53280
    99
    val completion1 =
wenzelm@58853
   100
      if (replace == Some("")) completion
wenzelm@53280
   101
      else completion + (name, replace getOrElse name)
wenzelm@58900
   102
    new Outer_Syntax(keywords1, completion1, language_context, true)
wenzelm@53280
   103
  }
wenzelm@48706
   104
wenzelm@48873
   105
  def add_keywords(keywords: Thy_Header.Keywords): Outer_Syntax =
wenzelm@48873
   106
    (this /: keywords) {
wenzelm@58901
   107
      case (syntax, (name, opt_spec, replace)) =>
wenzelm@58901
   108
        val opt_kind = opt_spec.map(_._1)
wenzelm@50128
   109
        syntax +
wenzelm@58901
   110
          (Symbol.decode(name), opt_kind, replace) +
wenzelm@58901
   111
          (Symbol.encode(name), opt_kind, replace)
wenzelm@46940
   112
    }
wenzelm@34166
   113
wenzelm@58695
   114
wenzelm@59073
   115
  /* merge */
wenzelm@59073
   116
wenzelm@59077
   117
  def ++ (other: Prover.Syntax): Prover.Syntax =
wenzelm@59073
   118
    if (this eq other) this
wenzelm@59073
   119
    else {
wenzelm@59077
   120
      val keywords1 = keywords ++ other.asInstanceOf[Outer_Syntax].keywords
wenzelm@59077
   121
      val completion1 = completion ++ other.asInstanceOf[Outer_Syntax].completion
wenzelm@59077
   122
      if ((keywords eq keywords1) && (completion eq completion1)) this
wenzelm@59077
   123
      else new Outer_Syntax(keywords1, completion1, language_context, has_tokens)
wenzelm@59073
   124
    }
wenzelm@59073
   125
wenzelm@59073
   126
wenzelm@58900
   127
  /* load commands */
wenzelm@58900
   128
wenzelm@58900
   129
  def load_command(name: String): Option[List[String]] = keywords.load_command(name)
wenzelm@58900
   130
  def load_commands_in(text: String): Boolean = keywords.load_commands_in(text)
wenzelm@58900
   131
wenzelm@58900
   132
wenzelm@58706
   133
  /* language context */
wenzelm@34166
   134
wenzelm@58706
   135
  def set_language_context(context: Completion.Language_Context): Outer_Syntax =
wenzelm@58900
   136
    new Outer_Syntax(keywords, completion, context, has_tokens)
wenzelm@58706
   137
wenzelm@58706
   138
  def no_tokens: Outer_Syntax =
wenzelm@46969
   139
  {
wenzelm@58900
   140
    require(keywords.is_empty)
wenzelm@58706
   141
    new Outer_Syntax(
wenzelm@58706
   142
      completion = completion,
wenzelm@58706
   143
      language_context = language_context,
wenzelm@58706
   144
      has_tokens = false)
wenzelm@46969
   145
  }
wenzelm@40454
   146
wenzelm@58706
   147
wenzelm@40454
   148
wenzelm@58706
   149
  /** parsing **/
wenzelm@34166
   150
wenzelm@58697
   151
  /* line-oriented structure */
wenzelm@58696
   152
wenzelm@58700
   153
  def line_structure(tokens: List[Token], struct: Outer_Syntax.Line_Structure)
wenzelm@58700
   154
    : Outer_Syntax.Line_Structure =
wenzelm@58696
   155
  {
wenzelm@58700
   156
    val improper1 = tokens.forall(_.is_improper)
wenzelm@58700
   157
    val command1 = tokens.exists(_.is_command)
wenzelm@58700
   158
wenzelm@58696
   159
    val depth1 =
wenzelm@58901
   160
      if (tokens.exists(tok => keywords.is_command_kind(tok, Keyword.theory))) 0
wenzelm@58700
   161
      else if (command1) struct.after_span_depth
wenzelm@58700
   162
      else struct.span_depth
wenzelm@58700
   163
wenzelm@58700
   164
    val (span_depth1, after_span_depth1) =
wenzelm@58700
   165
      ((struct.span_depth, struct.after_span_depth) /: tokens) {
wenzelm@58703
   166
        case ((x, y), tok) =>
wenzelm@58703
   167
          if (tok.is_command) {
wenzelm@58901
   168
            if (keywords.is_command_kind(tok, Keyword.theory_goal))
wenzelm@58900
   169
              (2, 1)
wenzelm@58901
   170
            else if (keywords.is_command_kind(tok, Keyword.theory))
wenzelm@58900
   171
              (1, 0)
wenzelm@58901
   172
            else if (keywords.is_command_kind(tok, Keyword.proof_goal) || tok.is_begin_block)
wenzelm@58900
   173
              (y + 2, y + 1)
wenzelm@58901
   174
            else if (keywords.is_command_kind(tok, Keyword.qed) || tok.is_end_block)
wenzelm@58900
   175
              (y + 1, y - 1)
wenzelm@58901
   176
            else if (keywords.is_command_kind(tok, Keyword.qed_global))
wenzelm@58900
   177
              (1, 0)
wenzelm@58703
   178
            else (x, y)
wenzelm@58703
   179
          }
wenzelm@58703
   180
          else (x, y)
wenzelm@58696
   181
      }
wenzelm@58700
   182
wenzelm@58700
   183
    Outer_Syntax.Line_Structure(improper1, command1, depth1, span_depth1, after_span_depth1)
wenzelm@58696
   184
  }
wenzelm@58696
   185
wenzelm@58696
   186
wenzelm@53280
   187
  /* token language */
wenzelm@53280
   188
wenzelm@57907
   189
  def scan(input: CharSequence): List[Token] =
wenzelm@52066
   190
  {
wenzelm@58503
   191
    val in: Reader[Char] = new CharSequenceReader(input)
wenzelm@58900
   192
    Token.Parsers.parseAll(Token.Parsers.rep(Token.Parsers.token(keywords)), in) match {
wenzelm@55494
   193
      case Token.Parsers.Success(tokens, _) => tokens
wenzelm@57907
   194
      case _ => error("Unexpected failure of tokenizing input:\n" + input.toString)
wenzelm@34166
   195
    }
wenzelm@52066
   196
  }
wenzelm@34166
   197
wenzelm@58748
   198
  def scan_line(input: CharSequence, context: Scan.Line_Context): (List[Token], Scan.Line_Context) =
wenzelm@52066
   199
  {
wenzelm@52066
   200
    var in: Reader[Char] = new CharSequenceReader(input)
wenzelm@52066
   201
    val toks = new mutable.ListBuffer[Token]
wenzelm@52066
   202
    var ctxt = context
wenzelm@52066
   203
    while (!in.atEnd) {
wenzelm@58900
   204
      Token.Parsers.parse(Token.Parsers.token_line(keywords, ctxt), in) match {
wenzelm@55494
   205
        case Token.Parsers.Success((x, c), rest) => { toks += x; ctxt = c; in = rest }
wenzelm@55494
   206
        case Token.Parsers.NoSuccess(_, rest) =>
wenzelm@52066
   207
          error("Unexpected failure of tokenizing input:\n" + rest.source.toString)
wenzelm@43411
   208
      }
wenzelm@43411
   209
    }
wenzelm@58748
   210
    (toks.toList, ctxt)
wenzelm@52066
   211
  }
wenzelm@55616
   212
wenzelm@55616
   213
wenzelm@58706
   214
  /* command spans */
wenzelm@57905
   215
wenzelm@57905
   216
  def parse_spans(toks: List[Token]): List[Command_Span.Span] =
wenzelm@57905
   217
  {
wenzelm@57905
   218
    val result = new mutable.ListBuffer[Command_Span.Span]
wenzelm@57905
   219
    val content = new mutable.ListBuffer[Token]
wenzelm@57905
   220
    val improper = new mutable.ListBuffer[Token]
wenzelm@57905
   221
wenzelm@57905
   222
    def ship(span: List[Token])
wenzelm@57905
   223
    {
wenzelm@57905
   224
      val kind =
wenzelm@57910
   225
        if (!span.isEmpty && span.head.is_command && !span.exists(_.is_error)) {
wenzelm@57910
   226
          val name = span.head.source
wenzelm@57911
   227
          val pos = Position.Range(Text.Range(0, Symbol.iterator(name).length) + 1)
wenzelm@57910
   228
          Command_Span.Command_Span(name, pos)
wenzelm@57910
   229
        }
wenzelm@57905
   230
        else if (span.forall(_.is_improper)) Command_Span.Ignored_Span
wenzelm@57905
   231
        else Command_Span.Malformed_Span
wenzelm@57905
   232
      result += Command_Span.Span(kind, span)
wenzelm@57905
   233
    }
wenzelm@57905
   234
wenzelm@57905
   235
    def flush()
wenzelm@57905
   236
    {
wenzelm@57905
   237
      if (!content.isEmpty) { ship(content.toList); content.clear }
wenzelm@57905
   238
      if (!improper.isEmpty) { ship(improper.toList); improper.clear }
wenzelm@57905
   239
    }
wenzelm@57905
   240
wenzelm@57905
   241
    for (tok <- toks) {
wenzelm@57905
   242
      if (tok.is_command) { flush(); content += tok }
wenzelm@57905
   243
      else if (tok.is_improper) improper += tok
wenzelm@57905
   244
      else { content ++= improper; improper.clear; content += tok }
wenzelm@57905
   245
    }
wenzelm@57905
   246
    flush()
wenzelm@57905
   247
wenzelm@57905
   248
    result.toList
wenzelm@57905
   249
  }
wenzelm@57905
   250
wenzelm@57906
   251
  def parse_spans(input: CharSequence): List[Command_Span.Span] =
wenzelm@57906
   252
    parse_spans(scan(input))
wenzelm@57906
   253
wenzelm@57905
   254
wenzelm@58706
   255
  /* overall document structure */
wenzelm@55616
   256
wenzelm@58706
   257
  def heading_level(command: Command): Option[Int] =
wenzelm@58706
   258
  {
wenzelm@58868
   259
    command.name match {
wenzelm@58868
   260
      case "chapter" => Some(0)
wenzelm@58868
   261
      case "section" | "header" => Some(1)
wenzelm@58868
   262
      case "subsection" => Some(2)
wenzelm@58868
   263
      case "subsubsection" => Some(3)
wenzelm@58868
   264
      case _ =>
wenzelm@58901
   265
        keywords.command_kind(command.name) match {
wenzelm@58938
   266
          case Some(kind) if Keyword.theory(kind) && kind != Keyword.THY_END => Some(4)
wenzelm@58868
   267
          case _ => None
wenzelm@58868
   268
        }
wenzelm@58706
   269
    }
wenzelm@58706
   270
  }
wenzelm@58706
   271
wenzelm@58743
   272
  def parse_document(node_name: Document.Node.Name, text: CharSequence):
wenzelm@58743
   273
    List[Outer_Syntax.Document] =
wenzelm@58706
   274
  {
wenzelm@58706
   275
    /* stack operations */
wenzelm@58706
   276
wenzelm@58706
   277
    def buffer(): mutable.ListBuffer[Outer_Syntax.Document] =
wenzelm@58706
   278
      new mutable.ListBuffer[Outer_Syntax.Document]
wenzelm@58706
   279
wenzelm@58747
   280
    var stack: List[(Int, Command, mutable.ListBuffer[Outer_Syntax.Document])] =
wenzelm@58747
   281
      List((0, Command.empty, buffer()))
wenzelm@55616
   282
wenzelm@58706
   283
    @tailrec def close(level: Int => Boolean)
wenzelm@58706
   284
    {
wenzelm@58706
   285
      stack match {
wenzelm@58747
   286
        case (lev, command, body) :: (_, _, body2) :: rest if level(lev) =>
wenzelm@58747
   287
          body2 += Outer_Syntax.Document_Block(command.name, command.source, body.toList)
wenzelm@58706
   288
          stack = stack.tail
wenzelm@58706
   289
          close(level)
wenzelm@58706
   290
        case _ =>
wenzelm@58706
   291
      }
wenzelm@58706
   292
    }
wenzelm@58706
   293
wenzelm@58743
   294
    def result(): List[Outer_Syntax.Document] =
wenzelm@58706
   295
    {
wenzelm@58706
   296
      close(_ => true)
wenzelm@58743
   297
      stack.head._3.toList
wenzelm@58706
   298
    }
wenzelm@58706
   299
wenzelm@58706
   300
    def add(command: Command)
wenzelm@58706
   301
    {
wenzelm@58706
   302
      heading_level(command) match {
wenzelm@58706
   303
        case Some(i) =>
wenzelm@58706
   304
          close(_ > i)
wenzelm@58747
   305
          stack = (i + 1, command, buffer()) :: stack
wenzelm@58706
   306
        case None =>
wenzelm@58706
   307
      }
wenzelm@58706
   308
      stack.head._3 += Outer_Syntax.Document_Atom(command)
wenzelm@58706
   309
    }
wenzelm@58706
   310
wenzelm@58706
   311
wenzelm@58706
   312
    /* result structure */
wenzelm@58706
   313
wenzelm@58706
   314
    val spans = parse_spans(text)
wenzelm@58706
   315
    spans.foreach(span => add(Command(Document_ID.none, node_name, Nil, span)))
wenzelm@58706
   316
    result()
wenzelm@55616
   317
  }
wenzelm@34166
   318
}