src/Pure/Isar/outer_syntax.scala
author wenzelm
Sat Oct 18 20:56:16 2014 +0200 (2014-10-18)
changeset 58700 4717d18cc619
parent 58697 5bc1d6c4a499
child 58703 883efcc7a50d
permissions -rw-r--r--
clarified Line_Structure wrt. command span;
wenzelm@34166
     1
/*  Title:      Pure/Isar/outer_syntax.scala
wenzelm@34166
     2
    Author:     Makarius
wenzelm@34166
     3
wenzelm@34166
     4
Isabelle/Isar outer syntax.
wenzelm@34166
     5
*/
wenzelm@34166
     6
wenzelm@34166
     7
package isabelle
wenzelm@34166
     8
wenzelm@34166
     9
wenzelm@34166
    10
import scala.util.parsing.input.{Reader, CharSequenceReader}
wenzelm@43411
    11
import scala.collection.mutable
wenzelm@34166
    12
wenzelm@34166
    13
wenzelm@43774
    14
object Outer_Syntax
wenzelm@43774
    15
{
wenzelm@43774
    16
  def quote_string(str: String): String =
wenzelm@43774
    17
  {
wenzelm@43774
    18
    val result = new StringBuilder(str.length + 10)
wenzelm@43774
    19
    result += '"'
wenzelm@43774
    20
    for (s <- Symbol.iterator(str)) {
wenzelm@43774
    21
      if (s.length == 1) {
wenzelm@43774
    22
        val c = s(0)
wenzelm@43774
    23
        if (c < 32 && c != YXML.X && c != YXML.Y || c == '\\' || c == '"') {
wenzelm@43774
    24
          result += '\\'
wenzelm@43774
    25
          if (c < 10) result += '0'
wenzelm@43774
    26
          if (c < 100) result += '0'
wenzelm@43774
    27
          result ++= (c.asInstanceOf[Int].toString)
wenzelm@43774
    28
        }
wenzelm@43774
    29
        else result += c
wenzelm@43774
    30
      }
wenzelm@43774
    31
      else result ++= s
wenzelm@43774
    32
    }
wenzelm@43774
    33
    result += '"'
wenzelm@43774
    34
    result.toString
wenzelm@43774
    35
  }
wenzelm@46626
    36
wenzelm@46941
    37
  val empty: Outer_Syntax = new Outer_Syntax()
wenzelm@48870
    38
wenzelm@46941
    39
  def init(): Outer_Syntax = new Outer_Syntax(completion = Completion.init())
wenzelm@58696
    40
wenzelm@58696
    41
wenzelm@58697
    42
  /* line-oriented structure */
wenzelm@58696
    43
wenzelm@58697
    44
  object Line_Structure
wenzelm@58696
    45
  {
wenzelm@58700
    46
    val init = Line_Structure()
wenzelm@58696
    47
  }
wenzelm@58696
    48
wenzelm@58700
    49
  sealed case class Line_Structure(
wenzelm@58700
    50
    improper: Boolean = true,
wenzelm@58700
    51
    command: Boolean = false,
wenzelm@58700
    52
    depth: Int = 0,
wenzelm@58700
    53
    span_depth: Int = 0,
wenzelm@58700
    54
    after_span_depth: Int = 0)
wenzelm@43774
    55
}
wenzelm@43774
    56
wenzelm@46712
    57
final class Outer_Syntax private(
wenzelm@48864
    58
  keywords: Map[String, (String, List[String])] = Map.empty,
wenzelm@46626
    59
  lexicon: Scan.Lexicon = Scan.Lexicon.empty,
wenzelm@53280
    60
  val completion: Completion = Completion.empty,
wenzelm@55749
    61
  val language_context: Completion.Language_Context = Completion.Language_Context.outer,
wenzelm@56393
    62
  val has_tokens: Boolean = true) extends Prover.Syntax
wenzelm@34166
    63
{
wenzelm@48660
    64
  override def toString: String =
wenzelm@48864
    65
    (for ((name, (kind, files)) <- keywords) yield {
wenzelm@48660
    66
      if (kind == Keyword.MINOR) quote(name)
wenzelm@48864
    67
      else
wenzelm@48864
    68
        quote(name) + " :: " + quote(kind) +
wenzelm@48864
    69
        (if (files.isEmpty) "" else " (" + commas_quote(files) + ")")
wenzelm@48671
    70
    }).toList.sorted.mkString("keywords\n  ", " and\n  ", "")
wenzelm@48660
    71
wenzelm@58695
    72
wenzelm@58695
    73
  /* keyword kind */
wenzelm@58695
    74
wenzelm@48864
    75
  def keyword_kind_files(name: String): Option[(String, List[String])] = keywords.get(name)
wenzelm@48864
    76
  def keyword_kind(name: String): Option[String] = keyword_kind_files(name).map(_._1)
wenzelm@38471
    77
wenzelm@58695
    78
  def is_command(name: String): Boolean =
wenzelm@58695
    79
    keyword_kind(name) match {
wenzelm@58695
    80
      case Some(kind) => kind != Keyword.MINOR
wenzelm@58695
    81
      case None => false
wenzelm@58695
    82
    }
wenzelm@58695
    83
wenzelm@58696
    84
  def command_kind(token: Token, pred: String => Boolean): Boolean =
wenzelm@58696
    85
    token.is_command && is_command(token.source) &&
wenzelm@58696
    86
      pred(keyword_kind(token.source).get)
wenzelm@58696
    87
wenzelm@58695
    88
wenzelm@58695
    89
  /* load commands */
wenzelm@58695
    90
wenzelm@57901
    91
  def load_command(name: String): Option[List[String]] =
wenzelm@57901
    92
    keywords.get(name) match {
wenzelm@54513
    93
      case Some((Keyword.THY_LOAD, exts)) => Some(exts)
wenzelm@54462
    94
      case _ => None
wenzelm@54462
    95
    }
wenzelm@54462
    96
wenzelm@56314
    97
  val load_commands: List[(String, List[String])] =
wenzelm@48885
    98
    (for ((name, (Keyword.THY_LOAD, files)) <- keywords.iterator) yield (name, files)).toList
wenzelm@48872
    99
wenzelm@56393
   100
  def load_commands_in(text: String): Boolean =
wenzelm@56393
   101
    load_commands.exists({ case (cmd, _) => text.containsSlice(cmd) })
wenzelm@56393
   102
wenzelm@58695
   103
wenzelm@58695
   104
  /* add keywords */
wenzelm@58695
   105
wenzelm@50128
   106
  def + (name: String, kind: (String, List[String]), replace: Option[String]): Outer_Syntax =
wenzelm@53280
   107
  {
wenzelm@53280
   108
    val keywords1 = keywords + (name -> kind)
wenzelm@53280
   109
    val lexicon1 = lexicon + name
wenzelm@53280
   110
    val completion1 =
wenzelm@50128
   111
      if (Keyword.control(kind._1) || replace == Some("")) completion
wenzelm@53280
   112
      else completion + (name, replace getOrElse name)
wenzelm@55749
   113
    new Outer_Syntax(keywords1, lexicon1, completion1, language_context, true)
wenzelm@53280
   114
  }
wenzelm@34166
   115
wenzelm@53280
   116
  def + (name: String, kind: (String, List[String])): Outer_Syntax =
wenzelm@53280
   117
    this + (name, kind, Some(name))
wenzelm@53280
   118
  def + (name: String, kind: String): Outer_Syntax =
wenzelm@53280
   119
    this + (name, (kind, Nil), Some(name))
wenzelm@50128
   120
  def + (name: String, replace: Option[String]): Outer_Syntax =
wenzelm@50128
   121
    this + (name, (Keyword.MINOR, Nil), replace)
wenzelm@50128
   122
  def + (name: String): Outer_Syntax = this + (name, None)
wenzelm@48706
   123
wenzelm@48873
   124
  def add_keywords(keywords: Thy_Header.Keywords): Outer_Syntax =
wenzelm@48873
   125
    (this /: keywords) {
wenzelm@52439
   126
      case (syntax, (name, Some((kind, _)), replace)) =>
wenzelm@50128
   127
        syntax +
wenzelm@50128
   128
          (Symbol.decode(name), kind, replace) +
wenzelm@50128
   129
          (Symbol.encode(name), kind, replace)
wenzelm@52439
   130
      case (syntax, (name, None, replace)) =>
wenzelm@50128
   131
        syntax +
wenzelm@50128
   132
          (Symbol.decode(name), replace) +
wenzelm@50128
   133
          (Symbol.encode(name), replace)
wenzelm@46940
   134
    }
wenzelm@34166
   135
wenzelm@58695
   136
wenzelm@58695
   137
  /* document headings */
wenzelm@34166
   138
wenzelm@40454
   139
  def heading_level(name: String): Option[Int] =
wenzelm@46969
   140
  {
wenzelm@46969
   141
    keyword_kind(name) match {
wenzelm@46969
   142
      case _ if name == "header" => Some(0)
wenzelm@46969
   143
      case Some(Keyword.THY_HEADING1) => Some(1)
wenzelm@46969
   144
      case Some(Keyword.THY_HEADING2) | Some(Keyword.PRF_HEADING2) => Some(2)
wenzelm@46969
   145
      case Some(Keyword.THY_HEADING3) | Some(Keyword.PRF_HEADING3) => Some(3)
wenzelm@46969
   146
      case Some(Keyword.THY_HEADING4) | Some(Keyword.PRF_HEADING4) => Some(4)
wenzelm@46969
   147
      case Some(kind) if Keyword.theory(kind) => Some(5)
wenzelm@46969
   148
      case _ => None
wenzelm@40454
   149
    }
wenzelm@46969
   150
  }
wenzelm@40454
   151
wenzelm@40454
   152
  def heading_level(command: Command): Option[Int] =
wenzelm@40454
   153
    heading_level(command.name)
wenzelm@40454
   154
wenzelm@34166
   155
wenzelm@58697
   156
  /* line-oriented structure */
wenzelm@58696
   157
wenzelm@58700
   158
  def line_structure(tokens: List[Token], struct: Outer_Syntax.Line_Structure)
wenzelm@58700
   159
    : Outer_Syntax.Line_Structure =
wenzelm@58696
   160
  {
wenzelm@58700
   161
    val improper1 = tokens.forall(_.is_improper)
wenzelm@58700
   162
    val command1 = tokens.exists(_.is_command)
wenzelm@58700
   163
wenzelm@58696
   164
    val depth1 =
wenzelm@58696
   165
      if (tokens.exists(tok => command_kind(tok, Keyword.theory))) 0
wenzelm@58700
   166
      else if (command1) struct.after_span_depth
wenzelm@58700
   167
      else struct.span_depth
wenzelm@58700
   168
wenzelm@58700
   169
    val (span_depth1, after_span_depth1) =
wenzelm@58700
   170
      ((struct.span_depth, struct.after_span_depth) /: tokens) {
wenzelm@58700
   171
        case ((d0, d), tok) =>
wenzelm@58700
   172
          if (command_kind(tok, Keyword.theory_goal)) (2, 1)
wenzelm@58700
   173
          else if (command_kind(tok, Keyword.theory)) (1, 0)
wenzelm@58700
   174
          else if (command_kind(tok, Keyword.proof_goal)) (d + 2, d + 1)
wenzelm@58700
   175
          else if (command_kind(tok, Keyword.qed)) (d + 1, d - 1)
wenzelm@58700
   176
          else if (command_kind(tok, Keyword.qed_global)) (1, 0)
wenzelm@58700
   177
          else (d0, d)
wenzelm@58696
   178
      }
wenzelm@58700
   179
wenzelm@58700
   180
    Outer_Syntax.Line_Structure(improper1, command1, depth1, span_depth1, after_span_depth1)
wenzelm@58696
   181
  }
wenzelm@58696
   182
wenzelm@58696
   183
wenzelm@53280
   184
  /* token language */
wenzelm@53280
   185
wenzelm@57907
   186
  def scan(input: CharSequence): List[Token] =
wenzelm@52066
   187
  {
wenzelm@58503
   188
    val in: Reader[Char] = new CharSequenceReader(input)
wenzelm@55616
   189
    Token.Parsers.parseAll(
wenzelm@57907
   190
        Token.Parsers.rep(Token.Parsers.token(lexicon, is_command)), in) match {
wenzelm@55494
   191
      case Token.Parsers.Success(tokens, _) => tokens
wenzelm@57907
   192
      case _ => error("Unexpected failure of tokenizing input:\n" + input.toString)
wenzelm@34166
   193
    }
wenzelm@52066
   194
  }
wenzelm@34166
   195
wenzelm@58697
   196
  def scan_line(
wenzelm@58697
   197
    input: CharSequence,
wenzelm@58697
   198
    context: Scan.Line_Context,
wenzelm@58697
   199
    structure: Outer_Syntax.Line_Structure)
wenzelm@58697
   200
    : (List[Token], Scan.Line_Context, Outer_Syntax.Line_Structure) =
wenzelm@52066
   201
  {
wenzelm@52066
   202
    var in: Reader[Char] = new CharSequenceReader(input)
wenzelm@52066
   203
    val toks = new mutable.ListBuffer[Token]
wenzelm@52066
   204
    var ctxt = context
wenzelm@52066
   205
    while (!in.atEnd) {
wenzelm@55510
   206
      Token.Parsers.parse(Token.Parsers.token_line(lexicon, is_command, ctxt), in) match {
wenzelm@55494
   207
        case Token.Parsers.Success((x, c), rest) => { toks += x; ctxt = c; in = rest }
wenzelm@55494
   208
        case Token.Parsers.NoSuccess(_, rest) =>
wenzelm@52066
   209
          error("Unexpected failure of tokenizing input:\n" + rest.source.toString)
wenzelm@43411
   210
      }
wenzelm@43411
   211
    }
wenzelm@58696
   212
    val tokens = toks.toList
wenzelm@58700
   213
    (tokens, ctxt, line_structure(tokens, structure))
wenzelm@52066
   214
  }
wenzelm@55616
   215
wenzelm@55616
   216
wenzelm@57905
   217
  /* parse_spans */
wenzelm@57905
   218
wenzelm@57905
   219
  def parse_spans(toks: List[Token]): List[Command_Span.Span] =
wenzelm@57905
   220
  {
wenzelm@57905
   221
    val result = new mutable.ListBuffer[Command_Span.Span]
wenzelm@57905
   222
    val content = new mutable.ListBuffer[Token]
wenzelm@57905
   223
    val improper = new mutable.ListBuffer[Token]
wenzelm@57905
   224
wenzelm@57905
   225
    def ship(span: List[Token])
wenzelm@57905
   226
    {
wenzelm@57905
   227
      val kind =
wenzelm@57910
   228
        if (!span.isEmpty && span.head.is_command && !span.exists(_.is_error)) {
wenzelm@57910
   229
          val name = span.head.source
wenzelm@57911
   230
          val pos = Position.Range(Text.Range(0, Symbol.iterator(name).length) + 1)
wenzelm@57910
   231
          Command_Span.Command_Span(name, pos)
wenzelm@57910
   232
        }
wenzelm@57905
   233
        else if (span.forall(_.is_improper)) Command_Span.Ignored_Span
wenzelm@57905
   234
        else Command_Span.Malformed_Span
wenzelm@57905
   235
      result += Command_Span.Span(kind, span)
wenzelm@57905
   236
    }
wenzelm@57905
   237
wenzelm@57905
   238
    def flush()
wenzelm@57905
   239
    {
wenzelm@57905
   240
      if (!content.isEmpty) { ship(content.toList); content.clear }
wenzelm@57905
   241
      if (!improper.isEmpty) { ship(improper.toList); improper.clear }
wenzelm@57905
   242
    }
wenzelm@57905
   243
wenzelm@57905
   244
    for (tok <- toks) {
wenzelm@57905
   245
      if (tok.is_command) { flush(); content += tok }
wenzelm@57905
   246
      else if (tok.is_improper) improper += tok
wenzelm@57905
   247
      else { content ++= improper; improper.clear; content += tok }
wenzelm@57905
   248
    }
wenzelm@57905
   249
    flush()
wenzelm@57905
   250
wenzelm@57905
   251
    result.toList
wenzelm@57905
   252
  }
wenzelm@57905
   253
wenzelm@57906
   254
  def parse_spans(input: CharSequence): List[Command_Span.Span] =
wenzelm@57906
   255
    parse_spans(scan(input))
wenzelm@57906
   256
wenzelm@57905
   257
wenzelm@55616
   258
  /* language context */
wenzelm@55616
   259
wenzelm@55749
   260
  def set_language_context(context: Completion.Language_Context): Outer_Syntax =
wenzelm@55616
   261
    new Outer_Syntax(keywords, lexicon, completion, context, has_tokens)
wenzelm@55616
   262
wenzelm@55616
   263
  def no_tokens: Outer_Syntax =
wenzelm@55616
   264
  {
wenzelm@55616
   265
    require(keywords.isEmpty && lexicon.isEmpty)
wenzelm@55616
   266
    new Outer_Syntax(
wenzelm@55616
   267
      completion = completion,
wenzelm@55749
   268
      language_context = language_context,
wenzelm@55616
   269
      has_tokens = false)
wenzelm@55616
   270
  }
wenzelm@34166
   271
}