wenzelm@34166: /*  Title:      Pure/Isar/outer_syntax.scala
wenzelm@34166:     Author:     Makarius
wenzelm@34166: 
wenzelm@34166: Isabelle/Isar outer syntax.
wenzelm@34166: */
wenzelm@34166: 
wenzelm@34166: package isabelle
wenzelm@34166: 
wenzelm@34166: 
wenzelm@34166: import scala.util.parsing.input.{Reader, CharSequenceReader}
wenzelm@43411: import scala.collection.mutable
wenzelm@58706: import scala.annotation.tailrec
wenzelm@34166: 
wenzelm@34166: 
wenzelm@43774: object Outer_Syntax
wenzelm@43774: {
wenzelm@58706:   /* syntax */
wenzelm@58706: 
wenzelm@58706:   val empty: Outer_Syntax = new Outer_Syntax()
wenzelm@58706: 
wenzelm@58706:   def init(): Outer_Syntax = new Outer_Syntax(completion = Completion.init())
wenzelm@58706: 
wenzelm@58706: 
wenzelm@58706:   /* string literals */
wenzelm@58706: 
wenzelm@43774:   def quote_string(str: String): String =
wenzelm@43774:   {
wenzelm@43774:     val result = new StringBuilder(str.length + 10)
wenzelm@43774:     result += '"'
wenzelm@43774:     for (s <- Symbol.iterator(str)) {
wenzelm@43774:       if (s.length == 1) {
wenzelm@43774:         val c = s(0)
wenzelm@43774:         if (c < 32 && c != YXML.X && c != YXML.Y || c == '\\' || c == '"') {
wenzelm@43774:           result += '\\'
wenzelm@43774:           if (c < 10) result += '0'
wenzelm@43774:           if (c < 100) result += '0'
wenzelm@43774:           result ++= (c.asInstanceOf[Int].toString)
wenzelm@43774:         }
wenzelm@43774:         else result += c
wenzelm@43774:       }
wenzelm@43774:       else result ++= s
wenzelm@43774:     }
wenzelm@43774:     result += '"'
wenzelm@43774:     result.toString
wenzelm@43774:   }
wenzelm@46626: 
wenzelm@58696: 
wenzelm@58697:   /* line-oriented structure */
wenzelm@58696: 
wenzelm@58697:   object Line_Structure
wenzelm@58696:   {
wenzelm@58700:     val init = Line_Structure()
wenzelm@58696:   }
wenzelm@58696: 
wenzelm@58700:   sealed case class Line_Structure(
wenzelm@58700:     improper: Boolean = true,
wenzelm@58700:     command: Boolean = false,
wenzelm@58700:     depth: Int = 0,
wenzelm@58700:     span_depth: Int = 0,
wenzelm@58700:     after_span_depth: Int = 0)
wenzelm@58706: 
wenzelm@58706: 
wenzelm@58706:   /* overall document structure */
wenzelm@58706: 
wenzelm@58706:   sealed abstract class Document { def length: Int }
wenzelm@58747:   case class Document_Block(name: String, text: String, body: List[Document]) extends Document
wenzelm@58706:   {
wenzelm@58706:     val length: Int = (0 /: body)(_ + _.length)
wenzelm@58706:   }
wenzelm@58747:   case class Document_Atom(command: Command) extends Document
wenzelm@58706:   {
wenzelm@58706:     def length: Int = command.length
wenzelm@58706:   }
wenzelm@43774: }
wenzelm@43774: 
wenzelm@46712: final class Outer_Syntax private(
wenzelm@58900:   val keywords: Keyword.Keywords = Keyword.Keywords.empty,
wenzelm@53280:   val completion: Completion = Completion.empty,
wenzelm@55749:   val language_context: Completion.Language_Context = Completion.Language_Context.outer,
wenzelm@56393:   val has_tokens: Boolean = true) extends Prover.Syntax
wenzelm@34166: {
wenzelm@58706:   /** syntax content **/
wenzelm@58706: 
wenzelm@58900:   override def toString: String = keywords.toString
wenzelm@56393: 
wenzelm@58695: 
wenzelm@58695:   /* add keywords */
wenzelm@58695: 
wenzelm@58907:   def + (name: String): Outer_Syntax = this + (name, None, None)
wenzelm@58907:   def + (name: String, kind: String): Outer_Syntax = this + (name, Some((kind, Nil)), None)
wenzelm@58901:   def + (name: String, opt_kind: Option[(String, List[String])], replace: Option[String])
wenzelm@58901:     : Outer_Syntax =
wenzelm@53280:   {
wenzelm@58901:     val keywords1 =
wenzelm@58901:       opt_kind match {
wenzelm@58901:         case None => keywords + name
wenzelm@58901:         case Some(kind) => keywords + (name, kind)
wenzelm@58901:       }
wenzelm@53280:     val completion1 =
wenzelm@58853:       if (replace == Some("")) completion
wenzelm@53280:       else completion + (name, replace getOrElse name)
wenzelm@58900:     new Outer_Syntax(keywords1, completion1, language_context, true)
wenzelm@53280:   }
wenzelm@48706: 
wenzelm@48873:   def add_keywords(keywords: Thy_Header.Keywords): Outer_Syntax =
wenzelm@48873:     (this /: keywords) {
wenzelm@58901:       case (syntax, (name, opt_spec, replace)) =>
wenzelm@58901:         val opt_kind = opt_spec.map(_._1)
wenzelm@50128:         syntax +
wenzelm@58901:           (Symbol.decode(name), opt_kind, replace) +
wenzelm@58901:           (Symbol.encode(name), opt_kind, replace)
wenzelm@46940:     }
wenzelm@34166: 
wenzelm@58695: 
wenzelm@59073:   /* merge */
wenzelm@59073: 
wenzelm@59073:   def ++ (other: Outer_Syntax): Outer_Syntax =
wenzelm@59073:     if (this eq other) this
wenzelm@59073:     else {
wenzelm@59073:       val keywords1 = keywords ++ other.keywords
wenzelm@59073:       val completion1 = completion ++ other.completion
wenzelm@59073:       new Outer_Syntax(keywords1, completion1, language_context, has_tokens)
wenzelm@59073:     }
wenzelm@59073: 
wenzelm@59073: 
wenzelm@58900:   /* load commands */
wenzelm@58900: 
wenzelm@58900:   def load_command(name: String): Option[List[String]] = keywords.load_command(name)
wenzelm@58900:   def load_commands_in(text: String): Boolean = keywords.load_commands_in(text)
wenzelm@58900: 
wenzelm@58900: 
wenzelm@58706:   /* language context */
wenzelm@34166: 
wenzelm@58706:   def set_language_context(context: Completion.Language_Context): Outer_Syntax =
wenzelm@58900:     new Outer_Syntax(keywords, completion, context, has_tokens)
wenzelm@58706: 
wenzelm@58706:   def no_tokens: Outer_Syntax =
wenzelm@46969:   {
wenzelm@58900:     require(keywords.is_empty)
wenzelm@58706:     new Outer_Syntax(
wenzelm@58706:       completion = completion,
wenzelm@58706:       language_context = language_context,
wenzelm@58706:       has_tokens = false)
wenzelm@46969:   }
wenzelm@40454: 
wenzelm@58706: 
wenzelm@40454: 
wenzelm@58706:   /** parsing **/
wenzelm@34166: 
wenzelm@58697:   /* line-oriented structure */
wenzelm@58696: 
wenzelm@58700:   def line_structure(tokens: List[Token], struct: Outer_Syntax.Line_Structure)
wenzelm@58700:     : Outer_Syntax.Line_Structure =
wenzelm@58696:   {
wenzelm@58700:     val improper1 = tokens.forall(_.is_improper)
wenzelm@58700:     val command1 = tokens.exists(_.is_command)
wenzelm@58700: 
wenzelm@58696:     val depth1 =
wenzelm@58901:       if (tokens.exists(tok => keywords.is_command_kind(tok, Keyword.theory))) 0
wenzelm@58700:       else if (command1) struct.after_span_depth
wenzelm@58700:       else struct.span_depth
wenzelm@58700: 
wenzelm@58700:     val (span_depth1, after_span_depth1) =
wenzelm@58700:       ((struct.span_depth, struct.after_span_depth) /: tokens) {
wenzelm@58703:         case ((x, y), tok) =>
wenzelm@58703:           if (tok.is_command) {
wenzelm@58901:             if (keywords.is_command_kind(tok, Keyword.theory_goal))
wenzelm@58900:               (2, 1)
wenzelm@58901:             else if (keywords.is_command_kind(tok, Keyword.theory))
wenzelm@58900:               (1, 0)
wenzelm@58901:             else if (keywords.is_command_kind(tok, Keyword.proof_goal) || tok.is_begin_block)
wenzelm@58900:               (y + 2, y + 1)
wenzelm@58901:             else if (keywords.is_command_kind(tok, Keyword.qed) || tok.is_end_block)
wenzelm@58900:               (y + 1, y - 1)
wenzelm@58901:             else if (keywords.is_command_kind(tok, Keyword.qed_global))
wenzelm@58900:               (1, 0)
wenzelm@58703:             else (x, y)
wenzelm@58703:           }
wenzelm@58703:           else (x, y)
wenzelm@58696:       }
wenzelm@58700: 
wenzelm@58700:     Outer_Syntax.Line_Structure(improper1, command1, depth1, span_depth1, after_span_depth1)
wenzelm@58696:   }
wenzelm@58696: 
wenzelm@58696: 
wenzelm@53280:   /* token language */
wenzelm@53280: 
wenzelm@57907:   def scan(input: CharSequence): List[Token] =
wenzelm@52066:   {
wenzelm@58503:     val in: Reader[Char] = new CharSequenceReader(input)
wenzelm@58900:     Token.Parsers.parseAll(Token.Parsers.rep(Token.Parsers.token(keywords)), in) match {
wenzelm@55494:       case Token.Parsers.Success(tokens, _) => tokens
wenzelm@57907:       case _ => error("Unexpected failure of tokenizing input:\n" + input.toString)
wenzelm@34166:     }
wenzelm@52066:   }
wenzelm@34166: 
wenzelm@58748:   def scan_line(input: CharSequence, context: Scan.Line_Context): (List[Token], Scan.Line_Context) =
wenzelm@52066:   {
wenzelm@52066:     var in: Reader[Char] = new CharSequenceReader(input)
wenzelm@52066:     val toks = new mutable.ListBuffer[Token]
wenzelm@52066:     var ctxt = context
wenzelm@52066:     while (!in.atEnd) {
wenzelm@58900:       Token.Parsers.parse(Token.Parsers.token_line(keywords, ctxt), in) match {
wenzelm@55494:         case Token.Parsers.Success((x, c), rest) => { toks += x; ctxt = c; in = rest }
wenzelm@55494:         case Token.Parsers.NoSuccess(_, rest) =>
wenzelm@52066:           error("Unexpected failure of tokenizing input:\n" + rest.source.toString)
wenzelm@43411:       }
wenzelm@43411:     }
wenzelm@58748:     (toks.toList, ctxt)
wenzelm@52066:   }
wenzelm@55616: 
wenzelm@55616: 
wenzelm@58706:   /* command spans */
wenzelm@57905: 
wenzelm@57905:   def parse_spans(toks: List[Token]): List[Command_Span.Span] =
wenzelm@57905:   {
wenzelm@57905:     val result = new mutable.ListBuffer[Command_Span.Span]
wenzelm@57905:     val content = new mutable.ListBuffer[Token]
wenzelm@57905:     val improper = new mutable.ListBuffer[Token]
wenzelm@57905: 
wenzelm@57905:     def ship(span: List[Token])
wenzelm@57905:     {
wenzelm@57905:       val kind =
wenzelm@57910:         if (!span.isEmpty && span.head.is_command && !span.exists(_.is_error)) {
wenzelm@57910:           val name = span.head.source
wenzelm@57911:           val pos = Position.Range(Text.Range(0, Symbol.iterator(name).length) + 1)
wenzelm@57910:           Command_Span.Command_Span(name, pos)
wenzelm@57910:         }
wenzelm@57905:         else if (span.forall(_.is_improper)) Command_Span.Ignored_Span
wenzelm@57905:         else Command_Span.Malformed_Span
wenzelm@57905:       result += Command_Span.Span(kind, span)
wenzelm@57905:     }
wenzelm@57905: 
wenzelm@57905:     def flush()
wenzelm@57905:     {
wenzelm@57905:       if (!content.isEmpty) { ship(content.toList); content.clear }
wenzelm@57905:       if (!improper.isEmpty) { ship(improper.toList); improper.clear }
wenzelm@57905:     }
wenzelm@57905: 
wenzelm@57905:     for (tok <- toks) {
wenzelm@57905:       if (tok.is_command) { flush(); content += tok }
wenzelm@57905:       else if (tok.is_improper) improper += tok
wenzelm@57905:       else { content ++= improper; improper.clear; content += tok }
wenzelm@57905:     }
wenzelm@57905:     flush()
wenzelm@57905: 
wenzelm@57905:     result.toList
wenzelm@57905:   }
wenzelm@57905: 
wenzelm@57906:   def parse_spans(input: CharSequence): List[Command_Span.Span] =
wenzelm@57906:     parse_spans(scan(input))
wenzelm@57906: 
wenzelm@57905: 
wenzelm@58706:   /* overall document structure */
wenzelm@55616: 
wenzelm@58706:   def heading_level(command: Command): Option[Int] =
wenzelm@58706:   {
wenzelm@58868:     command.name match {
wenzelm@58868:       case "chapter" => Some(0)
wenzelm@58868:       case "section" | "header" => Some(1)
wenzelm@58868:       case "subsection" => Some(2)
wenzelm@58868:       case "subsubsection" => Some(3)
wenzelm@58868:       case _ =>
wenzelm@58901:         keywords.command_kind(command.name) match {
wenzelm@58938:           case Some(kind) if Keyword.theory(kind) && kind != Keyword.THY_END => Some(4)
wenzelm@58868:           case _ => None
wenzelm@58868:         }
wenzelm@58706:     }
wenzelm@58706:   }
wenzelm@58706: 
wenzelm@58743:   def parse_document(node_name: Document.Node.Name, text: CharSequence):
wenzelm@58743:     List[Outer_Syntax.Document] =
wenzelm@58706:   {
wenzelm@58706:     /* stack operations */
wenzelm@58706: 
wenzelm@58706:     def buffer(): mutable.ListBuffer[Outer_Syntax.Document] =
wenzelm@58706:       new mutable.ListBuffer[Outer_Syntax.Document]
wenzelm@58706: 
wenzelm@58747:     var stack: List[(Int, Command, mutable.ListBuffer[Outer_Syntax.Document])] =
wenzelm@58747:       List((0, Command.empty, buffer()))
wenzelm@55616: 
wenzelm@58706:     @tailrec def close(level: Int => Boolean)
wenzelm@58706:     {
wenzelm@58706:       stack match {
wenzelm@58747:         case (lev, command, body) :: (_, _, body2) :: rest if level(lev) =>
wenzelm@58747:           body2 += Outer_Syntax.Document_Block(command.name, command.source, body.toList)
wenzelm@58706:           stack = stack.tail
wenzelm@58706:           close(level)
wenzelm@58706:         case _ =>
wenzelm@58706:       }
wenzelm@58706:     }
wenzelm@58706: 
wenzelm@58743:     def result(): List[Outer_Syntax.Document] =
wenzelm@58706:     {
wenzelm@58706:       close(_ => true)
wenzelm@58743:       stack.head._3.toList
wenzelm@58706:     }
wenzelm@58706: 
wenzelm@58706:     def add(command: Command)
wenzelm@58706:     {
wenzelm@58706:       heading_level(command) match {
wenzelm@58706:         case Some(i) =>
wenzelm@58706:           close(_ > i)
wenzelm@58747:           stack = (i + 1, command, buffer()) :: stack
wenzelm@58706:         case None =>
wenzelm@58706:       }
wenzelm@58706:       stack.head._3 += Outer_Syntax.Document_Atom(command)
wenzelm@58706:     }
wenzelm@58706: 
wenzelm@58706: 
wenzelm@58706:     /* result structure */
wenzelm@58706: 
wenzelm@58706:     val spans = parse_spans(text)
wenzelm@58706:     spans.foreach(span => add(Command(Document_ID.none, node_name, Nil, span)))
wenzelm@58706:     result()
wenzelm@55616:   }
wenzelm@34166: }