src/Pure/Tools/build_job.scala
changeset 75393 87ebf5a50283
parent 74755 510296c0d8d1
child 75394 42267c650205
equal deleted inserted replaced
75388:b3ca4a6ed74b 75393:87ebf5a50283
    10 import java.util.HashMap
    10 import java.util.HashMap
    11 
    11 
    12 import scala.collection.mutable
    12 import scala.collection.mutable
    13 
    13 
    14 
    14 
    15 object Build_Job
    15 object Build_Job {
    16 {
       
    17   /* theory markup/messages from database */
    16   /* theory markup/messages from database */
    18 
    17 
    19   def read_theory(
    18   def read_theory(
    20     db_context: Sessions.Database_Context,
    19     db_context: Sessions.Database_Context,
    21     session_hierarchy: List[String],
    20     session_hierarchy: List[String],
    22     theory: String,
    21     theory: String,
    23     unicode_symbols: Boolean = false): Option[Command] =
    22     unicode_symbols: Boolean = false
    24   {
    23   ): Option[Command] = {
    25     def read(name: String): Export.Entry =
    24     def read(name: String): Export.Entry =
    26       db_context.get_export(session_hierarchy, theory, name)
    25       db_context.get_export(session_hierarchy, theory, name)
    27 
    26 
    28     def read_xml(name: String): XML.Body =
    27     def read_xml(name: String): XML.Body =
    29       YXML.parse_body(
    28       YXML.parse_body(
    38           Command.Results.make(
    37           Command.Results.make(
    39             for (elem @ XML.Elem(Markup(_, Markup.Serial(i)), _) <- read_xml(Export.MESSAGES))
    38             for (elem @ XML.Elem(Markup(_, Markup.Serial(i)), _) <- read_xml(Export.MESSAGES))
    40               yield i -> elem)
    39               yield i -> elem)
    41 
    40 
    42         val blobs =
    41         val blobs =
    43           blobs_files.map(file =>
    42           blobs_files.map(file => {
    44           {
       
    45             val path = Path.explode(file)
    43             val path = Path.explode(file)
    46             val name = Resources.file_node(path)
    44             val name = Resources.file_node(path)
    47             val src_path = File.relative_path(node_name.master_dir_path, path).getOrElse(path)
    45             val src_path = File.relative_path(node_name.master_dir_path, path).getOrElse(path)
    48             Command.Blob(name, src_path, None)
    46             Command.Blob(name, src_path, None)
    49           })
    47           })
    89     verbose: Boolean = false,
    87     verbose: Boolean = false,
    90     progress: Progress = new Progress,
    88     progress: Progress = new Progress,
    91     margin: Double = Pretty.default_margin,
    89     margin: Double = Pretty.default_margin,
    92     breakgain: Double = Pretty.default_breakgain,
    90     breakgain: Double = Pretty.default_breakgain,
    93     metric: Pretty.Metric = Symbol.Metric,
    91     metric: Pretty.Metric = Symbol.Metric,
    94     unicode_symbols: Boolean = false): Unit =
    92     unicode_symbols: Boolean = false
    95   {
    93   ): Unit = {
    96     val store = Sessions.store(options)
    94     val store = Sessions.store(options)
    97     val session = new Session(options, Resources.empty)
    95     val session = new Session(options, Resources.empty)
    98 
    96 
    99     using(store.open_database_context())(db_context =>
    97     using(store.open_database_context())(db_context => {
   100     {
       
   101       val result =
    98       val result =
   102         db_context.input_database(session_name)((db, _) =>
    99         db_context.input_database(session_name)((db, _) => {
   103         {
       
   104           val theories = store.read_theories(db, session_name)
   100           val theories = store.read_theories(db, session_name)
   105           val errors = store.read_errors(db, session_name)
   101           val errors = store.read_errors(db, session_name)
   106           store.read_build(db, session_name).map(info => (theories, errors, info.return_code))
   102           store.read_build(db, session_name).map(info => (theories, errors, info.return_code))
   107         })
   103         })
   108       result match {
   104       result match {
   152 
   148 
   153 
   149 
   154   /* Isabelle tool wrapper */
   150   /* Isabelle tool wrapper */
   155 
   151 
   156   val isabelle_tool = Isabelle_Tool("log", "print messages from build database",
   152   val isabelle_tool = Isabelle_Tool("log", "print messages from build database",
   157     Scala_Project.here, args =>
   153     Scala_Project.here, args => {
   158   {
       
   159     /* arguments */
   154     /* arguments */
   160 
   155 
   161     var unicode_symbols = false
   156     var unicode_symbols = false
   162     var theories: List[String] = Nil
   157     var theories: List[String] = Nil
   163     var margin = Pretty.default_margin
   158     var margin = Pretty.default_margin
   205   store: Sessions.Store,
   200   store: Sessions.Store,
   206   do_store: Boolean,
   201   do_store: Boolean,
   207   log: Logger,
   202   log: Logger,
   208   session_setup: (String, Session) => Unit,
   203   session_setup: (String, Session) => Unit,
   209   val numa_node: Option[Int],
   204   val numa_node: Option[Int],
   210   command_timings0: List[Properties.T])
   205   command_timings0: List[Properties.T]
   211 {
   206 ) {
   212   val options: Options = NUMA.policy_options(info.options, numa_node)
   207   val options: Options = NUMA.policy_options(info.options, numa_node)
   213 
   208 
   214   private val sessions_structure = deps.sessions_structure
   209   private val sessions_structure = deps.sessions_structure
   215 
   210 
   216   private val future_result: Future[Process_Result] =
   211   private val future_result: Future[Process_Result] =
   238         new Resources(sessions_structure, base, log = log, command_timings = command_timings0)
   233         new Resources(sessions_structure, base, log = log, command_timings = command_timings0)
   239       val session =
   234       val session =
   240         new Session(options, resources) {
   235         new Session(options, resources) {
   241           override val cache: Term.Cache = store.cache
   236           override val cache: Term.Cache = store.cache
   242 
   237 
   243           override def build_blobs_info(name: Document.Node.Name): Command.Blobs_Info =
   238           override def build_blobs_info(name: Document.Node.Name): Command.Blobs_Info = {
   244           {
       
   245             result_base.load_commands.get(name.expand) match {
   239             result_base.load_commands.get(name.expand) match {
   246               case Some(spans) =>
   240               case Some(spans) =>
   247                 val syntax = result_base.theory_syntax(name)
   241                 val syntax = result_base.theory_syntax(name)
   248                 Command.build_blobs_info(syntax, name, spans)
   242                 Command.build_blobs_info(syntax, name, spans)
   249               case None => Command.Blobs_Info.none
   243               case None => Command.Blobs_Info.none
   250             }
   244             }
   251           }
   245           }
   252         }
   246         }
   253 
   247 
   254       object Build_Session_Errors
   248       object Build_Session_Errors {
   255       {
       
   256         private val promise: Promise[List[String]] = Future.promise
   249         private val promise: Promise[List[String]] = Future.promise
   257 
   250 
   258         def result: Exn.Result[List[String]] = promise.join_result
   251         def result: Exn.Result[List[String]] = promise.join_result
   259         def cancel(): Unit = promise.cancel()
   252         def cancel(): Unit = promise.cancel()
   260         def apply(errs: List[String]): Unit =
   253         def apply(errs: List[String]): Unit = {
   261         {
       
   262           try { promise.fulfill(errs) }
   254           try { promise.fulfill(errs) }
   263           catch { case _: IllegalStateException => }
   255           catch { case _: IllegalStateException => }
   264         }
   256         }
   265       }
   257       }
   266 
   258 
   277       val task_statistics = new mutable.ListBuffer[Properties.T]
   269       val task_statistics = new mutable.ListBuffer[Properties.T]
   278 
   270 
   279       def fun(
   271       def fun(
   280         name: String,
   272         name: String,
   281         acc: mutable.ListBuffer[Properties.T],
   273         acc: mutable.ListBuffer[Properties.T],
   282         unapply: Properties.T => Option[Properties.T]): (String, Session.Protocol_Function) =
   274         unapply: Properties.T => Option[Properties.T]
   283       {
   275       ): (String, Session.Protocol_Function) = {
   284         name -> ((msg: Prover.Protocol_Output) =>
   276         name -> ((msg: Prover.Protocol_Output) =>
   285           unapply(msg.properties) match {
   277           unapply(msg.properties) match {
   286             case Some(props) => acc += props; true
   278             case Some(props) => acc += props; true
   287             case _ => false
   279             case _ => false
   288           })
   280           })
   289       }
   281       }
   290 
   282 
   291       session.init_protocol_handler(new Session.Protocol_Handler
   283       session.init_protocol_handler(new Session.Protocol_Handler {
   292         {
       
   293           override def exit(): Unit = Build_Session_Errors.cancel()
   284           override def exit(): Unit = Build_Session_Errors.cancel()
   294 
   285 
   295           private def build_session_finished(msg: Prover.Protocol_Output): Boolean =
   286           private def build_session_finished(msg: Prover.Protocol_Output): Boolean = {
   296           {
       
   297             val (rc, errors) =
   287             val (rc, errors) =
   298               try {
   288               try {
   299                 val (rc, errs) =
   289                 val (rc, errs) = {
   300                 {
       
   301                   import XML.Decode._
   290                   import XML.Decode._
   302                   pair(int, list(x => x))(Symbol.decode_yxml(msg.text))
   291                   pair(int, list(x => x))(Symbol.decode_yxml(msg.text))
   303                 }
   292                 }
   304                 val errors =
   293                 val errors =
   305                   for (err <- errs) yield {
   294                   for (err <- errs) yield {
   339               fun(Markup.Theory_Timing.name, theory_timings, Markup.Theory_Timing.unapply),
   328               fun(Markup.Theory_Timing.name, theory_timings, Markup.Theory_Timing.unapply),
   340               fun(Markup.Session_Timing.name, session_timings, Markup.Session_Timing.unapply),
   329               fun(Markup.Session_Timing.name, session_timings, Markup.Session_Timing.unapply),
   341               fun(Markup.Task_Statistics.name, task_statistics, Markup.Task_Statistics.unapply))
   330               fun(Markup.Task_Statistics.name, task_statistics, Markup.Task_Statistics.unapply))
   342         })
   331         })
   343 
   332 
   344       session.command_timings += Session.Consumer("command_timings")
   333       session.command_timings += Session.Consumer("command_timings") {
   345         {
   334         case Session.Command_Timing(props) =>
   346           case Session.Command_Timing(props) =>
   335           for {
   347             for {
   336             elapsed <- Markup.Elapsed.unapply(props)
   348               elapsed <- Markup.Elapsed.unapply(props)
   337             elapsed_time = Time.seconds(elapsed)
   349               elapsed_time = Time.seconds(elapsed)
   338             if elapsed_time.is_relevant && elapsed_time >= options.seconds("command_timing_threshold")
   350               if elapsed_time.is_relevant && elapsed_time >= options.seconds("command_timing_threshold")
   339           } command_timings += props.filter(Markup.command_timing_property)
   351             } command_timings += props.filter(Markup.command_timing_property)
   340       }
   352         }
   341 
   353 
   342       session.runtime_statistics += Session.Consumer("ML_statistics") {
   354       session.runtime_statistics += Session.Consumer("ML_statistics")
   343         case Session.Runtime_Statistics(props) => runtime_statistics += props
   355         {
   344       }
   356           case Session.Runtime_Statistics(props) => runtime_statistics += props
   345 
   357         }
   346       session.finished_theories += Session.Consumer[Document.Snapshot]("finished_theories") {
   358 
   347         case snapshot =>
   359       session.finished_theories += Session.Consumer[Document.Snapshot]("finished_theories")
   348           if (!progress.stopped) {
   360         {
   349             val rendering = new Rendering(snapshot, options, session)
   361           case snapshot =>
   350 
   362             if (!progress.stopped) {
   351             def export_(name: String, xml: XML.Body, compress: Boolean = true): Unit = {
   363               val rendering = new Rendering(snapshot, options, session)
   352               if (!progress.stopped) {
   364 
   353                 val theory_name = snapshot.node_name.theory
   365               def export_(name: String, xml: XML.Body, compress: Boolean = true): Unit =
   354                 val args =
   366               {
   355                   Protocol.Export.Args(theory_name = theory_name, name = name, compress = compress)
   367                 if (!progress.stopped) {
   356                 val bytes = Bytes(Symbol.encode(YXML.string_of_body(xml)))
   368                   val theory_name = snapshot.node_name.theory
   357                 if (!bytes.is_empty) export_consumer(session_name, args, bytes)
   369                   val args =
       
   370                     Protocol.Export.Args(theory_name = theory_name, name = name, compress = compress)
       
   371                   val bytes = Bytes(Symbol.encode(YXML.string_of_body(xml)))
       
   372                   if (!bytes.is_empty) export_consumer(session_name, args, bytes)
       
   373                 }
       
   374               }
   358               }
   375               def export_text(name: String, text: String, compress: Boolean = true): Unit =
   359             }
   376                 export_(name, List(XML.Text(text)), compress = compress)
   360             def export_text(name: String, text: String, compress: Boolean = true): Unit =
   377 
   361               export_(name, List(XML.Text(text)), compress = compress)
   378               for (command <- snapshot.snippet_command) {
   362 
   379                 export_text(Export.DOCUMENT_ID, command.id.toString, compress = false)
   363             for (command <- snapshot.snippet_command) {
   380               }
   364               export_text(Export.DOCUMENT_ID, command.id.toString, compress = false)
   381 
   365             }
   382               export_text(Export.FILES,
   366 
   383                 cat_lines(snapshot.node_files.map(_.symbolic.node)), compress = false)
   367             export_text(Export.FILES,
   384 
   368               cat_lines(snapshot.node_files.map(_.symbolic.node)), compress = false)
   385               for (((_, xml), i) <- snapshot.xml_markup_blobs().zipWithIndex) {
   369 
   386                 export_(Export.MARKUP + (i + 1), xml)
   370             for (((_, xml), i) <- snapshot.xml_markup_blobs().zipWithIndex) {
   387               }
   371               export_(Export.MARKUP + (i + 1), xml)
   388               export_(Export.MARKUP, snapshot.xml_markup())
   372             }
   389               export_(Export.MESSAGES, snapshot.messages.map(_._1))
   373             export_(Export.MARKUP, snapshot.xml_markup())
   390 
   374             export_(Export.MESSAGES, snapshot.messages.map(_._1))
   391               val citations = Library.distinct(rendering.citations(Text.Range.full).map(_.info))
   375 
   392               export_text(Export.DOCUMENT_CITATIONS, cat_lines(citations))
   376             val citations = Library.distinct(rendering.citations(Text.Range.full).map(_.info))
   393             }
   377             export_text(Export.DOCUMENT_CITATIONS, cat_lines(citations))
   394         }
   378           }
   395 
   379       }
   396       session.all_messages += Session.Consumer[Any]("build_session_output")
   380 
   397         {
   381       session.all_messages += Session.Consumer[Any]("build_session_output") {
   398           case msg: Prover.Output =>
   382         case msg: Prover.Output =>
   399             val message = msg.message
   383           val message = msg.message
   400             if (msg.is_system) resources.log(Protocol.message_text(message))
   384           if (msg.is_system) resources.log(Protocol.message_text(message))
   401 
   385 
   402             if (msg.is_stdout) {
   386           if (msg.is_stdout) {
   403               stdout ++= Symbol.encode(XML.content(message))
   387             stdout ++= Symbol.encode(XML.content(message))
   404             }
   388           }
   405             else if (msg.is_stderr) {
   389           else if (msg.is_stderr) {
   406               stderr ++= Symbol.encode(XML.content(message))
   390             stderr ++= Symbol.encode(XML.content(message))
   407             }
   391           }
   408             else if (msg.is_exit) {
   392           else if (msg.is_exit) {
   409               val err =
   393             val err =
   410                 "Prover terminated" +
   394               "Prover terminated" +
   411                   (msg.properties match {
   395                 (msg.properties match {
   412                     case Markup.Process_Result(result) => ": " + result.print_rc
   396                   case Markup.Process_Result(result) => ": " + result.print_rc
   413                     case _ => ""
   397                   case _ => ""
   414                   })
   398                 })
   415               Build_Session_Errors(List(err))
   399             Build_Session_Errors(List(err))
   416             }
   400           }
   417           case _ =>
   401         case _ =>
   418         }
   402       }
   419 
   403 
   420       session_setup(session_name, session)
   404       session_setup(session_name, session)
   421 
   405 
   422       val eval_main = Command_Line.ML_tool("Isabelle_Process.init_build ()" :: eval_store)
   406       val eval_main = Command_Line.ML_tool("Isabelle_Process.init_build ()" :: eval_store)
   423 
   407 
   456         export_consumer.shutdown(close = true).map(Output.error_message_text)
   440         export_consumer.shutdown(close = true).map(Output.error_message_text)
   457 
   441 
   458       val (document_output, document_errors) =
   442       val (document_output, document_errors) =
   459         try {
   443         try {
   460           if (build_errors.isInstanceOf[Exn.Res[_]] && process_result.ok && info.documents.nonEmpty) {
   444           if (build_errors.isInstanceOf[Exn.Res[_]] && process_result.ok && info.documents.nonEmpty) {
   461             using(store.open_database_context())(db_context =>
   445             using(store.open_database_context())(db_context => {
   462               {
       
   463                 val documents =
   446                 val documents =
   464                   Document_Build.build_documents(
   447                   Document_Build.build_documents(
   465                     Document_Build.context(session_name, deps, db_context, progress = progress),
   448                     Document_Build.context(session_name, deps, db_context, progress = progress),
   466                     output_sources = info.document_output,
   449                     output_sources = info.document_output,
   467                     output_pdf = info.document_output)
   450                     output_pdf = info.document_output)
   475         catch {
   458         catch {
   476           case exn: Document_Build.Build_Error => (exn.log_lines, List(exn.message))
   459           case exn: Document_Build.Build_Error => (exn.log_lines, List(exn.message))
   477           case Exn.Interrupt.ERROR(msg) => (Nil, List(msg))
   460           case Exn.Interrupt.ERROR(msg) => (Nil, List(msg))
   478         }
   461         }
   479 
   462 
   480       val result =
   463       val result = {
   481       {
       
   482         val theory_timing =
   464         val theory_timing =
   483           theory_timings.iterator.map(
   465           theory_timings.iterator.map({ case props @ Markup.Name(name) => name -> props }).toMap
   484             { case props @ Markup.Name(name) => name -> props }).toMap
       
   485         val used_theory_timings =
   466         val used_theory_timings =
   486           for { (name, _) <- deps(session_name).used_theories }
   467           for { (name, _) <- deps(session_name).used_theories }
   487             yield theory_timing.getOrElse(name.theory, Markup.Name(name.theory))
   468             yield theory_timing.getOrElse(name.theory, Markup.Name(name.theory))
   488 
   469 
   489         val more_output =
   470         val more_output =
   518     }
   499     }
   519 
   500 
   520   def terminate(): Unit = future_result.cancel()
   501   def terminate(): Unit = future_result.cancel()
   521   def is_finished: Boolean = future_result.is_finished
   502   def is_finished: Boolean = future_result.is_finished
   522 
   503 
   523   private val timeout_request: Option[Event_Timer.Request] =
   504   private val timeout_request: Option[Event_Timer.Request] = {
   524   {
       
   525     if (info.timeout_ignored) None
   505     if (info.timeout_ignored) None
   526     else Some(Event_Timer.request(Time.now() + info.timeout) { terminate() })
   506     else Some(Event_Timer.request(Time.now() + info.timeout) { terminate() })
   527   }
   507   }
   528 
   508 
   529   def join: (Process_Result, Option[String]) =
   509   def join: (Process_Result, Option[String]) = {
   530   {
       
   531     val result1 = future_result.join
   510     val result1 = future_result.join
   532 
   511 
   533     val was_timeout =
   512     val was_timeout =
   534       timeout_request match {
   513       timeout_request match {
   535         case None => false
   514         case None => false