--- a/src/Pure/Tools/build.scala Sat Feb 11 23:24:57 2023 +0100
+++ b/src/Pure/Tools/build.scala Sun Feb 12 13:45:06 2023 +0100
@@ -8,10 +8,6 @@
package isabelle
-import scala.collection.immutable.SortedSet
-import scala.annotation.tailrec
-
-
object Build {
/** auxiliary **/
@@ -28,31 +24,6 @@
}
- /* queue with scheduling information */
-
- private object Queue {
- def apply(build_context: Build_Process.Context): Queue = {
- val build_graph = build_context.sessions_structure.build_graph
- val build_order = SortedSet.from(build_graph.keys)(build_context.ordering)
- new Queue(build_graph, build_order)
- }
- }
-
- private class Queue(
- build_graph: Graph[String, Sessions.Info],
- build_order: SortedSet[String]
- ) {
- def is_empty: Boolean = build_graph.is_empty
-
- def - (name: String): Queue =
- new Queue(build_graph.del_node(name), build_order - name)
-
- def dequeue(skip: String => Boolean): Option[String] =
- build_order.iterator.dropWhile(name => skip(name) || !build_graph.is_minimal(name))
- .nextOption()
- }
-
-
/** build with results **/
@@ -76,16 +47,6 @@
override def toString: String = rc.toString
}
- def session_finished(session_name: String, process_result: Process_Result): String =
- "Finished " + session_name + " (" + process_result.timing.message_resources + ")"
-
- def session_timing(session_name: String, build_log: Build_Log.Session_Info): String = {
- val props = build_log.session_timing
- val threads = Markup.Session_Timing.Threads.unapply(props) getOrElse 1
- val timing = Markup.Timing_Properties.get(props)
- "Timing " + session_name + " (" + threads + " threads, " + timing.message_factor + ")"
- }
-
def build(
options: Options,
selection: Sessions.Selection = Sessions.Selection.empty,
@@ -170,10 +131,9 @@
}
- /* main build process */
+ /* build process and results */
- val build_context =
- Build_Process.Context(build_deps.sessions_structure, store, progress = progress)
+ val build_context = Build_Process.Context(store, build_deps, progress = progress)
store.prepare_output_dir()
@@ -187,188 +147,19 @@
}
}
- // scheduler loop
- case class Result(
- current: Boolean,
- output_heap: SHA1.Shasum,
- process_result: Process_Result
- ) {
- def ok: Boolean = process_result.ok
- }
-
- def sleep(): Unit =
- Isabelle_Thread.interrupt_handler(_ => progress.stop()) {
- build_options.seconds("editor_input_delay").sleep()
- }
-
- val log =
- build_options.string("system_log") match {
- case "" => No_Logger
- case "-" => Logger.make(progress)
- case log_file => Logger.make(Some(Path.explode(log_file)))
- }
-
- val numa_nodes = new NUMA.Nodes(numa_shuffling)
-
- @tailrec def loop(
- pending: Queue,
- running: Map[String, (SHA1.Shasum, Build_Job)],
- results: Map[String, Result]
- ): Map[String, Result] = {
- def used_node(i: Int): Boolean =
- running.iterator.exists(
- { case (_, (_, job)) => job.numa_node.isDefined && job.numa_node.get == i })
-
- if (pending.is_empty) results
- else {
- if (progress.stopped) {
- for ((_, (_, job)) <- running) job.terminate()
- }
-
- running.find({ case (_, (_, job)) => job.is_finished }) match {
- case Some((session_name, (input_heaps, job))) =>
- //{{{ finish job
-
- val process_result = job.join
-
- val output_heap =
- if (process_result.ok && job.do_store && store.output_heap(session_name).is_file) {
- SHA1.shasum(ML_Heap.write_digest(store.output_heap(session_name)), session_name)
- }
- else SHA1.no_shasum
-
- val log_lines = process_result.out_lines.filterNot(Protocol_Message.Marker.test)
- val process_result_tail = {
- val tail = job.info.options.int("process_output_tail")
- process_result.copy(
- out_lines =
- "(more details via \"isabelle log -H Error " + session_name + "\")" ::
- (if (tail == 0) log_lines else log_lines.drop(log_lines.length - tail max 0)))
- }
-
- val build_log =
- Build_Log.Log_File(session_name, process_result.out_lines).
- parse_session_info(
- command_timings = true,
- theory_timings = true,
- ml_statistics = true,
- task_statistics = true)
-
- // write log file
- if (process_result.ok) {
- File.write_gzip(store.output_log_gz(session_name), terminate_lines(log_lines))
- }
- else File.write(store.output_log(session_name), terminate_lines(log_lines))
-
- // write database
- using(store.open_database(session_name, output = true))(db =>
- store.write_session_info(db, session_name, job.session_sources,
- build_log =
- if (process_result.timeout) build_log.error("Timeout") else build_log,
- build =
- Session_Info(build_deps.sources_shasum(session_name), input_heaps, output_heap,
- process_result.rc, UUID.random().toString)))
-
- // messages
- process_result.err_lines.foreach(progress.echo)
-
- if (process_result.ok) {
- if (verbose) progress.echo(session_timing(session_name, build_log))
- progress.echo(session_finished(session_name, process_result))
- }
- else {
- progress.echo(session_name + " FAILED")
- if (!process_result.interrupted) progress.echo(process_result_tail.out)
- }
-
- loop(pending - session_name, running - session_name,
- results + (session_name -> Result(false, output_heap, process_result_tail)))
- //}}}
- case None if running.size < (max_jobs max 1) =>
- //{{{ check/start next job
- pending.dequeue(running.isDefinedAt) match {
- case Some(session_name) =>
- val ancestor_results =
- build_deps.sessions_structure.build_requirements(List(session_name)).
- filterNot(_ == session_name).map(results(_))
- val input_heaps =
- if (ancestor_results.isEmpty) {
- SHA1.shasum_meta_info(SHA1.digest(Path.explode("$POLYML_EXE")))
- }
- else SHA1.flat_shasum(ancestor_results.map(_.output_heap))
-
- val do_store = build_heap || build_context.build_heap(session_name)
- val (current, output_heap) = {
- store.try_open_database(session_name) match {
- case Some(db) =>
- using(db)(store.read_build(_, session_name)) match {
- case Some(build) =>
- val output_heap = store.find_heap_shasum(session_name)
- val current =
- !fresh_build &&
- build.ok &&
- build.sources == build_deps.sources_shasum(session_name) &&
- build.input_heaps == input_heaps &&
- build.output_heap == output_heap &&
- !(do_store && output_heap.is_empty)
- (current, output_heap)
- case None => (false, SHA1.no_shasum)
- }
- case None => (false, SHA1.no_shasum)
- }
- }
- val all_current = current && ancestor_results.forall(_.current)
-
- if (all_current) {
- loop(pending - session_name, running,
- results + (session_name -> Result(true, output_heap, Process_Result.ok)))
- }
- else if (no_build) {
- progress.echo_if(verbose, "Skipping " + session_name + " ...")
- loop(pending - session_name, running,
- results + (session_name -> Result(false, output_heap, Process_Result.error)))
- }
- else if (ancestor_results.forall(_.ok) && !progress.stopped) {
- progress.echo((if (do_store) "Building " else "Running ") + session_name + " ...")
-
- store.clean_output(session_name)
- using(store.open_database(session_name, output = true))(
- store.init_session_info(_, session_name))
-
- val session_background = build_deps.background(session_name)
- val resources =
- new Resources(session_background, log = log,
- command_timings = build_context(session_name).old_command_timings)
-
- val numa_node = numa_nodes.next(used_node)
- val job =
- new Build_Job(progress, session_background, store, do_store,
- resources, session_setup, numa_node)
- loop(pending, running + (session_name -> (input_heaps, job)), results)
- }
- else {
- progress.echo(session_name + " CANCELLED")
- loop(pending - session_name, running,
- results + (session_name -> Result(false, output_heap, Process_Result.undefined)))
- }
- case None => sleep(); loop(pending, running, results)
- }
- ///}}}
- case None => sleep(); loop(pending, running, results)
- }
- }
- }
-
-
- /* build results */
-
val results = {
val build_results =
if (build_deps.is_empty) {
progress.echo_warning("Nothing to build")
- Map.empty[String, Result]
+ Map.empty[String, Build_Process.Result]
}
- else Isabelle_Thread.uninterruptible { loop(Queue(build_context), Map.empty, Map.empty) }
+ else {
+ Isabelle_Thread.uninterruptible {
+ Build_Process.main(build_context, build_heap = build_heap,
+ numa_shuffling = numa_shuffling, max_jobs = max_jobs, fresh_build = fresh_build,
+ no_build = no_build, verbose = verbose, session_setup = session_setup)
+ }
+ }
val sessions_ok: List[String] =
(for {