author | wenzelm |
Sat, 18 Mar 2017 16:15:37 +0100 | |
changeset 65307 | c1ba192b4f96 |
parent 65296 | a71db30f3b2d |
child 65308 | 8f58102afa22 |
permissions | -rw-r--r-- |
50686 | 1 |
/* Title: Pure/Tools/build.scala |
48276 | 2 |
Author: Makarius |
57923 | 3 |
Options: :folding=explicit: |
48276 | 4 |
|
5 |
Build and manage Isabelle sessions. |
|
6 |
*/ |
|
7 |
||
8 |
package isabelle |
|
9 |
||
10 |
||
48548 | 11 |
import java.io.{BufferedInputStream, FileInputStream, |
48494 | 12 |
BufferedReader, InputStreamReader, IOException} |
13 |
import java.util.zip.GZIPInputStream |
|
48335 | 14 |
|
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
15 |
import scala.collection.SortedSet |
51397
03b586ee5930
support for 'chapter' specifications within session ROOT;
wenzelm
parents:
51300
diff
changeset
|
16 |
import scala.collection.mutable |
48340
6f4fc030882a
allow explicit specification of additional session directories;
wenzelm
parents:
48339
diff
changeset
|
17 |
import scala.annotation.tailrec |
48337
9c7f8e5805b4
cumulate semantic Session_Info, based on syntactic Session_Entry;
wenzelm
parents:
48336
diff
changeset
|
18 |
|
48335 | 19 |
|
48276 | 20 |
object Build |
21 |
{ |
|
62631 | 22 |
/** auxiliary **/ |
48424 | 23 |
|
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
24 |
/* persistent build info */ |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
25 |
|
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
26 |
sealed case class Session_Info( |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
27 |
sources: List[String], |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
28 |
input_heaps: List[String], |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
29 |
output_heap: Option[String], |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
30 |
return_code: Int) |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
31 |
|
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
32 |
|
65289 | 33 |
/* queue with scheduling information */ |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
34 |
|
62631 | 35 |
private object Queue |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
36 |
{ |
65289 | 37 |
def load_timings(store: Sessions.Store, name: String): (List[Properties.T], Double) = |
38 |
{ |
|
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
39 |
val no_timings: (List[Properties.T], Double) = (Nil, 0.0) |
65289 | 40 |
|
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
41 |
store.find_database(name) match { |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
42 |
case None => no_timings |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
43 |
case Some(database) => |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
44 |
def ignore_error(msg: String) = |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
45 |
{ |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
46 |
Output.warning("Ignoring bad database: " + database + (if (msg == "") "" else "\n" + msg)) |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
47 |
no_timings |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
48 |
} |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
49 |
try { |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
50 |
using(SQLite.open_database(database))(db => |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
51 |
{ |
65296 | 52 |
val build_log = store.read_build_log(db, name, command_timings = true) |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
53 |
val session_timing = Markup.Elapsed.unapply(build_log.session_timing) getOrElse 0.0 |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
54 |
(build_log.command_timings, session_timing) |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
55 |
}) |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
56 |
} |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
57 |
catch { |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
58 |
case ERROR(msg) => ignore_error(msg) |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
59 |
case exn: java.lang.Error => ignore_error(Exn.message(exn)) |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
60 |
case _: XML.Error => ignore_error("") |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
61 |
} |
65289 | 62 |
} |
63 |
} |
|
64 |
||
65 |
def apply(tree: Sessions.Tree, store: Sessions.Store): Queue = |
|
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
66 |
{ |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
67 |
val graph = tree.graph |
56372
fadb0fef09d7
more explicit iterator terminology, in accordance to Scala 2.8 library;
wenzelm
parents:
56208
diff
changeset
|
68 |
val sessions = graph.keys |
51220 | 69 |
|
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
70 |
val timings = sessions.map(name => (name, load_timings(store, name))) |
51220 | 71 |
val command_timings = |
72 |
Map(timings.map({ case (name, (ts, _)) => (name, ts) }): _*).withDefaultValue(Nil) |
|
73 |
val session_timing = |
|
74 |
Map(timings.map({ case (name, (_, t)) => (name, t) }): _*).withDefaultValue(0.0) |
|
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
75 |
|
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
76 |
def outdegree(name: String): Int = graph.imm_succs(name).size |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
77 |
|
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
78 |
object Ordering extends scala.math.Ordering[String] |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
79 |
{ |
51227
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
80 |
def compare_timing(name1: String, name2: String): Int = |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
81 |
{ |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
82 |
val t1 = session_timing(name1) |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
83 |
val t2 = session_timing(name2) |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
84 |
if (t1 == 0.0 || t2 == 0.0) 0 |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
85 |
else t1 compare t2 |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
86 |
} |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
87 |
|
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
88 |
def compare(name1: String, name2: String): Int = |
51229
6e40d0bb89e3
prefer outdegree in comparison again (cf. 88c96e836ed6) -- NB: big jobs might hide behind small ones in this naive queuing scheme;
wenzelm
parents:
51227
diff
changeset
|
89 |
outdegree(name2) compare outdegree(name1) match { |
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
90 |
case 0 => |
51229
6e40d0bb89e3
prefer outdegree in comparison again (cf. 88c96e836ed6) -- NB: big jobs might hide behind small ones in this naive queuing scheme;
wenzelm
parents:
51227
diff
changeset
|
91 |
compare_timing(name2, name1) match { |
51220 | 92 |
case 0 => |
61602 | 93 |
tree(name2).timeout compare tree(name1).timeout match { |
51220 | 94 |
case 0 => name1 compare name2 |
95 |
case ord => ord |
|
96 |
} |
|
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
97 |
case ord => ord |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
98 |
} |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
99 |
case ord => ord |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
100 |
} |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
101 |
} |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
102 |
|
51220 | 103 |
new Queue(graph, SortedSet(sessions: _*)(Ordering), command_timings) |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
104 |
} |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
105 |
} |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
106 |
|
62631 | 107 |
private final class Queue private( |
108 |
graph: Graph[String, Sessions.Info], |
|
51220 | 109 |
order: SortedSet[String], |
110 |
val command_timings: String => List[Properties.T]) |
|
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
111 |
{ |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
112 |
def is_inner(name: String): Boolean = !graph.is_maximal(name) |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
113 |
|
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
114 |
def is_empty: Boolean = graph.is_empty |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
115 |
|
51227
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
116 |
def - (name: String): Queue = |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
117 |
new Queue(graph.del_node(name), |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
118 |
order - name, // FIXME scala-2.10.0 TreeSet problem!? |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
119 |
command_timings) |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
120 |
|
62631 | 121 |
def dequeue(skip: String => Boolean): Option[(String, Sessions.Info)] = |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
122 |
{ |
51227
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
123 |
val it = order.iterator.dropWhile(name => |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
124 |
skip(name) |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
125 |
|| !graph.defined(name) // FIXME scala-2.10.0 TreeSet problem!? |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
126 |
|| !graph.is_minimal(name)) |
48680 | 127 |
if (it.hasNext) { val name = it.next; Some((name, graph.get_node(name))) } |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
128 |
else None |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
129 |
} |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
130 |
} |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
131 |
|
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
132 |
|
48424 | 133 |
/* jobs */ |
48341 | 134 |
|
62637
0189fe0f6452
support for Poly/ML heap hierarchy, which saves a lot of disk space;
wenzelm
parents:
62636
diff
changeset
|
135 |
private class Job(progress: Progress, name: String, val info: Sessions.Info, tree: Sessions.Tree, |
64265 | 136 |
store: Sessions.Store, do_output: Boolean, verbose: Boolean, val numa_node: Option[Int], |
62633 | 137 |
session_graph: Graph_Display.Graph, command_timings: List[Properties.T]) |
48418 | 138 |
{ |
62633 | 139 |
val output = store.output_dir + Path.basic(name) |
48674 | 140 |
def output_path: Option[Path] = if (do_output) Some(output) else None |
62630 | 141 |
def output_save_state: String = |
62825
e6e80a8bf624
structure PolyML is sealed after bootstrap: all ML system access is managed by Isabelle;
wenzelm
parents:
62714
diff
changeset
|
142 |
if (do_output) "ML_Heap.save_child " + ML_Syntax.print_string0(File.platform_path(output)) |
62630 | 143 |
else "" |
62633 | 144 |
output.file.delete |
48674 | 145 |
|
146 |
private val parent = info.parent.getOrElse("") |
|
48418 | 147 |
|
59445 | 148 |
private val graph_file = Isabelle_System.tmp_file("session_graph", "pdf") |
149 |
try { isabelle.graphview.Graph_File.write(info.options, graph_file, session_graph) } |
|
150 |
catch { case ERROR(_) => /*error should be exposed in ML*/ } |
|
151 |
||
62610
4c89504c76fb
more uniform signature for various process invocations;
wenzelm
parents:
62603
diff
changeset
|
152 |
private val env = |
4c89504c76fb
more uniform signature for various process invocations;
wenzelm
parents:
62603
diff
changeset
|
153 |
Isabelle_System.settings() + |
4c89504c76fb
more uniform signature for various process invocations;
wenzelm
parents:
62603
diff
changeset
|
154 |
("ISABELLE_ML_DEBUGGER" -> info.options.bool("ML_debugger").toString) |
48698
2585042b1a30
pass Isabelle/Scala system options into ML process of Isar tty or build jobs;
wenzelm
parents:
48684
diff
changeset
|
155 |
|
62573 | 156 |
private val future_result: Future[Process_Result] = |
61559
313eca3fa847
more direct task future implementation, with proper cancel operation;
wenzelm
parents:
61556
diff
changeset
|
157 |
Future.thread("build") { |
62944
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
158 |
val args_file = Isabelle_System.tmp_file("build") |
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
159 |
File.write(args_file, YXML.string_of_body( |
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
160 |
{ |
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
161 |
val theories = info.theories.map(x => (x._2, x._3)) |
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
162 |
import XML.Encode._ |
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
163 |
pair(list(pair(string, int)), pair(list(properties), pair(bool, pair(bool, |
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
164 |
pair(Path.encode, pair(list(pair(Path.encode, Path.encode)), pair(string, |
65307 | 165 |
pair(string, pair(string, pair(string, pair(Path.encode, |
166 |
list(pair(Options.encode, list(Path.encode))))))))))))))( |
|
62944
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
167 |
(Symbol.codes, (command_timings, (do_output, (verbose, |
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
168 |
(store.browser_info, (info.document_files, (File.standard_path(graph_file), |
65307 | 169 |
(parent, (info.chapter, (name, (Path.current, |
170 |
theories)))))))))))) |
|
62944
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
171 |
})) |
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
172 |
|
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
173 |
val eval = |
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
174 |
"Command_Line.tool0 (fn () => (" + |
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
175 |
"Build.build " + ML_Syntax.print_string0(File.standard_path(args_file)) + |
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
176 |
(if (do_output) "; ML_Heap.share_common_data (); " + output_save_state |
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
177 |
else "") + "));" |
64265 | 178 |
|
179 |
val process_options = |
|
180 |
numa_node match { |
|
181 |
case None => info.options |
|
182 |
case Some(n) => info.options.string("ML_process_policy") = NUMA.policy(n) |
|
183 |
} |
|
62573 | 184 |
val process = |
62883 | 185 |
if (Sessions.pure_name(name)) { |
64265 | 186 |
ML_Process(process_options, raw_ml_system = true, cwd = info.dir.file, |
62946 | 187 |
args = |
188 |
(for ((root, _) <- Thy_Header.ml_roots) yield List("--use", root)).flatten ::: |
|
189 |
List("--eval", eval), |
|
62944
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
190 |
env = env, tree = Some(tree), store = store, cleanup = () => args_file.delete) |
62573 | 191 |
} |
192 |
else { |
|
64265 | 193 |
ML_Process(process_options, parent, List("--eval", eval), cwd = info.dir.file, |
62637
0189fe0f6452
support for Poly/ML heap hierarchy, which saves a lot of disk space;
wenzelm
parents:
62636
diff
changeset
|
194 |
env = env, tree = Some(tree), store = store, cleanup = () => args_file.delete) |
62573 | 195 |
} |
64265 | 196 |
|
62573 | 197 |
process.result( |
51962
016cb7d8f297
limit build process output, to avoid bombing Isabelle/Scala process by ill-behaved jobs (e.g. Containers in AFP/9025435b29cf);
wenzelm
parents:
51663
diff
changeset
|
198 |
progress_stdout = (line: String) => |
50847 | 199 |
Library.try_unprefix("\floading_theory = ", line) match { |
200 |
case Some(theory) => progress.theory(name, theory) |
|
201 |
case None => |
|
51962
016cb7d8f297
limit build process output, to avoid bombing Isabelle/Scala process by ill-behaved jobs (e.g. Containers in AFP/9025435b29cf);
wenzelm
parents:
51663
diff
changeset
|
202 |
}, |
016cb7d8f297
limit build process output, to avoid bombing Isabelle/Scala process by ill-behaved jobs (e.g. Containers in AFP/9025435b29cf);
wenzelm
parents:
51663
diff
changeset
|
203 |
progress_limit = |
016cb7d8f297
limit build process output, to avoid bombing Isabelle/Scala process by ill-behaved jobs (e.g. Containers in AFP/9025435b29cf);
wenzelm
parents:
51663
diff
changeset
|
204 |
info.options.int("process_output_limit") match { |
016cb7d8f297
limit build process output, to avoid bombing Isabelle/Scala process by ill-behaved jobs (e.g. Containers in AFP/9025435b29cf);
wenzelm
parents:
51663
diff
changeset
|
205 |
case 0 => None |
016cb7d8f297
limit build process output, to avoid bombing Isabelle/Scala process by ill-behaved jobs (e.g. Containers in AFP/9025435b29cf);
wenzelm
parents:
51663
diff
changeset
|
206 |
case m => Some(m * 1000000L) |
56871
d06ff36b4fa7
expose interrupts more like ML version, but not in managed bash processes of Build;
wenzelm
parents:
56861
diff
changeset
|
207 |
}, |
d06ff36b4fa7
expose interrupts more like ML version, but not in managed bash processes of Build;
wenzelm
parents:
56861
diff
changeset
|
208 |
strict = false) |
50845 | 209 |
} |
48674 | 210 |
|
62572 | 211 |
def terminate: Unit = future_result.cancel |
212 |
def is_finished: Boolean = future_result.is_finished |
|
48674 | 213 |
|
62569 | 214 |
@volatile private var was_timeout = false |
56779 | 215 |
private val timeout_request: Option[Event_Timer.Request] = |
216 |
{ |
|
62569 | 217 |
if (info.timeout > Time.zero) |
218 |
Some(Event_Timer.request(Time.now() + info.timeout) { terminate; was_timeout = true }) |
|
48674 | 219 |
else None |
56779 | 220 |
} |
48674 | 221 |
|
62400 | 222 |
def join: Process_Result = |
50845 | 223 |
{ |
62572 | 224 |
val result = future_result.join |
50845 | 225 |
|
62944
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
226 |
if (result.ok) |
62633 | 227 |
Present.finish(progress, store.browser_info, graph_file, info, name) |
61372 | 228 |
|
59445 | 229 |
graph_file.delete |
56779 | 230 |
timeout_request.foreach(_.cancel) |
48674 | 231 |
|
62572 | 232 |
if (result.interrupted) { |
233 |
if (was_timeout) result.error(Output.error_text("Timeout")).was_timeout |
|
234 |
else result.error(Output.error_text("Interrupt")) |
|
52063
fd533ac64390
timeout counts as regular error, with rc = 1 (cf. special Exn.Interrupt vs. regular TimeLimit.TimeOut in Isabelle/ML);
wenzelm
parents:
51987
diff
changeset
|
235 |
} |
62572 | 236 |
else result |
48674 | 237 |
} |
48364 | 238 |
} |
239 |
||
48424 | 240 |
|
62631 | 241 |
|
62641 | 242 |
/** build with results **/ |
48424 | 243 |
|
63996 | 244 |
class Results private[Build](results: Map[String, (Option[Process_Result], Sessions.Info)]) |
62403 | 245 |
{ |
246 |
def sessions: Set[String] = results.keySet |
|
63082
6af03422535a
expose Sessions.Info in Build.Results
Lars Hupel <lars.hupel@mytum.de>
parents:
62946
diff
changeset
|
247 |
def cancelled(name: String): Boolean = results(name)._1.isEmpty |
6af03422535a
expose Sessions.Info in Build.Results
Lars Hupel <lars.hupel@mytum.de>
parents:
62946
diff
changeset
|
248 |
def apply(name: String): Process_Result = results(name)._1.getOrElse(Process_Result(1)) |
6af03422535a
expose Sessions.Info in Build.Results
Lars Hupel <lars.hupel@mytum.de>
parents:
62946
diff
changeset
|
249 |
def info(name: String): Sessions.Info = results(name)._2 |
65253 | 250 |
val rc = |
251 |
(0 /: results.iterator.map( |
|
252 |
{ case (_, (Some(r), _)) => r.rc case (_, (None, _)) => 1 }))(_ max _) |
|
62641 | 253 |
def ok: Boolean = rc == 0 |
62406 | 254 |
|
255 |
override def toString: String = rc.toString |
|
62403 | 256 |
} |
257 |
||
62641 | 258 |
def build( |
50404
898cac1dad5e
avoid startup within GUI thread -- it is only required later for dialog;
wenzelm
parents:
50367
diff
changeset
|
259 |
options: Options, |
64909 | 260 |
progress: Progress = No_Progress, |
48511
37999ee01156
remove old output heaps, to ensure that result is valid wrt. check_stamps;
wenzelm
parents:
48509
diff
changeset
|
261 |
build_heap: Boolean = false, |
48595 | 262 |
clean_build: Boolean = false, |
56890 | 263 |
dirs: List[Path] = Nil, |
264 |
select_dirs: List[Path] = Nil, |
|
64265 | 265 |
numa_shuffling: Boolean = false, |
48509 | 266 |
max_jobs: Int = 1, |
48903 | 267 |
list_files: Boolean = false, |
59891
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
268 |
check_keywords: Set[String] = Set.empty, |
48509 | 269 |
no_build: Boolean = false, |
270 |
system_mode: Boolean = false, |
|
271 |
verbose: Boolean = false, |
|
63224 | 272 |
requirements: Boolean = false, |
273 |
all_sessions: Boolean = false, |
|
274 |
exclude_session_groups: List[String] = Nil, |
|
59892
2a616319c171
added isabelle build option -x, to exclude sessions;
wenzelm
parents:
59891
diff
changeset
|
275 |
exclude_sessions: List[String] = Nil, |
63224 | 276 |
session_groups: List[String] = Nil, |
62641 | 277 |
sessions: List[String] = Nil): Results = |
48341 | 278 |
{ |
63224 | 279 |
build_selection( |
280 |
options = options, |
|
281 |
progress = progress, |
|
282 |
build_heap = build_heap, |
|
283 |
clean_build = clean_build, |
|
284 |
dirs = dirs, |
|
285 |
select_dirs = select_dirs, |
|
64265 | 286 |
numa_shuffling = numa_shuffling, |
63224 | 287 |
max_jobs = max_jobs, |
288 |
list_files = list_files, |
|
289 |
check_keywords = check_keywords, |
|
290 |
no_build = no_build, |
|
291 |
system_mode = system_mode, |
|
292 |
verbose = verbose, |
|
293 |
selection = { full_tree => |
|
294 |
full_tree.selection(requirements, all_sessions, |
|
295 |
exclude_session_groups, exclude_sessions, session_groups, sessions) }) |
|
296 |
} |
|
297 |
||
298 |
def build_selection( |
|
299 |
options: Options, |
|
64909 | 300 |
progress: Progress = No_Progress, |
63224 | 301 |
build_heap: Boolean = false, |
302 |
clean_build: Boolean = false, |
|
303 |
dirs: List[Path] = Nil, |
|
304 |
select_dirs: List[Path] = Nil, |
|
64265 | 305 |
numa_shuffling: Boolean = false, |
63224 | 306 |
max_jobs: Int = 1, |
307 |
list_files: Boolean = false, |
|
308 |
check_keywords: Set[String] = Set.empty, |
|
309 |
no_build: Boolean = false, |
|
310 |
system_mode: Boolean = false, |
|
311 |
verbose: Boolean = false, |
|
312 |
selection: Sessions.Tree => (List[String], Sessions.Tree) = |
|
313 |
(_.selection(all_sessions = true))): Results = |
|
314 |
{ |
|
315 |
/* session selection and dependencies */ |
|
51220 | 316 |
|
62714 | 317 |
val build_options = options.int.update("completion_limit", 0).bool.update("ML_statistics", true) |
318 |
val full_tree = Sessions.load(build_options, dirs, select_dirs) |
|
63224 | 319 |
val (selected, selected_tree) = selection(full_tree) |
65251 | 320 |
val deps = |
321 |
Sessions.dependencies(progress, true, verbose, list_files, check_keywords, selected_tree) |
|
48368 | 322 |
|
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
323 |
def sources_stamp(name: String): List[String] = |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
324 |
(selected_tree(name).meta_digest :: deps.sources(name)).map(_.toString).sorted |
48504
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
325 |
|
51220 | 326 |
|
327 |
/* main build process */ |
|
328 |
||
65289 | 329 |
val store = Sessions.store(system_mode) |
330 |
val queue = Queue(selected_tree, store) |
|
331 |
||
62632 | 332 |
store.prepare_output() |
48373 | 333 |
|
48595 | 334 |
// optional cleanup |
335 |
if (clean_build) { |
|
49131 | 336 |
for (name <- full_tree.graph.all_succs(selected)) { |
48595 | 337 |
val files = |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
338 |
List(Path.basic(name), store.database(name), store.log(name), store.log_gz(name)). |
62632 | 339 |
map(store.output_dir + _).filter(_.is_file) |
59319 | 340 |
if (files.nonEmpty) progress.echo("Cleaning " + name + " ...") |
50366 | 341 |
if (!files.forall(p => p.file.delete)) progress.echo(name + " FAILED to delete") |
48595 | 342 |
} |
343 |
} |
|
344 |
||
48425 | 345 |
// scheduler loop |
63082
6af03422535a
expose Sessions.Info in Build.Results
Lars Hupel <lars.hupel@mytum.de>
parents:
62946
diff
changeset
|
346 |
case class Result( |
6af03422535a
expose Sessions.Info in Build.Results
Lars Hupel <lars.hupel@mytum.de>
parents:
62946
diff
changeset
|
347 |
current: Boolean, heap_stamp: Option[String], |
6af03422535a
expose Sessions.Info in Build.Results
Lars Hupel <lars.hupel@mytum.de>
parents:
62946
diff
changeset
|
348 |
process: Option[Process_Result], info: Sessions.Info) |
62402 | 349 |
{ |
350 |
def ok: Boolean = |
|
351 |
process match { |
|
352 |
case None => false |
|
353 |
case Some(res) => res.rc == 0 |
|
354 |
} |
|
355 |
} |
|
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
356 |
|
56837
5a598f1eecfd
more robust interrupt handling for Scala_Console, which uses JVM Thread.interrupt instead of POSIX SIGINT;
wenzelm
parents:
56831
diff
changeset
|
357 |
def sleep() |
5a598f1eecfd
more robust interrupt handling for Scala_Console, which uses JVM Thread.interrupt instead of POSIX SIGINT;
wenzelm
parents:
56831
diff
changeset
|
358 |
{ |
5a598f1eecfd
more robust interrupt handling for Scala_Console, which uses JVM Thread.interrupt instead of POSIX SIGINT;
wenzelm
parents:
56831
diff
changeset
|
359 |
try { Thread.sleep(500) } |
56861 | 360 |
catch { case Exn.Interrupt() => Exn.Interrupt.impose() } |
56837
5a598f1eecfd
more robust interrupt handling for Scala_Console, which uses JVM Thread.interrupt instead of POSIX SIGINT;
wenzelm
parents:
56831
diff
changeset
|
361 |
} |
50366 | 362 |
|
64265 | 363 |
val numa_nodes = new NUMA.Nodes(numa_shuffling) |
364 |
||
48425 | 365 |
@tailrec def loop( |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
366 |
pending: Queue, |
62628 | 367 |
running: Map[String, (List[String], Job)], |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
368 |
results: Map[String, Result]): Map[String, Result] = |
48425 | 369 |
{ |
64265 | 370 |
def used_node(i: Int): Boolean = |
371 |
running.iterator.exists( |
|
372 |
{ case (_, (_, job)) => job.numa_node.isDefined && job.numa_node.get == i }) |
|
373 |
||
48425 | 374 |
if (pending.is_empty) results |
51253 | 375 |
else { |
376 |
if (progress.stopped) |
|
377 |
for ((_, (_, job)) <- running) job.terminate |
|
378 |
||
48674 | 379 |
running.find({ case (_, (_, job)) => job.is_finished }) match { |
62628 | 380 |
case Some((name, (input_heaps, job))) => |
50367 | 381 |
//{{{ finish job |
48424 | 382 |
|
62401 | 383 |
val process_result = job.join |
62573 | 384 |
process_result.err_lines.foreach(progress.echo(_)) |
385 |
if (process_result.ok) |
|
386 |
progress.echo("Finished " + name + " (" + process_result.timing.message_resources + ")") |
|
48373 | 387 |
|
65294 | 388 |
val log_lines = process_result.out_lines.filterNot(_.startsWith("\f")) |
62404
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
389 |
val process_result_tail = |
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
390 |
{ |
62409
e391528eff3b
proper option process_output_tail, more generous default;
wenzelm
parents:
62406
diff
changeset
|
391 |
val tail = job.info.options.int("process_output_tail") |
62632 | 392 |
process_result.copy( |
393 |
out_lines = |
|
65278 | 394 |
"(see also " + (store.output_dir + store.log(name)).file.toString + ")" :: |
65294 | 395 |
(if (tail == 0) log_lines else log_lines.drop(log_lines.length - tail max 0))) |
62404
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
396 |
} |
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
397 |
|
62636 | 398 |
val heap_stamp = |
62401 | 399 |
if (process_result.ok) { |
65278 | 400 |
(store.output_dir + store.log(name)).file.delete |
62636 | 401 |
val heap_stamp = |
62704
478b49f0d726
proper SHA1 digest as annex to heap file: Poly/ML reads precise segment length;
wenzelm
parents:
62643
diff
changeset
|
402 |
for (path <- job.output_path if path.is_file) |
478b49f0d726
proper SHA1 digest as annex to heap file: Poly/ML reads precise segment length;
wenzelm
parents:
62643
diff
changeset
|
403 |
yield Sessions.write_heap_digest(path) |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
404 |
|
65294 | 405 |
File.write_gzip(store.output_dir + store.log_gz(name), terminate_lines(log_lines)) |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
406 |
|
62636 | 407 |
heap_stamp |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
408 |
} |
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
409 |
else { |
62632 | 410 |
(store.output_dir + Path.basic(name)).file.delete |
65278 | 411 |
(store.output_dir + store.log_gz(name)).file.delete |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
412 |
|
65294 | 413 |
File.write(store.output_dir + store.log(name), terminate_lines(log_lines)) |
50366 | 414 |
progress.echo(name + " FAILED") |
62404
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
415 |
if (!process_result.interrupted) progress.echo(process_result_tail.out) |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
416 |
|
62636 | 417 |
None |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
418 |
} |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
419 |
|
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
420 |
// write database |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
421 |
{ |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
422 |
val database = store.output_dir + store.database(name) |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
423 |
database.file.delete |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
424 |
|
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
425 |
using(SQLite.open_database(database))(db => |
65296 | 426 |
store.write_session_info(db, |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
427 |
build_log = |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
428 |
Build_Log.Log_File(name, process_result.out_lines). |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
429 |
parse_session_info(name, |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
430 |
command_timings = true, ml_statistics = true, task_statistics = true), |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
431 |
build = |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
432 |
Session_Info(sources_stamp(name), input_heaps, heap_stamp, process_result.rc))) |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
433 |
} |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
434 |
|
50707
5b2bf7611662
maintain session index on Scala side, for more determistic results;
wenzelm
parents:
50686
diff
changeset
|
435 |
loop(pending - name, running - name, |
63082
6af03422535a
expose Sessions.Info in Build.Results
Lars Hupel <lars.hupel@mytum.de>
parents:
62946
diff
changeset
|
436 |
results + (name -> Result(false, heap_stamp, Some(process_result_tail), job.info))) |
50367 | 437 |
//}}} |
60215 | 438 |
case None if running.size < (max_jobs max 1) => |
50367 | 439 |
//{{{ check/start next job |
48547 | 440 |
pending.dequeue(running.isDefinedAt(_)) match { |
441 |
case Some((name, info)) => |
|
62628 | 442 |
val ancestor_results = selected_tree.ancestors(name).map(results(_)) |
62636 | 443 |
val ancestor_heaps = ancestor_results.flatMap(_.heap_stamp) |
62628 | 444 |
|
62883 | 445 |
val do_output = build_heap || Sessions.pure_name(name) || queue.is_inner(name) |
48547 | 446 |
|
62636 | 447 |
val (current, heap_stamp) = |
48547 | 448 |
{ |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
449 |
store.find_database_heap(name) match { |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
450 |
case Some((database, heap_stamp)) => |
65296 | 451 |
using(SQLite.open_database(database))(store.read_build(_)) match { |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
452 |
case Some(build) => |
62628 | 453 |
val current = |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
454 |
build.sources == sources_stamp(name) && |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
455 |
build.input_heaps == ancestor_heaps && |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
456 |
build.output_heap == heap_stamp && |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
457 |
!(do_output && heap_stamp.isEmpty) && |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
458 |
build.return_code == 0 |
62636 | 459 |
(current, heap_stamp) |
460 |
case None => (false, None) |
|
48547 | 461 |
} |
62636 | 462 |
case None => (false, None) |
48504
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
463 |
} |
48547 | 464 |
} |
62628 | 465 |
val all_current = current && ancestor_results.forall(_.current) |
48528
784c6f63d79c
proper all_current, which regards parent status as well;
wenzelm
parents:
48511
diff
changeset
|
466 |
|
48547 | 467 |
if (all_current) |
62402 | 468 |
loop(pending - name, running, |
63082
6af03422535a
expose Sessions.Info in Build.Results
Lars Hupel <lars.hupel@mytum.de>
parents:
62946
diff
changeset
|
469 |
results + (name -> Result(true, heap_stamp, Some(Process_Result(0)), info))) |
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
470 |
else if (no_build) { |
50366 | 471 |
if (verbose) progress.echo("Skipping " + name + " ...") |
62402 | 472 |
loop(pending - name, running, |
63082
6af03422535a
expose Sessions.Info in Build.Results
Lars Hupel <lars.hupel@mytum.de>
parents:
62946
diff
changeset
|
473 |
results + (name -> Result(false, heap_stamp, Some(Process_Result(1)), info))) |
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
474 |
} |
62628 | 475 |
else if (ancestor_results.forall(_.ok) && !progress.stopped) { |
64265 | 476 |
val numa_node = numa_nodes.next(used_node(_)) |
50366 | 477 |
progress.echo((if (do_output) "Building " else "Running ") + name + " ...") |
51220 | 478 |
val job = |
62637
0189fe0f6452
support for Poly/ML heap hierarchy, which saves a lot of disk space;
wenzelm
parents:
62636
diff
changeset
|
479 |
new Job(progress, name, info, selected_tree, store, do_output, verbose, |
64265 | 480 |
numa_node, deps(name).session_graph, queue.command_timings(name)) |
62628 | 481 |
loop(pending, running + (name -> (ancestor_heaps, job)), results) |
48547 | 482 |
} |
483 |
else { |
|
50366 | 484 |
progress.echo(name + " CANCELLED") |
65253 | 485 |
loop(pending - name, running, |
486 |
results + (name -> Result(false, heap_stamp, None, info))) |
|
48547 | 487 |
} |
488 |
case None => sleep(); loop(pending, running, results) |
|
48425 | 489 |
} |
50367 | 490 |
///}}} |
48425 | 491 |
case None => sleep(); loop(pending, running, results) |
48373 | 492 |
} |
51253 | 493 |
} |
48425 | 494 |
} |
495 |
||
51220 | 496 |
|
497 |
/* build results */ |
|
498 |
||
62641 | 499 |
val results0 = |
48583 | 500 |
if (deps.is_empty) { |
56782
433cf57550fa
more systematic Isabelle output, like in classic Isabelle/ML (without markup);
wenzelm
parents:
56780
diff
changeset
|
501 |
progress.echo(Output.warning_text("Nothing to build")) |
50707
5b2bf7611662
maintain session index on Scala side, for more determistic results;
wenzelm
parents:
50686
diff
changeset
|
502 |
Map.empty[String, Result] |
48583 | 503 |
} |
504 |
else loop(queue, Map.empty, Map.empty) |
|
505 |
||
62641 | 506 |
val results = |
64265 | 507 |
new Results( |
508 |
(for ((name, result) <- results0.iterator) |
|
509 |
yield (name, (result.process, result.info))).toMap) |
|
62641 | 510 |
|
511 |
if (results.rc != 0 && (verbose || !no_build)) { |
|
512 |
val unfinished = |
|
513 |
(for { |
|
514 |
name <- results.sessions.iterator |
|
515 |
if !results(name).ok |
|
516 |
} yield name).toList.sorted |
|
517 |
progress.echo("Unfinished session(s): " + commas(unfinished)) |
|
518 |
} |
|
519 |
||
51418
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
520 |
|
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
521 |
/* global browser info */ |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
522 |
|
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
523 |
if (!no_build) { |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
524 |
val browser_chapters = |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
525 |
(for { |
62641 | 526 |
(name, result) <- results0.iterator |
62944
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
527 |
if result.ok |
51418
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
528 |
info = full_tree(name) |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
529 |
if info.options.bool("browser_info") |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
530 |
} yield (info.chapter, (name, info.description))).toList.groupBy(_._1). |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
531 |
map({ case (chapter, es) => (chapter, es.map(_._2)) }).filterNot(_._2.isEmpty) |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
532 |
|
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
533 |
for ((chapter, entries) <- browser_chapters) |
62632 | 534 |
Present.update_chapter_index(store.browser_info, chapter, entries) |
51418
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
535 |
|
62632 | 536 |
if (browser_chapters.nonEmpty) Present.make_global_index(store.browser_info) |
51418
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
537 |
} |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
538 |
|
62641 | 539 |
results |
48341 | 540 |
} |
541 |
||
542 |
||
62833 | 543 |
/* Isabelle tool wrapper */ |
48341 | 544 |
|
62833 | 545 |
val isabelle_tool = Isabelle_Tool("build", "build and manage Isabelle sessions", args => |
48341 | 546 |
{ |
62833 | 547 |
val build_options = Word.explode(Isabelle_System.getenv("ISABELLE_BUILD_OPTIONS")) |
62590 | 548 |
|
62833 | 549 |
var select_dirs: List[Path] = Nil |
64265 | 550 |
var numa_shuffling = false |
62833 | 551 |
var requirements = false |
552 |
var exclude_session_groups: List[String] = Nil |
|
553 |
var all_sessions = false |
|
554 |
var build_heap = false |
|
555 |
var clean_build = false |
|
556 |
var dirs: List[Path] = Nil |
|
557 |
var session_groups: List[String] = Nil |
|
558 |
var max_jobs = 1 |
|
559 |
var check_keywords: Set[String] = Set.empty |
|
560 |
var list_files = false |
|
561 |
var no_build = false |
|
562 |
var options = (Options.init() /: build_options)(_ + _) |
|
563 |
var system_mode = false |
|
564 |
var verbose = false |
|
565 |
var exclude_sessions: List[String] = Nil |
|
62590 | 566 |
|
62833 | 567 |
val getopts = Getopts(""" |
62590 | 568 |
Usage: isabelle build [OPTIONS] [SESSIONS ...] |
569 |
||
570 |
Options are: |
|
571 |
-D DIR include session directory and select its sessions |
|
64265 | 572 |
-N cyclic shuffling of NUMA CPU nodes (performance tuning) |
62590 | 573 |
-R operate on requirements of selected sessions |
574 |
-X NAME exclude sessions from group NAME and all descendants |
|
575 |
-a select all sessions |
|
576 |
-b build heap images |
|
577 |
-c clean build |
|
578 |
-d DIR include session directory |
|
579 |
-g NAME select session group NAME |
|
580 |
-j INT maximum number of parallel jobs (default 1) |
|
581 |
-k KEYWORD check theory sources for conflicts with proposed keywords |
|
582 |
-l list session source files |
|
583 |
-n no build -- test dependencies only |
|
584 |
-o OPTION override Isabelle system OPTION (via NAME=VAL or NAME) |
|
585 |
-s system build mode: produce output in ISABELLE_HOME |
|
586 |
-v verbose |
|
587 |
-x NAME exclude session NAME and all descendants |
|
588 |
||
62596 | 589 |
Build and manage Isabelle sessions, depending on implicit settings: |
590 |
||
64455 | 591 |
""" + Library.prefix_lines(" ", Build_Log.Settings.show()) + "\n", |
62833 | 592 |
"D:" -> (arg => select_dirs = select_dirs ::: List(Path.explode(arg))), |
64265 | 593 |
"N" -> (_ => numa_shuffling = true), |
62833 | 594 |
"R" -> (_ => requirements = true), |
595 |
"X:" -> (arg => exclude_session_groups = exclude_session_groups ::: List(arg)), |
|
596 |
"a" -> (_ => all_sessions = true), |
|
597 |
"b" -> (_ => build_heap = true), |
|
598 |
"c" -> (_ => clean_build = true), |
|
599 |
"d:" -> (arg => dirs = dirs ::: List(Path.explode(arg))), |
|
600 |
"g:" -> (arg => session_groups = session_groups ::: List(arg)), |
|
63805 | 601 |
"j:" -> (arg => max_jobs = Value.Int.parse(arg)), |
62833 | 602 |
"k:" -> (arg => check_keywords = check_keywords + arg), |
603 |
"l" -> (_ => list_files = true), |
|
604 |
"n" -> (_ => no_build = true), |
|
605 |
"o:" -> (arg => options = options + arg), |
|
606 |
"s" -> (_ => system_mode = true), |
|
607 |
"v" -> (_ => verbose = true), |
|
608 |
"x:" -> (arg => exclude_sessions = exclude_sessions ::: List(arg))) |
|
62590 | 609 |
|
62833 | 610 |
val sessions = getopts(args) |
62590 | 611 |
|
64115 | 612 |
val progress = new Console_Progress(verbose = verbose) |
62590 | 613 |
|
64140 | 614 |
val start_date = Date.now() |
615 |
||
62833 | 616 |
if (verbose) { |
617 |
progress.echo( |
|
64155 | 618 |
"Started at " + Build_Log.print_date(start_date) + |
64140 | 619 |
" (" + Isabelle_System.getenv("ML_IDENTIFIER") + " on " + Isabelle_System.hostname() +")") |
64081 | 620 |
progress.echo(Build_Log.Settings.show() + "\n") |
62833 | 621 |
} |
62590 | 622 |
|
62833 | 623 |
val results = |
624 |
progress.interrupt_handler { |
|
63224 | 625 |
build(options, progress, |
626 |
build_heap = build_heap, |
|
627 |
clean_build = clean_build, |
|
628 |
dirs = dirs, |
|
629 |
select_dirs = select_dirs, |
|
64265 | 630 |
numa_shuffling = NUMA.enabled_warning(numa_shuffling), |
63224 | 631 |
max_jobs = max_jobs, |
632 |
list_files = list_files, |
|
633 |
check_keywords = check_keywords, |
|
634 |
no_build = no_build, |
|
635 |
system_mode = system_mode, |
|
636 |
verbose = verbose, |
|
637 |
requirements = requirements, |
|
638 |
all_sessions = all_sessions, |
|
639 |
exclude_session_groups = exclude_session_groups, |
|
640 |
exclude_sessions = exclude_sessions, |
|
641 |
session_groups = session_groups, |
|
642 |
sessions = sessions) |
|
62833 | 643 |
} |
64140 | 644 |
val end_date = Date.now() |
645 |
val elapsed_time = end_date.time - start_date.time |
|
62590 | 646 |
|
62833 | 647 |
if (verbose) { |
64155 | 648 |
progress.echo("\nFinished at " + Build_Log.print_date(end_date)) |
62833 | 649 |
} |
62590 | 650 |
|
62833 | 651 |
val total_timing = |
652 |
(Timing.zero /: results.sessions.iterator.map(a => results(a).timing))(_ + _). |
|
653 |
copy(elapsed = elapsed_time) |
|
654 |
progress.echo(total_timing.message_resources) |
|
62590 | 655 |
|
62833 | 656 |
sys.exit(results.rc) |
657 |
}) |
|
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
658 |
|
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
659 |
|
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
660 |
/* PIDE protocol */ |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
661 |
|
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
662 |
def build_theories( |
59369
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
663 |
session: Session, master_dir: Path, theories: List[(Options, List[Path])]): Promise[XML.Body] = |
59367 | 664 |
session.get_protocol_handler(classOf[Handler].getName) match { |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
665 |
case Some(handler: Handler) => handler.build_theories(session, master_dir, theories) |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
666 |
case _ => error("Cannot invoke build_theories: bad protocol handler") |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
667 |
} |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
668 |
|
59367 | 669 |
class Handler(progress: Progress, session_name: String) extends Session.Protocol_Handler |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
670 |
{ |
59369
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
671 |
private val pending = Synchronized(Map.empty[String, Promise[XML.Body]]) |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
672 |
|
65220 | 673 |
override def exit(): Unit = |
674 |
pending.change(promises => { for ((_, promise) <- promises) promise.cancel; Map.empty }) |
|
675 |
||
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
676 |
def build_theories( |
59369
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
677 |
session: Session, master_dir: Path, theories: List[(Options, List[Path])]): Promise[XML.Body] = |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
678 |
{ |
59369
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
679 |
val promise = Future.promise[XML.Body] |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
680 |
val id = Document_ID.make().toString |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
681 |
pending.change(promises => promises + (id -> promise)) |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
682 |
session.build_theories(id, master_dir, theories) |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
683 |
promise |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
684 |
} |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
685 |
|
65219 | 686 |
private def loading_theory(msg: Prover.Protocol_Output): Boolean = |
59367 | 687 |
msg.properties match { |
688 |
case Markup.Loading_Theory(name) => progress.theory(session_name, name); true |
|
689 |
case _ => false |
|
690 |
} |
|
691 |
||
65219 | 692 |
private def build_theories_result(msg: Prover.Protocol_Output): Boolean = |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
693 |
msg.properties match { |
59369
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
694 |
case Markup.Build_Theories_Result(id) => |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
695 |
pending.change_result(promises => |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
696 |
promises.get(id) match { |
59369
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
697 |
case Some(promise) => |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
698 |
val error_message = |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
699 |
try { YXML.parse_body(Symbol.decode(msg.text)) } |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
700 |
catch { case exn: Throwable => List(XML.Text(Exn.message(exn))) } |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
701 |
promise.fulfill(error_message) |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
702 |
(true, promises - id) |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
703 |
case None => |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
704 |
(false, promises) |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
705 |
}) |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
706 |
case _ => false |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
707 |
} |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
708 |
|
59367 | 709 |
val functions = |
65219 | 710 |
List( |
59367 | 711 |
Markup.BUILD_THEORIES_RESULT -> build_theories_result _, |
712 |
Markup.LOADING_THEORY -> loading_theory _) |
|
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
713 |
} |
48276 | 714 |
} |