author | wenzelm |
Sun, 12 Feb 2023 13:45:06 +0100 | |
changeset 77257 | 68a7ad1385bc |
parent 77256 | 25e923c57af7 |
child 77258 | ca8eda3c1808 |
permissions | -rw-r--r-- |
77238 | 1 |
/* Title: Pure/Tools/build_process.scala |
2 |
Author: Makarius |
|
3 |
||
4 |
Build process for sessions, with build database, optional heap, and |
|
5 |
optional presentation. |
|
6 |
*/ |
|
7 |
||
8 |
package isabelle |
|
9 |
||
10 |
||
77246 | 11 |
import scala.math.Ordering |
77257 | 12 |
import scala.collection.immutable.SortedSet |
13 |
import scala.annotation.tailrec |
|
77246 | 14 |
|
15 |
||
77238 | 16 |
object Build_Process { |
77244 | 17 |
/* static information */ |
77238 | 18 |
|
77244 | 19 |
object Session_Context { |
77249
f3f1b7ad1d0d
clarified data structure: more direct access to timeout;
wenzelm
parents:
77248
diff
changeset
|
20 |
def empty(session: String, timeout: Time): Session_Context = |
f3f1b7ad1d0d
clarified data structure: more direct access to timeout;
wenzelm
parents:
77248
diff
changeset
|
21 |
new Session_Context(session, timeout, Time.zero, Nil) |
77238 | 22 |
|
77246 | 23 |
def apply( |
77239 | 24 |
session: String, |
77249
f3f1b7ad1d0d
clarified data structure: more direct access to timeout;
wenzelm
parents:
77248
diff
changeset
|
25 |
timeout: Time, |
77239 | 26 |
store: Sessions.Store, |
27 |
progress: Progress = new Progress |
|
77244 | 28 |
): Session_Context = { |
77239 | 29 |
store.try_open_database(session) match { |
77249
f3f1b7ad1d0d
clarified data structure: more direct access to timeout;
wenzelm
parents:
77248
diff
changeset
|
30 |
case None => empty(session, timeout) |
77238 | 31 |
case Some(db) => |
32 |
def ignore_error(msg: String) = { |
|
33 |
progress.echo_warning("Ignoring bad database " + db + |
|
77239 | 34 |
" for session " + quote(session) + (if (msg == "") "" else ":\n" + msg)) |
77249
f3f1b7ad1d0d
clarified data structure: more direct access to timeout;
wenzelm
parents:
77248
diff
changeset
|
35 |
empty(session, timeout) |
77238 | 36 |
} |
37 |
try { |
|
77239 | 38 |
val command_timings = store.read_command_timings(db, session) |
39 |
val elapsed = |
|
40 |
store.read_session_timing(db, session) match { |
|
41 |
case Markup.Elapsed(s) => Time.seconds(s) |
|
42 |
case _ => Time.zero |
|
77238 | 43 |
} |
77249
f3f1b7ad1d0d
clarified data structure: more direct access to timeout;
wenzelm
parents:
77248
diff
changeset
|
44 |
new Session_Context(session, timeout, elapsed, command_timings) |
77238 | 45 |
} |
46 |
catch { |
|
47 |
case ERROR(msg) => ignore_error(msg) |
|
48 |
case exn: java.lang.Error => ignore_error(Exn.message(exn)) |
|
49 |
case _: XML.Error => ignore_error("XML.Error") |
|
50 |
} |
|
51 |
finally { db.close() } |
|
52 |
} |
|
53 |
} |
|
54 |
} |
|
77239 | 55 |
|
77244 | 56 |
final class Session_Context( |
77239 | 57 |
val session: String, |
77249
f3f1b7ad1d0d
clarified data structure: more direct access to timeout;
wenzelm
parents:
77248
diff
changeset
|
58 |
val timeout: Time, |
77244 | 59 |
val old_time: Time, |
60 |
val old_command_timings: List[Properties.T] |
|
77239 | 61 |
) { |
77244 | 62 |
def is_empty: Boolean = old_time.is_zero && old_command_timings.isEmpty |
77240 | 63 |
|
77245
1e2670d9dc18
tuned message: old_time not sufficiently prominent nor accurate to be printed;
wenzelm
parents:
77244
diff
changeset
|
64 |
override def toString: String = session |
77239 | 65 |
} |
77246 | 66 |
|
67 |
object Context { |
|
68 |
def apply( |
|
69 |
store: Sessions.Store, |
|
77257 | 70 |
deps: Sessions.Deps, |
77246 | 71 |
progress: Progress = new Progress |
72 |
): Context = { |
|
77257 | 73 |
val sessions_structure = deps.sessions_structure |
77247 | 74 |
val build_graph = sessions_structure.build_graph |
75 |
||
77246 | 76 |
val sessions = |
77 |
Map.from( |
|
77247 | 78 |
for (name <- build_graph.keys_iterator) |
77249
f3f1b7ad1d0d
clarified data structure: more direct access to timeout;
wenzelm
parents:
77248
diff
changeset
|
79 |
yield { |
f3f1b7ad1d0d
clarified data structure: more direct access to timeout;
wenzelm
parents:
77248
diff
changeset
|
80 |
val timeout = sessions_structure(name).timeout |
f3f1b7ad1d0d
clarified data structure: more direct access to timeout;
wenzelm
parents:
77248
diff
changeset
|
81 |
name -> Build_Process.Session_Context(name, timeout, store, progress = progress) |
f3f1b7ad1d0d
clarified data structure: more direct access to timeout;
wenzelm
parents:
77248
diff
changeset
|
82 |
}) |
77247 | 83 |
|
77248 | 84 |
val sessions_time = { |
85 |
val maximals = build_graph.maximals.toSet |
|
86 |
def descendants_time(name: String): Double = { |
|
87 |
if (maximals.contains(name)) sessions(name).old_time.seconds |
|
88 |
else { |
|
89 |
val descendants = build_graph.all_succs(List(name)).toSet |
|
90 |
val g = build_graph.restrict(descendants) |
|
91 |
(0.0 :: g.maximals.flatMap { desc => |
|
92 |
val ps = g.all_preds(List(desc)) |
|
93 |
if (ps.exists(p => !sessions.isDefinedAt(p))) None |
|
94 |
else Some(ps.map(p => sessions(p).old_time.seconds).sum) |
|
95 |
}).max |
|
96 |
} |
|
77247 | 97 |
} |
77248 | 98 |
Map.from( |
99 |
for (name <- sessions.keysIterator) |
|
100 |
yield name -> descendants_time(name)).withDefaultValue(0.0) |
|
77247 | 101 |
} |
102 |
||
77246 | 103 |
val ordering = |
104 |
new Ordering[String] { |
|
105 |
def compare(name1: String, name2: String): Int = |
|
77248 | 106 |
sessions_time(name2) compare sessions_time(name1) match { |
77246 | 107 |
case 0 => |
77249
f3f1b7ad1d0d
clarified data structure: more direct access to timeout;
wenzelm
parents:
77248
diff
changeset
|
108 |
sessions(name2).timeout compare sessions(name1).timeout match { |
77246 | 109 |
case 0 => name1 compare name2 |
110 |
case ord => ord |
|
111 |
} |
|
112 |
case ord => ord |
|
113 |
} |
|
114 |
} |
|
115 |
||
77257 | 116 |
new Context(store, deps, sessions, ordering, progress) |
77246 | 117 |
} |
118 |
} |
|
119 |
||
120 |
final class Context private( |
|
77257 | 121 |
val store: Sessions.Store, |
122 |
val deps: Sessions.Deps, |
|
77246 | 123 |
sessions: Map[String, Session_Context], |
77257 | 124 |
val ordering: Ordering[String], |
125 |
val progress: Progress |
|
77246 | 126 |
) { |
77257 | 127 |
def sessions_structure: Sessions.Structure = deps.sessions_structure |
128 |
||
77246 | 129 |
def apply(session: String): Session_Context = |
77249
f3f1b7ad1d0d
clarified data structure: more direct access to timeout;
wenzelm
parents:
77248
diff
changeset
|
130 |
sessions.getOrElse(session, Session_Context.empty(session, Time.zero)) |
77255
b810e99b5afb
clarified static build_context vs. dynamic queue;
wenzelm
parents:
77254
diff
changeset
|
131 |
|
77256 | 132 |
def build_heap(session: String): Boolean = |
133 |
Sessions.is_pure(session) || !sessions_structure.build_graph.is_maximal(session) |
|
77246 | 134 |
} |
77257 | 135 |
|
136 |
||
137 |
/* queue with scheduling information */ |
|
138 |
||
139 |
private object Queue { |
|
140 |
def apply(build_context: Build_Process.Context): Queue = { |
|
141 |
val build_graph = build_context.sessions_structure.build_graph |
|
142 |
val build_order = SortedSet.from(build_graph.keys)(build_context.ordering) |
|
143 |
new Queue(build_graph, build_order) |
|
144 |
} |
|
145 |
} |
|
146 |
||
147 |
private class Queue( |
|
148 |
build_graph: Graph[String, Sessions.Info], |
|
149 |
build_order: SortedSet[String] |
|
150 |
) { |
|
151 |
def is_empty: Boolean = build_graph.is_empty |
|
152 |
||
153 |
def - (name: String): Queue = |
|
154 |
new Queue(build_graph.del_node(name), build_order - name) |
|
155 |
||
156 |
def dequeue(skip: String => Boolean): Option[String] = |
|
157 |
build_order.iterator.dropWhile(name => skip(name) || !build_graph.is_minimal(name)) |
|
158 |
.nextOption() |
|
159 |
} |
|
160 |
||
161 |
||
162 |
/* main */ |
|
163 |
||
164 |
private def session_finished(session_name: String, process_result: Process_Result): String = |
|
165 |
"Finished " + session_name + " (" + process_result.timing.message_resources + ")" |
|
166 |
||
167 |
private def session_timing(session_name: String, build_log: Build_Log.Session_Info): String = { |
|
168 |
val props = build_log.session_timing |
|
169 |
val threads = Markup.Session_Timing.Threads.unapply(props) getOrElse 1 |
|
170 |
val timing = Markup.Timing_Properties.get(props) |
|
171 |
"Timing " + session_name + " (" + threads + " threads, " + timing.message_factor + ")" |
|
172 |
} |
|
173 |
||
174 |
case class Result( |
|
175 |
current: Boolean, |
|
176 |
output_heap: SHA1.Shasum, |
|
177 |
process_result: Process_Result |
|
178 |
) { |
|
179 |
def ok: Boolean = process_result.ok |
|
180 |
} |
|
181 |
||
182 |
def main( |
|
183 |
build_context: Context, |
|
184 |
build_heap: Boolean = false, |
|
185 |
numa_shuffling: Boolean = false, |
|
186 |
max_jobs: Int = 1, |
|
187 |
fresh_build: Boolean = false, |
|
188 |
no_build: Boolean = false, |
|
189 |
verbose: Boolean = false, |
|
190 |
session_setup: (String, Session) => Unit = (_, _) => () |
|
191 |
): Map[String, Result] = { |
|
192 |
val store = build_context.store |
|
193 |
val build_options = store.options |
|
194 |
val build_deps = build_context.deps |
|
195 |
val progress = build_context.progress |
|
196 |
||
197 |
def sleep(): Unit = |
|
198 |
Isabelle_Thread.interrupt_handler(_ => progress.stop()) { |
|
199 |
build_options.seconds("editor_input_delay").sleep() |
|
200 |
} |
|
201 |
||
202 |
val log = |
|
203 |
build_options.string("system_log") match { |
|
204 |
case "" => No_Logger |
|
205 |
case "-" => Logger.make(progress) |
|
206 |
case log_file => Logger.make(Some(Path.explode(log_file))) |
|
207 |
} |
|
208 |
||
209 |
val numa_nodes = new NUMA.Nodes(numa_shuffling) |
|
210 |
||
211 |
@tailrec def loop( |
|
212 |
pending: Queue, |
|
213 |
running: Map[String, (SHA1.Shasum, Build_Job)], |
|
214 |
results: Map[String, Result] |
|
215 |
): Map[String, Result] = { |
|
216 |
def used_node(i: Int): Boolean = |
|
217 |
running.iterator.exists( |
|
218 |
{ case (_, (_, job)) => job.numa_node.isDefined && job.numa_node.get == i }) |
|
219 |
||
220 |
if (pending.is_empty) results |
|
221 |
else { |
|
222 |
if (progress.stopped) { |
|
223 |
for ((_, (_, job)) <- running) job.terminate() |
|
224 |
} |
|
225 |
||
226 |
running.find({ case (_, (_, job)) => job.is_finished }) match { |
|
227 |
case Some((session_name, (input_heaps, job))) => |
|
228 |
//{{{ finish job |
|
229 |
||
230 |
val process_result = job.join |
|
231 |
||
232 |
val output_heap = |
|
233 |
if (process_result.ok && job.do_store && store.output_heap(session_name).is_file) { |
|
234 |
SHA1.shasum(ML_Heap.write_digest(store.output_heap(session_name)), session_name) |
|
235 |
} |
|
236 |
else SHA1.no_shasum |
|
237 |
||
238 |
val log_lines = process_result.out_lines.filterNot(Protocol_Message.Marker.test) |
|
239 |
val process_result_tail = { |
|
240 |
val tail = job.info.options.int("process_output_tail") |
|
241 |
process_result.copy( |
|
242 |
out_lines = |
|
243 |
"(more details via \"isabelle log -H Error " + session_name + "\")" :: |
|
244 |
(if (tail == 0) log_lines else log_lines.drop(log_lines.length - tail max 0))) |
|
245 |
} |
|
246 |
||
247 |
val build_log = |
|
248 |
Build_Log.Log_File(session_name, process_result.out_lines). |
|
249 |
parse_session_info( |
|
250 |
command_timings = true, |
|
251 |
theory_timings = true, |
|
252 |
ml_statistics = true, |
|
253 |
task_statistics = true) |
|
254 |
||
255 |
// write log file |
|
256 |
if (process_result.ok) { |
|
257 |
File.write_gzip(store.output_log_gz(session_name), terminate_lines(log_lines)) |
|
258 |
} |
|
259 |
else File.write(store.output_log(session_name), terminate_lines(log_lines)) |
|
260 |
||
261 |
// write database |
|
262 |
using(store.open_database(session_name, output = true))(db => |
|
263 |
store.write_session_info(db, session_name, job.session_sources, |
|
264 |
build_log = |
|
265 |
if (process_result.timeout) build_log.error("Timeout") else build_log, |
|
266 |
build = |
|
267 |
Build.Session_Info(build_deps.sources_shasum(session_name), input_heaps, |
|
268 |
output_heap, process_result.rc, UUID.random().toString))) |
|
269 |
||
270 |
// messages |
|
271 |
process_result.err_lines.foreach(progress.echo) |
|
272 |
||
273 |
if (process_result.ok) { |
|
274 |
if (verbose) progress.echo(session_timing(session_name, build_log)) |
|
275 |
progress.echo(session_finished(session_name, process_result)) |
|
276 |
} |
|
277 |
else { |
|
278 |
progress.echo(session_name + " FAILED") |
|
279 |
if (!process_result.interrupted) progress.echo(process_result_tail.out) |
|
280 |
} |
|
281 |
||
282 |
loop(pending - session_name, running - session_name, |
|
283 |
results + (session_name -> Result(false, output_heap, process_result_tail))) |
|
284 |
//}}} |
|
285 |
case None if running.size < (max_jobs max 1) => |
|
286 |
//{{{ check/start next job |
|
287 |
pending.dequeue(running.isDefinedAt) match { |
|
288 |
case Some(session_name) => |
|
289 |
val ancestor_results = |
|
290 |
build_deps.sessions_structure.build_requirements(List(session_name)). |
|
291 |
filterNot(_ == session_name).map(results(_)) |
|
292 |
val input_heaps = |
|
293 |
if (ancestor_results.isEmpty) { |
|
294 |
SHA1.shasum_meta_info(SHA1.digest(Path.explode("$POLYML_EXE"))) |
|
295 |
} |
|
296 |
else SHA1.flat_shasum(ancestor_results.map(_.output_heap)) |
|
297 |
||
298 |
val do_store = build_heap || build_context.build_heap(session_name) |
|
299 |
val (current, output_heap) = { |
|
300 |
store.try_open_database(session_name) match { |
|
301 |
case Some(db) => |
|
302 |
using(db)(store.read_build(_, session_name)) match { |
|
303 |
case Some(build) => |
|
304 |
val output_heap = store.find_heap_shasum(session_name) |
|
305 |
val current = |
|
306 |
!fresh_build && |
|
307 |
build.ok && |
|
308 |
build.sources == build_deps.sources_shasum(session_name) && |
|
309 |
build.input_heaps == input_heaps && |
|
310 |
build.output_heap == output_heap && |
|
311 |
!(do_store && output_heap.is_empty) |
|
312 |
(current, output_heap) |
|
313 |
case None => (false, SHA1.no_shasum) |
|
314 |
} |
|
315 |
case None => (false, SHA1.no_shasum) |
|
316 |
} |
|
317 |
} |
|
318 |
val all_current = current && ancestor_results.forall(_.current) |
|
319 |
||
320 |
if (all_current) { |
|
321 |
loop(pending - session_name, running, |
|
322 |
results + (session_name -> Result(true, output_heap, Process_Result.ok))) |
|
323 |
} |
|
324 |
else if (no_build) { |
|
325 |
progress.echo_if(verbose, "Skipping " + session_name + " ...") |
|
326 |
loop(pending - session_name, running, |
|
327 |
results + (session_name -> Result(false, output_heap, Process_Result.error))) |
|
328 |
} |
|
329 |
else if (ancestor_results.forall(_.ok) && !progress.stopped) { |
|
330 |
progress.echo((if (do_store) "Building " else "Running ") + session_name + " ...") |
|
331 |
||
332 |
store.clean_output(session_name) |
|
333 |
using(store.open_database(session_name, output = true))( |
|
334 |
store.init_session_info(_, session_name)) |
|
335 |
||
336 |
val session_background = build_deps.background(session_name) |
|
337 |
val resources = |
|
338 |
new Resources(session_background, log = log, |
|
339 |
command_timings = build_context(session_name).old_command_timings) |
|
340 |
||
341 |
val numa_node = numa_nodes.next(used_node) |
|
342 |
val job = |
|
343 |
new Build_Job(progress, session_background, store, do_store, |
|
344 |
resources, session_setup, numa_node) |
|
345 |
loop(pending, running + (session_name -> (input_heaps, job)), results) |
|
346 |
} |
|
347 |
else { |
|
348 |
progress.echo(session_name + " CANCELLED") |
|
349 |
loop(pending - session_name, running, |
|
350 |
results + (session_name -> Result(false, output_heap, Process_Result.undefined))) |
|
351 |
} |
|
352 |
case None => sleep(); loop(pending, running, results) |
|
353 |
} |
|
354 |
///}}} |
|
355 |
case None => sleep(); loop(pending, running, results) |
|
356 |
} |
|
357 |
} |
|
358 |
} |
|
359 |
||
360 |
loop(Queue(build_context), Map.empty, Map.empty) |
|
361 |
} |
|
77238 | 362 |
} |