author | wenzelm |
Sat, 11 Jul 2020 15:23:22 +0200 | |
changeset 72012 | c81e58a81b8c |
parent 72007 | 13890356df78 |
child 72013 | 6a24ecc4ff1b |
permissions | -rw-r--r-- |
50686 | 1 |
/* Title: Pure/Tools/build.scala |
48276 | 2 |
Author: Makarius |
57923 | 3 |
Options: :folding=explicit: |
48276 | 4 |
|
5 |
Build and manage Isabelle sessions. |
|
6 |
*/ |
|
7 |
||
8 |
package isabelle |
|
9 |
||
10 |
||
71648 | 11 |
import scala.collection.{SortedSet, mutable} |
48340
6f4fc030882a
allow explicit specification of additional session directories;
wenzelm
parents:
48339
diff
changeset
|
12 |
import scala.annotation.tailrec |
48337
9c7f8e5805b4
cumulate semantic Session_Info, based on syntactic Session_Entry;
wenzelm
parents:
48336
diff
changeset
|
13 |
|
48335 | 14 |
|
48276 | 15 |
object Build |
16 |
{ |
|
62631 | 17 |
/** auxiliary **/ |
48424 | 18 |
|
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
19 |
/* persistent build info */ |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
20 |
|
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
21 |
sealed case class Session_Info( |
66749
0445cfaf6132
more compact (second-order) digest for 10^2..10^3 source files, with slightly increased risk of collisions;
wenzelm
parents:
66747
diff
changeset
|
22 |
sources: String, |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
23 |
input_heaps: List[String], |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
24 |
output_heap: Option[String], |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
25 |
return_code: Int) |
66747 | 26 |
{ |
27 |
def ok: Boolean = return_code == 0 |
|
28 |
} |
|
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
29 |
|
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
30 |
|
65289 | 31 |
/* queue with scheduling information */ |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
32 |
|
62631 | 33 |
private object Queue |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
34 |
{ |
68086
9e1c670301b8
cleanup session output before starting build job;
wenzelm
parents:
67852
diff
changeset
|
35 |
type Timings = (List[Properties.T], Double) |
9e1c670301b8
cleanup session output before starting build job;
wenzelm
parents:
67852
diff
changeset
|
36 |
|
71669 | 37 |
def load_timings(progress: Progress, store: Sessions.Store, session_name: String): Timings = |
65289 | 38 |
{ |
68086
9e1c670301b8
cleanup session output before starting build job;
wenzelm
parents:
67852
diff
changeset
|
39 |
val no_timings: Timings = (Nil, 0.0) |
65289 | 40 |
|
71669 | 41 |
store.access_database(session_name) match { |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
42 |
case None => no_timings |
68214 | 43 |
case Some(db) => |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
44 |
def ignore_error(msg: String) = |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
45 |
{ |
68214 | 46 |
progress.echo_warning("Ignoring bad database " + db + (if (msg == "") "" else "\n" + msg)) |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
47 |
no_timings |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
48 |
} |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
49 |
try { |
71669 | 50 |
val command_timings = store.read_command_timings(db, session_name) |
68214 | 51 |
val session_timing = |
71669 | 52 |
store.read_session_timing(db, session_name) match { |
68214 | 53 |
case Markup.Elapsed(t) => t |
54 |
case _ => 0.0 |
|
55 |
} |
|
56 |
(command_timings, session_timing) |
|
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
57 |
} |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
58 |
catch { |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
59 |
case ERROR(msg) => ignore_error(msg) |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
60 |
case exn: java.lang.Error => ignore_error(Exn.message(exn)) |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
61 |
case _: XML.Error => ignore_error("") |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
62 |
} |
68214 | 63 |
finally { db.close } |
65289 | 64 |
} |
65 |
} |
|
66 |
||
68731 | 67 |
def make_session_timing(sessions_structure: Sessions.Structure, timing: Map[String, Double]) |
68486
6984a55f3cba
clarified queue ordering: take session descendants into account, notably for "slow" AFP sessions;
wenzelm
parents:
68331
diff
changeset
|
68 |
: Map[String, Double] = |
6984a55f3cba
clarified queue ordering: take session descendants into account, notably for "slow" AFP sessions;
wenzelm
parents:
68331
diff
changeset
|
69 |
{ |
68731 | 70 |
val maximals = sessions_structure.build_graph.maximals.toSet |
71669 | 71 |
def desc_timing(session_name: String): Double = |
68486
6984a55f3cba
clarified queue ordering: take session descendants into account, notably for "slow" AFP sessions;
wenzelm
parents:
68331
diff
changeset
|
72 |
{ |
71669 | 73 |
if (maximals.contains(session_name)) timing(session_name) |
68487 | 74 |
else { |
71669 | 75 |
val descendants = sessions_structure.build_descendants(List(session_name)).toSet |
68731 | 76 |
val g = sessions_structure.build_graph.restrict(descendants) |
68487 | 77 |
(0.0 :: g.maximals.flatMap(desc => { |
78 |
val ps = g.all_preds(List(desc)) |
|
71895 | 79 |
if (ps.exists(p => !timing.isDefinedAt(p))) None |
68487 | 80 |
else Some(ps.map(timing(_)).sum) |
81 |
})).max |
|
82 |
} |
|
68486
6984a55f3cba
clarified queue ordering: take session descendants into account, notably for "slow" AFP sessions;
wenzelm
parents:
68331
diff
changeset
|
83 |
} |
6984a55f3cba
clarified queue ordering: take session descendants into account, notably for "slow" AFP sessions;
wenzelm
parents:
68331
diff
changeset
|
84 |
timing.keySet.iterator.map(name => (name -> desc_timing(name))).toMap.withDefaultValue(0.0) |
6984a55f3cba
clarified queue ordering: take session descendants into account, notably for "slow" AFP sessions;
wenzelm
parents:
68331
diff
changeset
|
85 |
} |
6984a55f3cba
clarified queue ordering: take session descendants into account, notably for "slow" AFP sessions;
wenzelm
parents:
68331
diff
changeset
|
86 |
|
68731 | 87 |
def apply(progress: Progress, sessions_structure: Sessions.Structure, store: Sessions.Store) |
88 |
: Queue = |
|
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
89 |
{ |
68731 | 90 |
val graph = sessions_structure.build_graph |
65415
8cd54b18b68b
clarified signature: tree structure is not essential;
wenzelm
parents:
65406
diff
changeset
|
91 |
val names = graph.keys |
51220 | 92 |
|
65831 | 93 |
val timings = names.map(name => (name, load_timings(progress, store, name))) |
51220 | 94 |
val command_timings = |
68486
6984a55f3cba
clarified queue ordering: take session descendants into account, notably for "slow" AFP sessions;
wenzelm
parents:
68331
diff
changeset
|
95 |
timings.map({ case (name, (ts, _)) => (name, ts) }).toMap.withDefaultValue(Nil) |
51220 | 96 |
val session_timing = |
68731 | 97 |
make_session_timing(sessions_structure, |
98 |
timings.map({ case (name, (_, t)) => (name, t) }).toMap) |
|
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
99 |
|
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
100 |
object Ordering extends scala.math.Ordering[String] |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
101 |
{ |
51227
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
102 |
def compare_timing(name1: String, name2: String): Int = |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
103 |
{ |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
104 |
val t1 = session_timing(name1) |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
105 |
val t2 = session_timing(name2) |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
106 |
if (t1 == 0.0 || t2 == 0.0) 0 |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
107 |
else t1 compare t2 |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
108 |
} |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
109 |
|
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
110 |
def compare(name1: String, name2: String): Int = |
68486
6984a55f3cba
clarified queue ordering: take session descendants into account, notably for "slow" AFP sessions;
wenzelm
parents:
68331
diff
changeset
|
111 |
compare_timing(name2, name1) match { |
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
112 |
case 0 => |
68731 | 113 |
sessions_structure(name2).timeout compare sessions_structure(name1).timeout match { |
68486
6984a55f3cba
clarified queue ordering: take session descendants into account, notably for "slow" AFP sessions;
wenzelm
parents:
68331
diff
changeset
|
114 |
case 0 => name1 compare name2 |
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
115 |
case ord => ord |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
116 |
} |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
117 |
case ord => ord |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
118 |
} |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
119 |
} |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
120 |
|
65415
8cd54b18b68b
clarified signature: tree structure is not essential;
wenzelm
parents:
65406
diff
changeset
|
121 |
new Queue(graph, SortedSet(names: _*)(Ordering), command_timings) |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
122 |
} |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
123 |
} |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
124 |
|
65314 | 125 |
private class Queue( |
62631 | 126 |
graph: Graph[String, Sessions.Info], |
51220 | 127 |
order: SortedSet[String], |
128 |
val command_timings: String => List[Properties.T]) |
|
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
129 |
{ |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
130 |
def is_inner(name: String): Boolean = !graph.is_maximal(name) |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
131 |
|
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
132 |
def is_empty: Boolean = graph.is_empty |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
133 |
|
51227
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
134 |
def - (name: String): Queue = |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
135 |
new Queue(graph.del_node(name), |
67011 | 136 |
order - name, // FIXME scala-2.10.0 .. 2.12.4 TreeSet problem!? |
51227
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
137 |
command_timings) |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
138 |
|
62631 | 139 |
def dequeue(skip: String => Boolean): Option[(String, Sessions.Info)] = |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
140 |
{ |
51227
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
141 |
val it = order.iterator.dropWhile(name => |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
142 |
skip(name) |
67011 | 143 |
|| !graph.defined(name) // FIXME scala-2.10.0 .. 2.12.4 TreeSet problem!? |
51227
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
144 |
|| !graph.is_minimal(name)) |
48680 | 145 |
if (it.hasNext) { val name = it.next; Some((name, graph.get_node(name))) } |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
146 |
else None |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
147 |
} |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
148 |
} |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
149 |
|
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
150 |
|
65313 | 151 |
/* PIDE protocol handler */ |
152 |
||
65308 | 153 |
/* job: running prover process */ |
48341 | 154 |
|
65308 | 155 |
private class Job(progress: Progress, |
71669 | 156 |
session_name: String, |
65308 | 157 |
val info: Sessions.Info, |
65313 | 158 |
deps: Sessions.Deps, |
65308 | 159 |
store: Sessions.Store, |
71614 | 160 |
do_store: Boolean, |
65308 | 161 |
verbose: Boolean, |
162 |
val numa_node: Option[Int], |
|
163 |
command_timings: List[Properties.T]) |
|
48418 | 164 |
{ |
71895 | 165 |
val options: Options = NUMA.policy_options(info.options, numa_node) |
65312 | 166 |
|
71610 | 167 |
private val sessions_structure = deps.sessions_structure |
70683
8c7706b053c7
find theory files via session structure: much faster Prover IDE startup;
wenzelm
parents:
70671
diff
changeset
|
168 |
|
59445 | 169 |
private val graph_file = Isabelle_System.tmp_file("session_graph", "pdf") |
71677 | 170 |
graphview.Graph_File.write(options, graph_file, deps(session_name).session_graph_display) |
59445 | 171 |
|
68198
6710167e17af
avoid race condition wrt. ISABELLE_TMP, which is removed in Bash.cleanup() before Bash.result(progress_stdout);
wenzelm
parents:
68148
diff
changeset
|
172 |
private val export_tmp_dir = Isabelle_System.tmp_dir("export") |
68289 | 173 |
private val export_consumer = |
71669 | 174 |
Export.consumer(store.open_database(session_name, output = true), cache = store.xz_cache) |
68092
888d35a19866
store exports in session database, with asynchronous / parallel compression;
wenzelm
parents:
68088
diff
changeset
|
175 |
|
62573 | 176 |
private val future_result: Future[Process_Result] = |
71718 | 177 |
Future.thread("build", uninterruptible = true) { |
65308 | 178 |
val parent = info.parent.getOrElse("") |
71645 | 179 |
val base = deps(parent) |
65308 | 180 |
val args_yxml = |
181 |
YXML.string_of_body( |
|
62944
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
182 |
{ |
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
183 |
import XML.Encode._ |
71611
fb6953e77000
eliminated pointless flag (see also 6533ceee4cd7);
wenzelm
parents:
71610
diff
changeset
|
184 |
pair(list(pair(string, int)), pair(list(properties), pair(bool, |
62944
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
185 |
pair(Path.encode, pair(list(pair(Path.encode, Path.encode)), pair(string, |
65307 | 186 |
pair(string, pair(string, pair(string, pair(Path.encode, |
71677 | 187 |
pair(list(pair(Options.encode, list(pair(string, properties)))), |
70683
8c7706b053c7
find theory files via session structure: much faster Prover IDE startup;
wenzelm
parents:
70671
diff
changeset
|
188 |
pair(list(pair(string, properties)), |
67493
c4e9e0c50487
treat sessions as entities with defining position;
wenzelm
parents:
67471
diff
changeset
|
189 |
pair(list(pair(string, string)), |
70683
8c7706b053c7
find theory files via session structure: much faster Prover IDE startup;
wenzelm
parents:
70671
diff
changeset
|
190 |
pair(list(string), pair(list(pair(string, string)), |
71611
fb6953e77000
eliminated pointless flag (see also 6533ceee4cd7);
wenzelm
parents:
71610
diff
changeset
|
191 |
pair(list(string), list(string)))))))))))))))))( |
fb6953e77000
eliminated pointless flag (see also 6533ceee4cd7);
wenzelm
parents:
71610
diff
changeset
|
192 |
(Symbol.codes, (command_timings, (verbose, |
62944
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
193 |
(store.browser_info, (info.document_files, (File.standard_path(graph_file), |
71669 | 194 |
(parent, (info.chapter, (session_name, (Path.current, |
70683
8c7706b053c7
find theory files via session structure: much faster Prover IDE startup;
wenzelm
parents:
70671
diff
changeset
|
195 |
(info.theories, |
8c7706b053c7
find theory files via session structure: much faster Prover IDE startup;
wenzelm
parents:
70671
diff
changeset
|
196 |
(sessions_structure.session_positions, |
8c7706b053c7
find theory files via session structure: much faster Prover IDE startup;
wenzelm
parents:
70671
diff
changeset
|
197 |
(sessions_structure.dest_session_directories, |
8c7706b053c7
find theory files via session structure: much faster Prover IDE startup;
wenzelm
parents:
70671
diff
changeset
|
198 |
(base.doc_names, (base.global_theories.toList, |
71611
fb6953e77000
eliminated pointless flag (see also 6533ceee4cd7);
wenzelm
parents:
71610
diff
changeset
|
199 |
(base.loaded_theories.keys, info.bibtex_entries.map(_.info)))))))))))))))))) |
65308 | 200 |
}) |
201 |
||
202 |
val env = |
|
203 |
Isabelle_System.settings() + |
|
68198
6710167e17af
avoid race condition wrt. ISABELLE_TMP, which is removed in Bash.cleanup() before Bash.result(progress_stdout);
wenzelm
parents:
68148
diff
changeset
|
204 |
("ISABELLE_EXPORT_TMP" -> File.standard_path(export_tmp_dir)) + |
71677 | 205 |
("ISABELLE_ML_DEBUGGER" -> options.bool("ML_debugger").toString) |
65308 | 206 |
|
71669 | 207 |
val is_pure = Sessions.is_pure(session_name) |
71610 | 208 |
|
71639
ec84f542e411
clarified signature of ML_Process vs. Isabelle_Process: proper support for "isabelle build -P -b";
wenzelm
parents:
71632
diff
changeset
|
209 |
val use_prelude = if (is_pure) Thy_Header.ml_roots.map(_._1) else Nil |
ec84f542e411
clarified signature of ML_Process vs. Isabelle_Process: proper support for "isabelle build -P -b";
wenzelm
parents:
71632
diff
changeset
|
210 |
|
71613 | 211 |
val eval_store = |
71639
ec84f542e411
clarified signature of ML_Process vs. Isabelle_Process: proper support for "isabelle build -P -b";
wenzelm
parents:
71632
diff
changeset
|
212 |
if (do_store) { |
71613 | 213 |
(if (info.theories.nonEmpty) List("ML_Heap.share_common_data ()") else Nil) ::: |
214 |
List("ML_Heap.save_child " + |
|
71669 | 215 |
ML_Syntax.print_string_bytes(File.platform_path(store.output_heap(session_name)))) |
71613 | 216 |
} |
71639
ec84f542e411
clarified signature of ML_Process vs. Isabelle_Process: proper support for "isabelle build -P -b";
wenzelm
parents:
71632
diff
changeset
|
217 |
else Nil |
62944
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
218 |
|
71877 | 219 |
if (options.bool("pide_session")) { |
71643 | 220 |
val resources = new Resources(sessions_structure, deps(parent)) |
71971 | 221 |
val session = |
222 |
new Session(options, resources) { |
|
223 |
override val xml_cache: XML.Cache = store.xml_cache |
|
224 |
override val xz_cache: XZ.Cache = store.xz_cache |
|
225 |
} |
|
65313 | 226 |
|
72007 | 227 |
object Build_Session_Errors |
228 |
{ |
|
229 |
private val promise: Promise[List[String]] = Future.promise |
|
230 |
||
231 |
def result: Exn.Result[List[String]] = promise.join_result |
|
232 |
def cancel: Unit = promise.cancel |
|
233 |
def apply(errs: List[String]) |
|
234 |
{ |
|
235 |
try { promise.fulfill(errs) } |
|
236 |
catch { case _: IllegalStateException => } |
|
237 |
} |
|
238 |
} |
|
239 |
||
71648 | 240 |
val stdout = new StringBuilder(1000) |
72002
5c4800f6b25a
more robust protocol for "Timing ..." messages, notably for pide_session=true;
wenzelm
parents:
71992
diff
changeset
|
241 |
val stderr = new StringBuilder(1000) |
71978
3e54088a7119
clarified use of memory: prefer share tree structures over fresh strings;
wenzelm
parents:
71971
diff
changeset
|
242 |
val messages = new mutable.ListBuffer[XML.Elem] |
71652 | 243 |
val command_timings = new mutable.ListBuffer[Properties.T] |
244 |
val theory_timings = new mutable.ListBuffer[Properties.T] |
|
72002
5c4800f6b25a
more robust protocol for "Timing ..." messages, notably for pide_session=true;
wenzelm
parents:
71992
diff
changeset
|
245 |
val session_timings = new mutable.ListBuffer[Properties.T] |
71652 | 246 |
val runtime_statistics = new mutable.ListBuffer[Properties.T] |
247 |
val task_statistics = new mutable.ListBuffer[Properties.T] |
|
71648 | 248 |
|
71970 | 249 |
def fun( |
250 |
name: String, |
|
251 |
acc: mutable.ListBuffer[Properties.T], |
|
252 |
unapply: Properties.T => Option[Properties.T]): (String, Session.Protocol_Function) = |
|
253 |
{ |
|
254 |
name -> ((msg: Prover.Protocol_Output) => |
|
255 |
unapply(msg.properties) match { |
|
256 |
case Some(props) => acc += props; true |
|
257 |
case _ => false |
|
258 |
}) |
|
259 |
} |
|
260 |
||
71672 | 261 |
session.init_protocol_handler(new Session.Protocol_Handler |
262 |
{ |
|
72007 | 263 |
override def exit() { Build_Session_Errors.cancel } |
71672 | 264 |
|
265 |
private def build_session_finished(msg: Prover.Protocol_Output): Boolean = |
|
266 |
{ |
|
71879 | 267 |
val (rc, errors) = |
71672 | 268 |
try { |
71879 | 269 |
val (rc, errs) = |
270 |
{ |
|
271 |
import XML.Decode._ |
|
272 |
pair(int, list(x => x))(Symbol.decode_yxml(msg.text)) |
|
71672 | 273 |
} |
71879 | 274 |
val errors = |
275 |
for (err <- errs) yield { |
|
276 |
val prt = Protocol_Message.expose_no_reports(err) |
|
277 |
Pretty.string_of(prt, metric = Symbol.Metric) |
|
278 |
} |
|
279 |
(rc, errors) |
|
71672 | 280 |
} |
71879 | 281 |
catch { case ERROR(err) => (2, List(err)) } |
282 |
||
283 |
session.protocol_command("Prover.stop", rc.toString) |
|
72007 | 284 |
Build_Session_Errors(errors) |
71672 | 285 |
true |
286 |
} |
|
287 |
||
288 |
private def loading_theory(msg: Prover.Protocol_Output): Boolean = |
|
289 |
msg.properties match { |
|
290 |
case Markup.Loading_Theory(name) => |
|
291 |
progress.theory(Progress.Theory(name, session = session_name)) |
|
292 |
true |
|
293 |
case _ => false |
|
294 |
} |
|
295 |
||
71676
da49285a0adf
more official handling of protocol messages, including export;
wenzelm
parents:
71675
diff
changeset
|
296 |
private def export(msg: Prover.Protocol_Output): Boolean = |
da49285a0adf
more official handling of protocol messages, including export;
wenzelm
parents:
71675
diff
changeset
|
297 |
msg.properties match { |
da49285a0adf
more official handling of protocol messages, including export;
wenzelm
parents:
71675
diff
changeset
|
298 |
case Protocol.Export(args) => |
da49285a0adf
more official handling of protocol messages, including export;
wenzelm
parents:
71675
diff
changeset
|
299 |
export_consumer(session_name, args, msg.bytes) |
da49285a0adf
more official handling of protocol messages, including export;
wenzelm
parents:
71675
diff
changeset
|
300 |
true |
da49285a0adf
more official handling of protocol messages, including export;
wenzelm
parents:
71675
diff
changeset
|
301 |
case _ => false |
da49285a0adf
more official handling of protocol messages, including export;
wenzelm
parents:
71675
diff
changeset
|
302 |
} |
da49285a0adf
more official handling of protocol messages, including export;
wenzelm
parents:
71675
diff
changeset
|
303 |
|
71672 | 304 |
val functions = |
305 |
List( |
|
71673 | 306 |
Markup.Build_Session_Finished.name -> build_session_finished, |
71676
da49285a0adf
more official handling of protocol messages, including export;
wenzelm
parents:
71675
diff
changeset
|
307 |
Markup.Loading_Theory.name -> loading_theory, |
da49285a0adf
more official handling of protocol messages, including export;
wenzelm
parents:
71675
diff
changeset
|
308 |
Markup.EXPORT -> export, |
71970 | 309 |
fun(Markup.Command_Timing.name, command_timings, Markup.Command_Timing.unapply), |
310 |
fun(Markup.Theory_Timing.name, theory_timings, Markup.Theory_Timing.unapply), |
|
72002
5c4800f6b25a
more robust protocol for "Timing ..." messages, notably for pide_session=true;
wenzelm
parents:
71992
diff
changeset
|
311 |
fun(Markup.Session_Timing.name, session_timings, Markup.Session_Timing.unapply), |
71970 | 312 |
fun(Markup.ML_Statistics.name, runtime_statistics, Markup.ML_Statistics.unapply), |
313 |
fun(Markup.Task_Statistics.name, task_statistics, Markup.Task_Statistics.unapply)) |
|
71672 | 314 |
}) |
315 |
||
71676
da49285a0adf
more official handling of protocol messages, including export;
wenzelm
parents:
71675
diff
changeset
|
316 |
session.all_messages += Session.Consumer[Any]("build_session_output") |
da49285a0adf
more official handling of protocol messages, including export;
wenzelm
parents:
71675
diff
changeset
|
317 |
{ |
71648 | 318 |
case msg: Prover.Output => |
319 |
val message = msg.message |
|
320 |
if (msg.is_stdout) { |
|
321 |
stdout ++= Symbol.encode(XML.content(message)) |
|
322 |
} |
|
72002
5c4800f6b25a
more robust protocol for "Timing ..." messages, notably for pide_session=true;
wenzelm
parents:
71992
diff
changeset
|
323 |
else if (msg.is_stderr) { |
5c4800f6b25a
more robust protocol for "Timing ..." messages, notably for pide_session=true;
wenzelm
parents:
71992
diff
changeset
|
324 |
stderr ++= Symbol.encode(XML.content(message)) |
5c4800f6b25a
more robust protocol for "Timing ..." messages, notably for pide_session=true;
wenzelm
parents:
71992
diff
changeset
|
325 |
} |
71648 | 326 |
else if (Protocol.is_exported(message)) { |
71978
3e54088a7119
clarified use of memory: prefer share tree structures over fresh strings;
wenzelm
parents:
71971
diff
changeset
|
327 |
messages += message |
71648 | 328 |
} |
72005
11c46b8e91c0
more robust build_session protocol: allow prover process to terminate/crash without build_session_finished message;
wenzelm
parents:
72002
diff
changeset
|
329 |
else if (msg.is_exit) { |
11c46b8e91c0
more robust build_session protocol: allow prover process to terminate/crash without build_session_finished message;
wenzelm
parents:
72002
diff
changeset
|
330 |
val err = |
11c46b8e91c0
more robust build_session protocol: allow prover process to terminate/crash without build_session_finished message;
wenzelm
parents:
72002
diff
changeset
|
331 |
"Prover terminated" + |
11c46b8e91c0
more robust build_session protocol: allow prover process to terminate/crash without build_session_finished message;
wenzelm
parents:
72002
diff
changeset
|
332 |
(msg.properties match { |
11c46b8e91c0
more robust build_session protocol: allow prover process to terminate/crash without build_session_finished message;
wenzelm
parents:
72002
diff
changeset
|
333 |
case Markup.Process_Result(result) => ": " + result.print_rc |
11c46b8e91c0
more robust build_session protocol: allow prover process to terminate/crash without build_session_finished message;
wenzelm
parents:
72002
diff
changeset
|
334 |
case _ => "" |
11c46b8e91c0
more robust build_session protocol: allow prover process to terminate/crash without build_session_finished message;
wenzelm
parents:
72002
diff
changeset
|
335 |
}) |
72007 | 336 |
Build_Session_Errors(List(err)) |
72005
11c46b8e91c0
more robust build_session protocol: allow prover process to terminate/crash without build_session_finished message;
wenzelm
parents:
72002
diff
changeset
|
337 |
} |
71648 | 338 |
case _ => |
339 |
} |
|
340 |
||
71644 | 341 |
val eval_main = Command_Line.ML_tool("Isabelle_Process.init_build ()" :: eval_store) |
71639
ec84f542e411
clarified signature of ML_Process vs. Isabelle_Process: proper support for "isabelle build -P -b";
wenzelm
parents:
71632
diff
changeset
|
342 |
|
71607 | 343 |
val process = |
71677 | 344 |
Isabelle_Process(session, options, sessions_structure, store, |
71639
ec84f542e411
clarified signature of ML_Process vs. Isabelle_Process: proper support for "isabelle build -P -b";
wenzelm
parents:
71632
diff
changeset
|
345 |
logic = parent, raw_ml_system = is_pure, |
ec84f542e411
clarified signature of ML_Process vs. Isabelle_Process: proper support for "isabelle build -P -b";
wenzelm
parents:
71632
diff
changeset
|
346 |
use_prelude = use_prelude, eval_main = eval_main, |
71668
25ef5c7287a7
proper treatment of startup errors, notably for Pure ROOT.ML;
wenzelm
parents:
71667
diff
changeset
|
347 |
cwd = info.dir.file, env = env) |
65313 | 348 |
|
71668
25ef5c7287a7
proper treatment of startup errors, notably for Pure ROOT.ML;
wenzelm
parents:
71667
diff
changeset
|
349 |
val errors = |
71718 | 350 |
Isabelle_Thread.interrupt_handler(_ => process.terminate) { |
351 |
Exn.capture { process.await_startup } match { |
|
352 |
case Exn.Res(_) => |
|
353 |
session.protocol_command("build_session", args_yxml) |
|
72007 | 354 |
Build_Session_Errors.result |
71718 | 355 |
case Exn.Exn(exn) => Exn.Res(List(Exn.message(exn))) |
356 |
} |
|
71668
25ef5c7287a7
proper treatment of startup errors, notably for Pure ROOT.ML;
wenzelm
parents:
71667
diff
changeset
|
357 |
} |
71607 | 358 |
|
71718 | 359 |
val process_result = |
360 |
Isabelle_Thread.interrupt_handler(_ => process.terminate) { process.await_shutdown } |
|
71652 | 361 |
val process_output = |
71978
3e54088a7119
clarified use of memory: prefer share tree structures over fresh strings;
wenzelm
parents:
71971
diff
changeset
|
362 |
stdout.toString :: |
3e54088a7119
clarified use of memory: prefer share tree structures over fresh strings;
wenzelm
parents:
71971
diff
changeset
|
363 |
messages.toList.map(message => |
3e54088a7119
clarified use of memory: prefer share tree structures over fresh strings;
wenzelm
parents:
71971
diff
changeset
|
364 |
Symbol.encode(Protocol.message_text(List(message), metric = Symbol.Metric))) ::: |
71652 | 365 |
command_timings.toList.map(Protocol.Command_Timing_Marker.apply) ::: |
366 |
theory_timings.toList.map(Protocol.Theory_Timing_Marker.apply) ::: |
|
72012 | 367 |
session_timings.toList.map(Protocol.Session_Timing_Marker.apply) ::: |
71652 | 368 |
runtime_statistics.toList.map(Protocol.ML_Statistics_Marker.apply) ::: |
369 |
task_statistics.toList.map(Protocol.Task_Statistics_Marker.apply) |
|
71648 | 370 |
|
72002
5c4800f6b25a
more robust protocol for "Timing ..." messages, notably for pide_session=true;
wenzelm
parents:
71992
diff
changeset
|
371 |
val result = |
5c4800f6b25a
more robust protocol for "Timing ..." messages, notably for pide_session=true;
wenzelm
parents:
71992
diff
changeset
|
372 |
process_result.output(process_output).error(Library.trim_line(stderr.toString)) |
71718 | 373 |
|
374 |
errors match { |
|
375 |
case Exn.Res(Nil) => result |
|
376 |
case Exn.Res(errs) => |
|
377 |
result.error_rc.output( |
|
378 |
errs.flatMap(s => split_lines(Output.error_message_text(s))) ::: |
|
379 |
errs.map(Protocol.Error_Message_Marker.apply)) |
|
380 |
case Exn.Exn(Exn.Interrupt()) => |
|
381 |
if (result.ok) result.copy(rc = Exn.Interrupt.return_code) else result |
|
382 |
case Exn.Exn(exn) => throw exn |
|
65313 | 383 |
} |
65308 | 384 |
} |
385 |
else { |
|
386 |
val args_file = Isabelle_System.tmp_file("build") |
|
387 |
File.write(args_file, args_yxml) |
|
388 |
||
71639
ec84f542e411
clarified signature of ML_Process vs. Isabelle_Process: proper support for "isabelle build -P -b";
wenzelm
parents:
71632
diff
changeset
|
389 |
val eval_build = |
ec84f542e411
clarified signature of ML_Process vs. Isabelle_Process: proper support for "isabelle build -P -b";
wenzelm
parents:
71632
diff
changeset
|
390 |
"Build.build " + ML_Syntax.print_string_bytes(File.standard_path(args_file)) |
ec84f542e411
clarified signature of ML_Process vs. Isabelle_Process: proper support for "isabelle build -P -b";
wenzelm
parents:
71632
diff
changeset
|
391 |
val eval_main = Command_Line.ML_tool(eval_build :: eval_store) |
64265 | 392 |
|
65308 | 393 |
val process = |
71677 | 394 |
ML_Process(options, deps.sessions_structure, store, |
71639
ec84f542e411
clarified signature of ML_Process vs. Isabelle_Process: proper support for "isabelle build -P -b";
wenzelm
parents:
71632
diff
changeset
|
395 |
logic = parent, raw_ml_system = is_pure, |
ec84f542e411
clarified signature of ML_Process vs. Isabelle_Process: proper support for "isabelle build -P -b";
wenzelm
parents:
71632
diff
changeset
|
396 |
use_prelude = use_prelude, eval_main = eval_main, |
ec84f542e411
clarified signature of ML_Process vs. Isabelle_Process: proper support for "isabelle build -P -b";
wenzelm
parents:
71632
diff
changeset
|
397 |
cwd = info.dir.file, env = env, cleanup = () => args_file.delete) |
64265 | 398 |
|
71718 | 399 |
Isabelle_Thread.interrupt_handler(_ => process.terminate) { |
400 |
process.result( |
|
401 |
progress_stdout = |
|
402 |
{ |
|
403 |
case Protocol.Loading_Theory_Marker(theory) => |
|
404 |
progress.theory(Progress.Theory(theory, session = session_name)) |
|
405 |
case Protocol.Export.Marker((args, path)) => |
|
406 |
val body = |
|
407 |
try { Bytes.read(path) } |
|
408 |
catch { |
|
409 |
case ERROR(msg) => |
|
410 |
error("Failed to read export " + quote(args.compound_name) + "\n" + msg) |
|
411 |
} |
|
412 |
path.file.delete |
|
413 |
export_consumer(session_name, args, body) |
|
414 |
case _ => |
|
415 |
}, |
|
416 |
progress_limit = |
|
417 |
options.int("process_output_limit") match { |
|
418 |
case 0 => None |
|
419 |
case m => Some(m * 1000000L) |
|
420 |
}, |
|
421 |
strict = false) |
|
422 |
} |
|
65308 | 423 |
} |
50845 | 424 |
} |
48674 | 425 |
|
62572 | 426 |
def terminate: Unit = future_result.cancel |
427 |
def is_finished: Boolean = future_result.is_finished |
|
48674 | 428 |
|
56779 | 429 |
private val timeout_request: Option[Event_Timer.Request] = |
430 |
{ |
|
62569 | 431 |
if (info.timeout > Time.zero) |
66943 | 432 |
Some(Event_Timer.request(Time.now() + info.timeout) { terminate }) |
48674 | 433 |
else None |
56779 | 434 |
} |
48674 | 435 |
|
68217 | 436 |
def join: (Process_Result, Option[String]) = |
50845 | 437 |
{ |
68217 | 438 |
val result0 = future_result.join |
439 |
val result1 = |
|
71601 | 440 |
export_consumer.shutdown(close = true).map(Output.error_message_text) match { |
68217 | 441 |
case Nil => result0 |
68928 | 442 |
case errs => result0.errors(errs).error_rc |
68092
888d35a19866
store exports in session database, with asynchronous / parallel compression;
wenzelm
parents:
68088
diff
changeset
|
443 |
} |
50845 | 444 |
|
68198
6710167e17af
avoid race condition wrt. ISABELLE_TMP, which is removed in Bash.cleanup() before Bash.result(progress_stdout);
wenzelm
parents:
68148
diff
changeset
|
445 |
Isabelle_System.rm_tree(export_tmp_dir) |
6710167e17af
avoid race condition wrt. ISABELLE_TMP, which is removed in Bash.cleanup() before Bash.result(progress_stdout);
wenzelm
parents:
68148
diff
changeset
|
446 |
|
69811
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
447 |
if (result1.ok) |
71669 | 448 |
Present.finish(progress, store.browser_info, graph_file, info, session_name) |
61372 | 449 |
|
59445 | 450 |
graph_file.delete |
66943 | 451 |
|
452 |
val was_timeout = |
|
453 |
timeout_request match { |
|
454 |
case None => false |
|
455 |
case Some(request) => !request.cancel |
|
456 |
} |
|
48674 | 457 |
|
68217 | 458 |
val result2 = |
459 |
if (result1.interrupted) { |
|
460 |
if (was_timeout) result1.error(Output.error_message_text("Timeout")).was_timeout |
|
461 |
else result1.error(Output.error_message_text("Interrupt")) |
|
462 |
} |
|
463 |
else result1 |
|
464 |
||
465 |
val heap_digest = |
|
71669 | 466 |
if (result2.ok && do_store && store.output_heap(session_name).is_file) |
467 |
Some(Sessions.write_heap_digest(store.output_heap(session_name))) |
|
68217 | 468 |
else None |
469 |
||
470 |
(result2, heap_digest) |
|
48674 | 471 |
} |
48364 | 472 |
} |
473 |
||
48424 | 474 |
|
62631 | 475 |
|
62641 | 476 |
/** build with results **/ |
48424 | 477 |
|
63996 | 478 |
class Results private[Build](results: Map[String, (Option[Process_Result], Sessions.Info)]) |
62403 | 479 |
{ |
480 |
def sessions: Set[String] = results.keySet |
|
63082
6af03422535a
expose Sessions.Info in Build.Results
Lars Hupel <lars.hupel@mytum.de>
parents:
62946
diff
changeset
|
481 |
def cancelled(name: String): Boolean = results(name)._1.isEmpty |
6af03422535a
expose Sessions.Info in Build.Results
Lars Hupel <lars.hupel@mytum.de>
parents:
62946
diff
changeset
|
482 |
def apply(name: String): Process_Result = results(name)._1.getOrElse(Process_Result(1)) |
6af03422535a
expose Sessions.Info in Build.Results
Lars Hupel <lars.hupel@mytum.de>
parents:
62946
diff
changeset
|
483 |
def info(name: String): Sessions.Info = results(name)._2 |
71601 | 484 |
val rc: Int = |
65253 | 485 |
(0 /: results.iterator.map( |
486 |
{ case (_, (Some(r), _)) => r.rc case (_, (None, _)) => 1 }))(_ max _) |
|
62641 | 487 |
def ok: Boolean = rc == 0 |
62406 | 488 |
|
489 |
override def toString: String = rc.toString |
|
62403 | 490 |
} |
491 |
||
62641 | 492 |
def build( |
50404
898cac1dad5e
avoid startup within GUI thread -- it is only required later for dialog;
wenzelm
parents:
50367
diff
changeset
|
493 |
options: Options, |
71981 | 494 |
selection: Sessions.Selection = Sessions.Selection.empty, |
71726
a5fda30edae2
clarified signature: more uniform treatment of stopped/interrupted state;
wenzelm
parents:
71718
diff
changeset
|
495 |
progress: Progress = new Progress, |
65832
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
496 |
check_unknown_files: Boolean = false, |
48511
37999ee01156
remove old output heaps, to ensure that result is valid wrt. check_stamps;
wenzelm
parents:
48509
diff
changeset
|
497 |
build_heap: Boolean = false, |
48595 | 498 |
clean_build: Boolean = false, |
56890 | 499 |
dirs: List[Path] = Nil, |
500 |
select_dirs: List[Path] = Nil, |
|
66968
9991671c98aa
allow to augment session context via explicit session infos;
wenzelm
parents:
66962
diff
changeset
|
501 |
infos: List[Sessions.Info] = Nil, |
64265 | 502 |
numa_shuffling: Boolean = false, |
48509 | 503 |
max_jobs: Int = 1, |
48903 | 504 |
list_files: Boolean = false, |
59891
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
505 |
check_keywords: Set[String] = Set.empty, |
66841 | 506 |
fresh_build: Boolean = false, |
48509 | 507 |
no_build: Boolean = false, |
66745 | 508 |
soft_build: Boolean = false, |
48509 | 509 |
verbose: Boolean = false, |
71896
ce06d6456cc8
clarified signature --- fit within limit of 22 arguments;
wenzelm
parents:
71895
diff
changeset
|
510 |
export_files: Boolean = false): Results = |
63224 | 511 |
{ |
71677 | 512 |
val build_options = |
71679
eeaa4021f080
proper "editor_tracing_messages=0" as in "isabelle dump";
wenzelm
parents:
71677
diff
changeset
|
513 |
options + |
eeaa4021f080
proper "editor_tracing_messages=0" as in "isabelle dump";
wenzelm
parents:
71677
diff
changeset
|
514 |
"ML_statistics" + |
eeaa4021f080
proper "editor_tracing_messages=0" as in "isabelle dump";
wenzelm
parents:
71677
diff
changeset
|
515 |
"completion_limit=0" + |
eeaa4021f080
proper "editor_tracing_messages=0" as in "isabelle dump";
wenzelm
parents:
71677
diff
changeset
|
516 |
"editor_tracing_messages=0" + |
eeaa4021f080
proper "editor_tracing_messages=0" as in "isabelle dump";
wenzelm
parents:
71677
diff
changeset
|
517 |
"pide_reports=false" |
51220 | 518 |
|
69854
cc0b3e177b49
system option "system_heaps" supersedes various command-line options for "system build mode";
wenzelm
parents:
69811
diff
changeset
|
519 |
val store = Sessions.store(build_options) |
66745 | 520 |
|
69369
6ecc85955e04
prefer "Isabelle DejaVu Sans", even for headless batch-build (session_graph.pdf);
wenzelm
parents:
68957
diff
changeset
|
521 |
Isabelle_Fonts.init() |
6ecc85955e04
prefer "Isabelle DejaVu Sans", even for headless batch-build (session_graph.pdf);
wenzelm
parents:
68957
diff
changeset
|
522 |
|
66745 | 523 |
|
524 |
/* session selection and dependencies */ |
|
65422 | 525 |
|
66961 | 526 |
val full_sessions = |
67026 | 527 |
Sessions.load_structure(build_options, dirs = dirs, select_dirs = select_dirs, infos = infos) |
65422 | 528 |
|
71669 | 529 |
def sources_stamp(deps: Sessions.Deps, session_name: String): String = |
66749
0445cfaf6132
more compact (second-order) digest for 10^2..10^3 source files, with slightly increased risk of collisions;
wenzelm
parents:
66747
diff
changeset
|
530 |
{ |
0445cfaf6132
more compact (second-order) digest for 10^2..10^3 source files, with slightly increased risk of collisions;
wenzelm
parents:
66747
diff
changeset
|
531 |
val digests = |
71669 | 532 |
full_sessions(session_name).meta_digest :: |
533 |
deps.sources(session_name) ::: |
|
534 |
deps.imported_sources(session_name) |
|
66749
0445cfaf6132
more compact (second-order) digest for 10^2..10^3 source files, with slightly increased risk of collisions;
wenzelm
parents:
66747
diff
changeset
|
535 |
SHA1.digest(cat_lines(digests.map(_.toString).sorted)).toString |
0445cfaf6132
more compact (second-order) digest for 10^2..10^3 source files, with slightly increased risk of collisions;
wenzelm
parents:
66747
diff
changeset
|
536 |
} |
66745 | 537 |
|
68204 | 538 |
val deps = |
66745 | 539 |
{ |
540 |
val deps0 = |
|
71896
ce06d6456cc8
clarified signature --- fit within limit of 22 arguments;
wenzelm
parents:
71895
diff
changeset
|
541 |
Sessions.deps(full_sessions.selection(selection), |
66962
e1bde71bace6
clarified signature: global_theories is always required;
wenzelm
parents:
66961
diff
changeset
|
542 |
progress = progress, inlined_files = true, verbose = verbose, |
e1bde71bace6
clarified signature: global_theories is always required;
wenzelm
parents:
66961
diff
changeset
|
543 |
list_files = list_files, check_keywords = check_keywords).check_errors |
66745 | 544 |
|
66841 | 545 |
if (soft_build && !fresh_build) { |
66745 | 546 |
val outdated = |
68204 | 547 |
deps0.sessions_structure.build_topological_order.flatMap(name => |
68221 | 548 |
store.access_database(name) match { |
68214 | 549 |
case Some(db) => |
68216 | 550 |
using(db)(store.read_build(_, name)) match { |
66745 | 551 |
case Some(build) |
66747 | 552 |
if build.ok && build.sources == sources_stamp(deps0, name) => None |
66745 | 553 |
case _ => Some(name) |
554 |
} |
|
555 |
case None => Some(name) |
|
556 |
}) |
|
68204 | 557 |
|
558 |
Sessions.deps(full_sessions.selection(Sessions.Selection(sessions = outdated)), |
|
70671
cb1776c8e216
clarified signature: retain global session information, unaffected by later restriction;
wenzelm
parents:
70509
diff
changeset
|
559 |
progress = progress, inlined_files = true).check_errors |
66745 | 560 |
} |
68204 | 561 |
else deps0 |
66745 | 562 |
} |
563 |
||
564 |
||
565 |
/* check unknown files */ |
|
48504
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
566 |
|
65832
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
567 |
if (check_unknown_files) { |
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
568 |
val source_files = |
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
569 |
(for { |
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
570 |
(_, base) <- deps.session_bases.iterator |
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
571 |
(path, _) <- base.sources.iterator |
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
572 |
} yield path).toList |
67098 | 573 |
val exclude_files = List(Path.explode("$POLYML_EXE")).map(_.canonical_file) |
574 |
val unknown_files = |
|
67782
7e223a05e6d8
clarified notion of unknown files: ignore files outside of a Mercurial repository;
wenzelm
parents:
67493
diff
changeset
|
575 |
Mercurial.check_files(source_files)._2. |
67098 | 576 |
filterNot(path => exclude_files.contains(path.canonical_file)) |
65832
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
577 |
if (unknown_files.nonEmpty) { |
67782
7e223a05e6d8
clarified notion of unknown files: ignore files outside of a Mercurial repository;
wenzelm
parents:
67493
diff
changeset
|
578 |
progress.echo_warning("Unknown files (not part of the underlying Mercurial repository):" + |
65832
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
579 |
unknown_files.map(path => path.expand.implode).sorted.mkString("\n ", "\n ", "")) |
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
580 |
} |
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
581 |
} |
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
582 |
|
51220 | 583 |
|
584 |
/* main build process */ |
|
585 |
||
68204 | 586 |
val queue = Queue(progress, deps.sessions_structure, store) |
65289 | 587 |
|
68219 | 588 |
store.prepare_output_dir() |
48373 | 589 |
|
48595 | 590 |
if (clean_build) { |
71896
ce06d6456cc8
clarified signature --- fit within limit of 22 arguments;
wenzelm
parents:
71895
diff
changeset
|
591 |
for (name <- full_sessions.imports_descendants(full_sessions.imports_selection(selection))) { |
68220
8fc4e3d1df86
clarified store.clean_output: cleanup user_output_dir even in system_mode;
wenzelm
parents:
68219
diff
changeset
|
592 |
val (relevant, ok) = store.clean_output(name) |
8fc4e3d1df86
clarified store.clean_output: cleanup user_output_dir even in system_mode;
wenzelm
parents:
68219
diff
changeset
|
593 |
if (relevant) { |
8fc4e3d1df86
clarified store.clean_output: cleanup user_output_dir even in system_mode;
wenzelm
parents:
68219
diff
changeset
|
594 |
if (ok) progress.echo("Cleaned " + name) |
8fc4e3d1df86
clarified store.clean_output: cleanup user_output_dir even in system_mode;
wenzelm
parents:
68219
diff
changeset
|
595 |
else progress.echo(name + " FAILED to clean") |
8fc4e3d1df86
clarified store.clean_output: cleanup user_output_dir even in system_mode;
wenzelm
parents:
68219
diff
changeset
|
596 |
} |
48595 | 597 |
} |
598 |
} |
|
599 |
||
48425 | 600 |
// scheduler loop |
63082
6af03422535a
expose Sessions.Info in Build.Results
Lars Hupel <lars.hupel@mytum.de>
parents:
62946
diff
changeset
|
601 |
case class Result( |
66594 | 602 |
current: Boolean, |
68213 | 603 |
heap_digest: Option[String], |
66594 | 604 |
process: Option[Process_Result], |
605 |
info: Sessions.Info) |
|
62402 | 606 |
{ |
607 |
def ok: Boolean = |
|
608 |
process match { |
|
609 |
case None => false |
|
610 |
case Some(res) => res.rc == 0 |
|
611 |
} |
|
612 |
} |
|
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
613 |
|
56837
5a598f1eecfd
more robust interrupt handling for Scala_Console, which uses JVM Thread.interrupt instead of POSIX SIGINT;
wenzelm
parents:
56831
diff
changeset
|
614 |
def sleep() |
5a598f1eecfd
more robust interrupt handling for Scala_Console, which uses JVM Thread.interrupt instead of POSIX SIGINT;
wenzelm
parents:
56831
diff
changeset
|
615 |
{ |
71727
0cb14b7455ee
more careful handling of interrupts, notably for Isabelle/jEdit Scala Console;
wenzelm
parents:
71726
diff
changeset
|
616 |
Isabelle_Thread.interrupt_handler(_ => progress.stop) { Time.seconds(0.5).sleep } |
56837
5a598f1eecfd
more robust interrupt handling for Scala_Console, which uses JVM Thread.interrupt instead of POSIX SIGINT;
wenzelm
parents:
56831
diff
changeset
|
617 |
} |
50366 | 618 |
|
64265 | 619 |
val numa_nodes = new NUMA.Nodes(numa_shuffling) |
620 |
||
48425 | 621 |
@tailrec def loop( |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
622 |
pending: Queue, |
62628 | 623 |
running: Map[String, (List[String], Job)], |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
624 |
results: Map[String, Result]): Map[String, Result] = |
48425 | 625 |
{ |
64265 | 626 |
def used_node(i: Int): Boolean = |
627 |
running.iterator.exists( |
|
628 |
{ case (_, (_, job)) => job.numa_node.isDefined && job.numa_node.get == i }) |
|
629 |
||
48425 | 630 |
if (pending.is_empty) results |
51253 | 631 |
else { |
71726
a5fda30edae2
clarified signature: more uniform treatment of stopped/interrupted state;
wenzelm
parents:
71718
diff
changeset
|
632 |
if (progress.stopped) { |
51253 | 633 |
for ((_, (_, job)) <- running) job.terminate |
71726
a5fda30edae2
clarified signature: more uniform treatment of stopped/interrupted state;
wenzelm
parents:
71718
diff
changeset
|
634 |
} |
51253 | 635 |
|
48674 | 636 |
running.find({ case (_, (_, job)) => job.is_finished }) match { |
71669 | 637 |
case Some((session_name, (input_heaps, job))) => |
50367 | 638 |
//{{{ finish job |
48424 | 639 |
|
68217 | 640 |
val (process_result, heap_digest) = job.join |
48373 | 641 |
|
71623
b3bddebe44ca
clarified signature: more explicit type Protocol_Message.Marker;
wenzelm
parents:
71614
diff
changeset
|
642 |
val log_lines = process_result.out_lines.filterNot(Protocol_Message.Marker.test) |
62404
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
643 |
val process_result_tail = |
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
644 |
{ |
62409
e391528eff3b
proper option process_output_tail, more generous default;
wenzelm
parents:
62406
diff
changeset
|
645 |
val tail = job.info.options.int("process_output_tail") |
62632 | 646 |
process_result.copy( |
647 |
out_lines = |
|
71669 | 648 |
"(see also " + store.output_log(session_name).file.toString + ")" :: |
65294 | 649 |
(if (tail == 0) log_lines else log_lines.drop(log_lines.length - tail max 0))) |
62404
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
650 |
} |
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
651 |
|
66666
1a620647285c
clarified messages: after writing all files (see also 27f90319a499 and 57c85c83c11b);
wenzelm
parents:
66595
diff
changeset
|
652 |
|
1a620647285c
clarified messages: after writing all files (see also 27f90319a499 and 57c85c83c11b);
wenzelm
parents:
66595
diff
changeset
|
653 |
// write log file |
68217 | 654 |
if (process_result.ok) { |
71669 | 655 |
File.write_gzip(store.output_log_gz(session_name), terminate_lines(log_lines)) |
68217 | 656 |
} |
71669 | 657 |
else File.write(store.output_log(session_name), terminate_lines(log_lines)) |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
658 |
|
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
659 |
// write database |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
660 |
{ |
66944
05df740cb54b
more informative timeout message, notably for build_status;
wenzelm
parents:
66943
diff
changeset
|
661 |
val build_log = |
71669 | 662 |
Build_Log.Log_File(session_name, process_result.out_lines). |
66944
05df740cb54b
more informative timeout message, notably for build_status;
wenzelm
parents:
66943
diff
changeset
|
663 |
parse_session_info( |
05df740cb54b
more informative timeout message, notably for build_status;
wenzelm
parents:
66943
diff
changeset
|
664 |
command_timings = true, |
05df740cb54b
more informative timeout message, notably for build_status;
wenzelm
parents:
66943
diff
changeset
|
665 |
theory_timings = true, |
05df740cb54b
more informative timeout message, notably for build_status;
wenzelm
parents:
66943
diff
changeset
|
666 |
ml_statistics = true, |
05df740cb54b
more informative timeout message, notably for build_status;
wenzelm
parents:
66943
diff
changeset
|
667 |
task_statistics = true) |
05df740cb54b
more informative timeout message, notably for build_status;
wenzelm
parents:
66943
diff
changeset
|
668 |
|
71669 | 669 |
using(store.open_database(session_name, output = true))(db => |
670 |
store.write_session_info(db, session_name, |
|
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
671 |
build_log = |
66944
05df740cb54b
more informative timeout message, notably for build_status;
wenzelm
parents:
66943
diff
changeset
|
672 |
if (process_result.timeout) build_log.error("Timeout") else build_log, |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
673 |
build = |
71669 | 674 |
Session_Info(sources_stamp(deps, session_name), input_heaps, heap_digest, |
66744 | 675 |
process_result.rc))) |
65291
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
676 |
} |
57c85c83c11b
maintain persistent session info in SQLite database instead of log file;
wenzelm
parents:
65289
diff
changeset
|
677 |
|
66666
1a620647285c
clarified messages: after writing all files (see also 27f90319a499 and 57c85c83c11b);
wenzelm
parents:
66595
diff
changeset
|
678 |
// messages |
71601 | 679 |
process_result.err_lines.foreach(progress.echo) |
66666
1a620647285c
clarified messages: after writing all files (see also 27f90319a499 and 57c85c83c11b);
wenzelm
parents:
66595
diff
changeset
|
680 |
|
1a620647285c
clarified messages: after writing all files (see also 27f90319a499 and 57c85c83c11b);
wenzelm
parents:
66595
diff
changeset
|
681 |
if (process_result.ok) |
71669 | 682 |
progress.echo( |
683 |
"Finished " + session_name + " (" + process_result.timing.message_resources + ")") |
|
66666
1a620647285c
clarified messages: after writing all files (see also 27f90319a499 and 57c85c83c11b);
wenzelm
parents:
66595
diff
changeset
|
684 |
else { |
71669 | 685 |
progress.echo(session_name + " FAILED") |
66666
1a620647285c
clarified messages: after writing all files (see also 27f90319a499 and 57c85c83c11b);
wenzelm
parents:
66595
diff
changeset
|
686 |
if (!process_result.interrupted) progress.echo(process_result_tail.out) |
1a620647285c
clarified messages: after writing all files (see also 27f90319a499 and 57c85c83c11b);
wenzelm
parents:
66595
diff
changeset
|
687 |
} |
1a620647285c
clarified messages: after writing all files (see also 27f90319a499 and 57c85c83c11b);
wenzelm
parents:
66595
diff
changeset
|
688 |
|
71669 | 689 |
loop(pending - session_name, running - session_name, |
690 |
results + |
|
691 |
(session_name -> Result(false, heap_digest, Some(process_result_tail), job.info))) |
|
50367 | 692 |
//}}} |
60215 | 693 |
case None if running.size < (max_jobs max 1) => |
50367 | 694 |
//{{{ check/start next job |
71601 | 695 |
pending.dequeue(running.isDefinedAt) match { |
71669 | 696 |
case Some((session_name, info)) => |
66848 | 697 |
val ancestor_results = |
71669 | 698 |
deps.sessions_structure.build_requirements(List(session_name)). |
699 |
filterNot(_ == session_name).map(results(_)) |
|
68213 | 700 |
val ancestor_heaps = ancestor_results.flatMap(_.heap_digest) |
62628 | 701 |
|
71669 | 702 |
val do_store = |
703 |
build_heap || Sessions.is_pure(session_name) || queue.is_inner(session_name) |
|
48547 | 704 |
|
68213 | 705 |
val (current, heap_digest) = |
48547 | 706 |
{ |
71669 | 707 |
store.access_database(session_name) match { |
68212
5a59fded83c7
clarified heap vs. database operations: discontinued correlation of directory;
wenzelm
parents:
68209
diff
changeset
|
708 |
case Some(db) => |
71669 | 709 |
using(db)(store.read_build(_, session_name)) match { |
68216 | 710 |
case Some(build) => |
71669 | 711 |
val heap_digest = store.find_heap_digest(session_name) |
68216 | 712 |
val current = |
713 |
!fresh_build && |
|
714 |
build.ok && |
|
71669 | 715 |
build.sources == sources_stamp(deps, session_name) && |
68216 | 716 |
build.input_heaps == ancestor_heaps && |
717 |
build.output_heap == heap_digest && |
|
71614 | 718 |
!(do_store && heap_digest.isEmpty) |
68216 | 719 |
(current, heap_digest) |
720 |
case None => (false, None) |
|
721 |
} |
|
62636 | 722 |
case None => (false, None) |
48504
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
723 |
} |
48547 | 724 |
} |
62628 | 725 |
val all_current = current && ancestor_results.forall(_.current) |
48528
784c6f63d79c
proper all_current, which regards parent status as well;
wenzelm
parents:
48511
diff
changeset
|
726 |
|
48547 | 727 |
if (all_current) |
71669 | 728 |
loop(pending - session_name, running, |
729 |
results + |
|
730 |
(session_name -> Result(true, heap_digest, Some(Process_Result(0)), info))) |
|
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
731 |
else if (no_build) { |
71894 | 732 |
progress.echo_if(verbose, "Skipping " + session_name + " ...") |
71669 | 733 |
loop(pending - session_name, running, |
734 |
results + |
|
735 |
(session_name -> Result(false, heap_digest, Some(Process_Result(1)), info))) |
|
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
736 |
} |
62628 | 737 |
else if (ancestor_results.forall(_.ok) && !progress.stopped) { |
71669 | 738 |
progress.echo((if (do_store) "Building " else "Running ") + session_name + " ...") |
68086
9e1c670301b8
cleanup session output before starting build job;
wenzelm
parents:
67852
diff
changeset
|
739 |
|
71669 | 740 |
store.clean_output(session_name) |
741 |
using(store.open_database(session_name, output = true))( |
|
742 |
store.init_session_info(_, session_name)) |
|
68086
9e1c670301b8
cleanup session output before starting build job;
wenzelm
parents:
67852
diff
changeset
|
743 |
|
71601 | 744 |
val numa_node = numa_nodes.next(used_node) |
51220 | 745 |
val job = |
71669 | 746 |
new Job(progress, session_name, info, deps, store, do_store, verbose, |
747 |
numa_node, queue.command_timings(session_name)) |
|
748 |
loop(pending, running + (session_name -> (ancestor_heaps, job)), results) |
|
48547 | 749 |
} |
750 |
else { |
|
71669 | 751 |
progress.echo(session_name + " CANCELLED") |
752 |
loop(pending - session_name, running, |
|
753 |
results + (session_name -> Result(false, heap_digest, None, info))) |
|
48547 | 754 |
} |
755 |
case None => sleep(); loop(pending, running, results) |
|
48425 | 756 |
} |
50367 | 757 |
///}}} |
48425 | 758 |
case None => sleep(); loop(pending, running, results) |
48373 | 759 |
} |
51253 | 760 |
} |
48425 | 761 |
} |
762 |
||
51220 | 763 |
|
764 |
/* build results */ |
|
765 |
||
62641 | 766 |
val results0 = |
48583 | 767 |
if (deps.is_empty) { |
65827 | 768 |
progress.echo_warning("Nothing to build") |
50707
5b2bf7611662
maintain session index on Scala side, for more determistic results;
wenzelm
parents:
50686
diff
changeset
|
769 |
Map.empty[String, Result] |
48583 | 770 |
} |
71727
0cb14b7455ee
more careful handling of interrupts, notably for Isabelle/jEdit Scala Console;
wenzelm
parents:
71726
diff
changeset
|
771 |
else Isabelle_Thread.uninterruptible { loop(queue, Map.empty, Map.empty) } |
48583 | 772 |
|
62641 | 773 |
val results = |
64265 | 774 |
new Results( |
775 |
(for ((name, result) <- results0.iterator) |
|
776 |
yield (name, (result.process, result.info))).toMap) |
|
62641 | 777 |
|
69811
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
778 |
if (export_files) { |
71896
ce06d6456cc8
clarified signature --- fit within limit of 22 arguments;
wenzelm
parents:
71895
diff
changeset
|
779 |
for (name <- full_sessions.imports_selection(selection).iterator if results(name).ok) { |
69811
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
780 |
val info = results.info(name) |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
781 |
if (info.export_files.nonEmpty) { |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
782 |
progress.echo("Exporting " + info.name + " ...") |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
783 |
for ((dir, prune, pats) <- info.export_files) { |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
784 |
Export.export_files(store, name, info.dir + dir, |
71726
a5fda30edae2
clarified signature: more uniform treatment of stopped/interrupted state;
wenzelm
parents:
71718
diff
changeset
|
785 |
progress = if (verbose) progress else new Progress, |
69811
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
786 |
export_prune = prune, |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
787 |
export_patterns = pats) |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
788 |
} |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
789 |
} |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
790 |
} |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
791 |
} |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
792 |
|
62641 | 793 |
if (results.rc != 0 && (verbose || !no_build)) { |
794 |
val unfinished = |
|
795 |
(for { |
|
796 |
name <- results.sessions.iterator |
|
797 |
if !results(name).ok |
|
798 |
} yield name).toList.sorted |
|
799 |
progress.echo("Unfinished session(s): " + commas(unfinished)) |
|
800 |
} |
|
801 |
||
51418
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
802 |
|
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
803 |
/* global browser info */ |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
804 |
|
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
805 |
if (!no_build) { |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
806 |
val browser_chapters = |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
807 |
(for { |
62641 | 808 |
(name, result) <- results0.iterator |
62944
3ee643c5ed00
more standard session build process, including browser_info;
wenzelm
parents:
62902
diff
changeset
|
809 |
if result.ok |
65415
8cd54b18b68b
clarified signature: tree structure is not essential;
wenzelm
parents:
65406
diff
changeset
|
810 |
info = full_sessions(name) |
51418
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
811 |
if info.options.bool("browser_info") |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
812 |
} yield (info.chapter, (name, info.description))).toList.groupBy(_._1). |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
813 |
map({ case (chapter, es) => (chapter, es.map(_._2)) }).filterNot(_._2.isEmpty) |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
814 |
|
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
815 |
for ((chapter, entries) <- browser_chapters) |
62632 | 816 |
Present.update_chapter_index(store.browser_info, chapter, entries) |
51418
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
817 |
|
62632 | 818 |
if (browser_chapters.nonEmpty) Present.make_global_index(store.browser_info) |
51418
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
819 |
} |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
820 |
|
62641 | 821 |
results |
48341 | 822 |
} |
823 |
||
824 |
||
62833 | 825 |
/* Isabelle tool wrapper */ |
48341 | 826 |
|
62833 | 827 |
val isabelle_tool = Isabelle_Tool("build", "build and manage Isabelle sessions", args => |
48341 | 828 |
{ |
62833 | 829 |
val build_options = Word.explode(Isabelle_System.getenv("ISABELLE_BUILD_OPTIONS")) |
62590 | 830 |
|
66737
2edc0c42c883
option -B for "isabelle build" and "isabelle imports";
wenzelm
parents:
66736
diff
changeset
|
831 |
var base_sessions: List[String] = Nil |
62833 | 832 |
var select_dirs: List[Path] = Nil |
64265 | 833 |
var numa_shuffling = false |
62833 | 834 |
var requirements = false |
66745 | 835 |
var soft_build = false |
62833 | 836 |
var exclude_session_groups: List[String] = Nil |
837 |
var all_sessions = false |
|
838 |
var build_heap = false |
|
839 |
var clean_build = false |
|
840 |
var dirs: List[Path] = Nil |
|
69811
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
841 |
var export_files = false |
66841 | 842 |
var fresh_build = false |
62833 | 843 |
var session_groups: List[String] = Nil |
844 |
var max_jobs = 1 |
|
845 |
var check_keywords: Set[String] = Set.empty |
|
846 |
var list_files = false |
|
847 |
var no_build = false |
|
67847 | 848 |
var options = Options.init(opts = build_options) |
62833 | 849 |
var verbose = false |
850 |
var exclude_sessions: List[String] = Nil |
|
62590 | 851 |
|
62833 | 852 |
val getopts = Getopts(""" |
62590 | 853 |
Usage: isabelle build [OPTIONS] [SESSIONS ...] |
854 |
||
855 |
Options are: |
|
66737
2edc0c42c883
option -B for "isabelle build" and "isabelle imports";
wenzelm
parents:
66736
diff
changeset
|
856 |
-B NAME include session NAME and all descendants |
62590 | 857 |
-D DIR include session directory and select its sessions |
64265 | 858 |
-N cyclic shuffling of NUMA CPU nodes (performance tuning) |
71807 | 859 |
-R refer to requirements of selected sessions |
66745 | 860 |
-S soft build: only observe changes of sources, not heap images |
62590 | 861 |
-X NAME exclude sessions from group NAME and all descendants |
862 |
-a select all sessions |
|
863 |
-b build heap images |
|
864 |
-c clean build |
|
865 |
-d DIR include session directory |
|
69811
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
866 |
-e export files from session specification into file-system |
66841 | 867 |
-f fresh build |
62590 | 868 |
-g NAME select session group NAME |
869 |
-j INT maximum number of parallel jobs (default 1) |
|
870 |
-k KEYWORD check theory sources for conflicts with proposed keywords |
|
871 |
-l list session source files |
|
872 |
-n no build -- test dependencies only |
|
873 |
-o OPTION override Isabelle system OPTION (via NAME=VAL or NAME) |
|
874 |
-v verbose |
|
875 |
-x NAME exclude session NAME and all descendants |
|
876 |
||
62596 | 877 |
Build and manage Isabelle sessions, depending on implicit settings: |
878 |
||
71992 | 879 |
""" + Library.prefix_lines(" ", Build_Log.Settings.show()) + "\n", |
66737
2edc0c42c883
option -B for "isabelle build" and "isabelle imports";
wenzelm
parents:
66736
diff
changeset
|
880 |
"B:" -> (arg => base_sessions = base_sessions ::: List(arg)), |
62833 | 881 |
"D:" -> (arg => select_dirs = select_dirs ::: List(Path.explode(arg))), |
64265 | 882 |
"N" -> (_ => numa_shuffling = true), |
62833 | 883 |
"R" -> (_ => requirements = true), |
66745 | 884 |
"S" -> (_ => soft_build = true), |
62833 | 885 |
"X:" -> (arg => exclude_session_groups = exclude_session_groups ::: List(arg)), |
886 |
"a" -> (_ => all_sessions = true), |
|
887 |
"b" -> (_ => build_heap = true), |
|
888 |
"c" -> (_ => clean_build = true), |
|
889 |
"d:" -> (arg => dirs = dirs ::: List(Path.explode(arg))), |
|
69811
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
890 |
"e" -> (_ => export_files = true), |
66841 | 891 |
"f" -> (_ => fresh_build = true), |
62833 | 892 |
"g:" -> (arg => session_groups = session_groups ::: List(arg)), |
63805 | 893 |
"j:" -> (arg => max_jobs = Value.Int.parse(arg)), |
62833 | 894 |
"k:" -> (arg => check_keywords = check_keywords + arg), |
895 |
"l" -> (_ => list_files = true), |
|
896 |
"n" -> (_ => no_build = true), |
|
897 |
"o:" -> (arg => options = options + arg), |
|
898 |
"v" -> (_ => verbose = true), |
|
899 |
"x:" -> (arg => exclude_sessions = exclude_sessions ::: List(arg))) |
|
62590 | 900 |
|
62833 | 901 |
val sessions = getopts(args) |
62590 | 902 |
|
64115 | 903 |
val progress = new Console_Progress(verbose = verbose) |
62590 | 904 |
|
64140 | 905 |
val start_date = Date.now() |
906 |
||
62833 | 907 |
if (verbose) { |
908 |
progress.echo( |
|
64155 | 909 |
"Started at " + Build_Log.print_date(start_date) + |
64140 | 910 |
" (" + Isabelle_System.getenv("ML_IDENTIFIER") + " on " + Isabelle_System.hostname() +")") |
64081 | 911 |
progress.echo(Build_Log.Settings.show() + "\n") |
62833 | 912 |
} |
62590 | 913 |
|
62833 | 914 |
val results = |
915 |
progress.interrupt_handler { |
|
67846 | 916 |
build(options, |
71981 | 917 |
selection = Sessions.Selection( |
71896
ce06d6456cc8
clarified signature --- fit within limit of 22 arguments;
wenzelm
parents:
71895
diff
changeset
|
918 |
requirements = requirements, |
ce06d6456cc8
clarified signature --- fit within limit of 22 arguments;
wenzelm
parents:
71895
diff
changeset
|
919 |
all_sessions = all_sessions, |
ce06d6456cc8
clarified signature --- fit within limit of 22 arguments;
wenzelm
parents:
71895
diff
changeset
|
920 |
base_sessions = base_sessions, |
ce06d6456cc8
clarified signature --- fit within limit of 22 arguments;
wenzelm
parents:
71895
diff
changeset
|
921 |
exclude_session_groups = exclude_session_groups, |
ce06d6456cc8
clarified signature --- fit within limit of 22 arguments;
wenzelm
parents:
71895
diff
changeset
|
922 |
exclude_sessions = exclude_sessions, |
ce06d6456cc8
clarified signature --- fit within limit of 22 arguments;
wenzelm
parents:
71895
diff
changeset
|
923 |
session_groups = session_groups, |
ce06d6456cc8
clarified signature --- fit within limit of 22 arguments;
wenzelm
parents:
71895
diff
changeset
|
924 |
sessions = sessions), |
67846 | 925 |
progress = progress, |
65832
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
926 |
check_unknown_files = Mercurial.is_repository(Path.explode("~~")), |
63224 | 927 |
build_heap = build_heap, |
928 |
clean_build = clean_build, |
|
929 |
dirs = dirs, |
|
930 |
select_dirs = select_dirs, |
|
65831 | 931 |
numa_shuffling = NUMA.enabled_warning(progress, numa_shuffling), |
63224 | 932 |
max_jobs = max_jobs, |
933 |
list_files = list_files, |
|
934 |
check_keywords = check_keywords, |
|
66841 | 935 |
fresh_build = fresh_build, |
63224 | 936 |
no_build = no_build, |
66745 | 937 |
soft_build = soft_build, |
63224 | 938 |
verbose = verbose, |
71896
ce06d6456cc8
clarified signature --- fit within limit of 22 arguments;
wenzelm
parents:
71895
diff
changeset
|
939 |
export_files = export_files) |
62833 | 940 |
} |
64140 | 941 |
val end_date = Date.now() |
942 |
val elapsed_time = end_date.time - start_date.time |
|
62590 | 943 |
|
62833 | 944 |
if (verbose) { |
64155 | 945 |
progress.echo("\nFinished at " + Build_Log.print_date(end_date)) |
62833 | 946 |
} |
62590 | 947 |
|
62833 | 948 |
val total_timing = |
949 |
(Timing.zero /: results.sessions.iterator.map(a => results(a).timing))(_ + _). |
|
950 |
copy(elapsed = elapsed_time) |
|
951 |
progress.echo(total_timing.message_resources) |
|
62590 | 952 |
|
62833 | 953 |
sys.exit(results.rc) |
954 |
}) |
|
68305 | 955 |
|
956 |
||
957 |
/* build logic image */ |
|
958 |
||
959 |
def build_logic(options: Options, logic: String, |
|
71726
a5fda30edae2
clarified signature: more uniform treatment of stopped/interrupted state;
wenzelm
parents:
71718
diff
changeset
|
960 |
progress: Progress = new Progress, |
68305 | 961 |
build_heap: Boolean = false, |
962 |
dirs: List[Path] = Nil, |
|
70791 | 963 |
fresh: Boolean = false, |
69540 | 964 |
strict: Boolean = false): Int = |
68305 | 965 |
{ |
71896
ce06d6456cc8
clarified signature --- fit within limit of 22 arguments;
wenzelm
parents:
71895
diff
changeset
|
966 |
val selection = Sessions.Selection.session(logic) |
69540 | 967 |
val rc = |
71981 | 968 |
if (!fresh && build(options, selection = selection, |
969 |
build_heap = build_heap, no_build = true, dirs = dirs).ok) 0 |
|
69540 | 970 |
else { |
971 |
progress.echo("Build started for Isabelle/" + logic + " ...") |
|
71981 | 972 |
Build.build(options, selection = selection, progress = progress, |
973 |
build_heap = build_heap, fresh_build = fresh, dirs = dirs).rc |
|
69540 | 974 |
} |
975 |
if (strict && rc != 0) error("Failed to build Isabelle/" + logic) else rc |
|
68305 | 976 |
} |
48276 | 977 |
} |