author | wenzelm |
Wed, 16 Mar 2016 21:45:04 +0100 | |
changeset 62641 | 0b1b7465f2ef |
parent 62638 | 751cf9f3d433 |
child 62643 | 6f7ac44365d7 |
permissions | -rw-r--r-- |
50686 | 1 |
/* Title: Pure/Tools/build.scala |
48276 | 2 |
Author: Makarius |
57923 | 3 |
Options: :folding=explicit: |
48276 | 4 |
|
5 |
Build and manage Isabelle sessions. |
|
6 |
*/ |
|
7 |
||
8 |
package isabelle |
|
9 |
||
10 |
||
48548 | 11 |
import java.io.{BufferedInputStream, FileInputStream, |
48494 | 12 |
BufferedReader, InputStreamReader, IOException} |
13 |
import java.util.zip.GZIPInputStream |
|
48335 | 14 |
|
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
15 |
import scala.collection.SortedSet |
51397
03b586ee5930
support for 'chapter' specifications within session ROOT;
wenzelm
parents:
51300
diff
changeset
|
16 |
import scala.collection.mutable |
48340
6f4fc030882a
allow explicit specification of additional session directories;
wenzelm
parents:
48339
diff
changeset
|
17 |
import scala.annotation.tailrec |
48337
9c7f8e5805b4
cumulate semantic Session_Info, based on syntactic Session_Entry;
wenzelm
parents:
48336
diff
changeset
|
18 |
|
48335 | 19 |
|
48276 | 20 |
object Build |
21 |
{ |
|
62631 | 22 |
/** auxiliary **/ |
48424 | 23 |
|
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
24 |
/* queue */ |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
25 |
|
62631 | 26 |
private object Queue |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
27 |
{ |
62631 | 28 |
def apply(tree: Sessions.Tree, load_timings: String => (List[Properties.T], Double)): Queue = |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
29 |
{ |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
30 |
val graph = tree.graph |
56372
fadb0fef09d7
more explicit iterator terminology, in accordance to Scala 2.8 library;
wenzelm
parents:
56208
diff
changeset
|
31 |
val sessions = graph.keys |
51220 | 32 |
|
59136
c2b23cb8a677
added Par_List in Scala, in accordance to ML version;
wenzelm
parents:
59083
diff
changeset
|
33 |
val timings = Par_List.map((name: String) => (name, load_timings(name)), sessions) |
51220 | 34 |
val command_timings = |
35 |
Map(timings.map({ case (name, (ts, _)) => (name, ts) }): _*).withDefaultValue(Nil) |
|
36 |
val session_timing = |
|
37 |
Map(timings.map({ case (name, (_, t)) => (name, t) }): _*).withDefaultValue(0.0) |
|
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
38 |
|
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
39 |
def outdegree(name: String): Int = graph.imm_succs(name).size |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
40 |
|
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
41 |
object Ordering extends scala.math.Ordering[String] |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
42 |
{ |
51227
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
43 |
def compare_timing(name1: String, name2: String): Int = |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
44 |
{ |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
45 |
val t1 = session_timing(name1) |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
46 |
val t2 = session_timing(name2) |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
47 |
if (t1 == 0.0 || t2 == 0.0) 0 |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
48 |
else t1 compare t2 |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
49 |
} |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
50 |
|
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
51 |
def compare(name1: String, name2: String): Int = |
51229
6e40d0bb89e3
prefer outdegree in comparison again (cf. 88c96e836ed6) -- NB: big jobs might hide behind small ones in this naive queuing scheme;
wenzelm
parents:
51227
diff
changeset
|
52 |
outdegree(name2) compare outdegree(name1) match { |
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
53 |
case 0 => |
51229
6e40d0bb89e3
prefer outdegree in comparison again (cf. 88c96e836ed6) -- NB: big jobs might hide behind small ones in this naive queuing scheme;
wenzelm
parents:
51227
diff
changeset
|
54 |
compare_timing(name2, name1) match { |
51220 | 55 |
case 0 => |
61602 | 56 |
tree(name2).timeout compare tree(name1).timeout match { |
51220 | 57 |
case 0 => name1 compare name2 |
58 |
case ord => ord |
|
59 |
} |
|
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
60 |
case ord => ord |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
61 |
} |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
62 |
case ord => ord |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
63 |
} |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
64 |
} |
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
65 |
|
51220 | 66 |
new Queue(graph, SortedSet(sessions: _*)(Ordering), command_timings) |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
67 |
} |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
68 |
} |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
69 |
|
62631 | 70 |
private final class Queue private( |
71 |
graph: Graph[String, Sessions.Info], |
|
51220 | 72 |
order: SortedSet[String], |
73 |
val command_timings: String => List[Properties.T]) |
|
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
74 |
{ |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
75 |
def is_inner(name: String): Boolean = !graph.is_maximal(name) |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
76 |
|
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
77 |
def is_empty: Boolean = graph.is_empty |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
78 |
|
51227
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
79 |
def - (name: String): Queue = |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
80 |
new Queue(graph.del_node(name), |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
81 |
order - name, // FIXME scala-2.10.0 TreeSet problem!? |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
82 |
command_timings) |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
83 |
|
62631 | 84 |
def dequeue(skip: String => Boolean): Option[(String, Sessions.Info)] = |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
85 |
{ |
51227
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
86 |
val it = order.iterator.dropWhile(name => |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
87 |
skip(name) |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
88 |
|| !graph.defined(name) // FIXME scala-2.10.0 TreeSet problem!? |
88c96e836ed6
prefer comparison of session timing, if this is known already;
wenzelm
parents:
51223
diff
changeset
|
89 |
|| !graph.is_minimal(name)) |
48680 | 90 |
if (it.hasNext) { val name = it.next; Some((name, graph.get_node(name))) } |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
91 |
else None |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
92 |
} |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
93 |
} |
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
94 |
|
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
95 |
|
48710
5b51ccdc8623
prefer static Build.session_content for loaded theories -- discontinued incremental protocol;
wenzelm
parents:
48708
diff
changeset
|
96 |
/* source dependencies and static content */ |
48422
9613780a805b
determine source dependencies, relatively to preloaded theories;
wenzelm
parents:
48421
diff
changeset
|
97 |
|
48710
5b51ccdc8623
prefer static Build.session_content for loaded theories -- discontinued incremental protocol;
wenzelm
parents:
48708
diff
changeset
|
98 |
sealed case class Session_Content( |
48422
9613780a805b
determine source dependencies, relatively to preloaded theories;
wenzelm
parents:
48421
diff
changeset
|
99 |
loaded_theories: Set[String], |
56801
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
100 |
known_theories: Map[String, Document.Node.Name], |
52439
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
101 |
keywords: Thy_Header.Keywords, |
48660
730ca503e955
static outer syntax based on session specifications;
wenzelm
parents:
48650
diff
changeset
|
102 |
syntax: Outer_Syntax, |
59444 | 103 |
sources: List[(Path, SHA1.Digest)], |
104 |
session_graph: Graph_Display.Graph) |
|
48422
9613780a805b
determine source dependencies, relatively to preloaded theories;
wenzelm
parents:
48421
diff
changeset
|
105 |
|
48710
5b51ccdc8623
prefer static Build.session_content for loaded theories -- discontinued incremental protocol;
wenzelm
parents:
48708
diff
changeset
|
106 |
sealed case class Deps(deps: Map[String, Session_Content]) |
48423
0ccf143a2a69
maintain set of source digests, including relevant parts of session entry;
wenzelm
parents:
48422
diff
changeset
|
107 |
{ |
48583 | 108 |
def is_empty: Boolean = deps.isEmpty |
48710
5b51ccdc8623
prefer static Build.session_content for loaded theories -- discontinued incremental protocol;
wenzelm
parents:
48708
diff
changeset
|
109 |
def apply(name: String): Session_Content = deps(name) |
48504
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
110 |
def sources(name: String): List[SHA1.Digest] = deps(name).sources.map(_._2) |
48423
0ccf143a2a69
maintain set of source digests, including relevant parts of session entry;
wenzelm
parents:
48422
diff
changeset
|
111 |
} |
0ccf143a2a69
maintain set of source digests, including relevant parts of session entry;
wenzelm
parents:
48422
diff
changeset
|
112 |
|
59891
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
113 |
def dependencies( |
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
114 |
progress: Progress = Ignore_Progress, |
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
115 |
inlined_files: Boolean = false, |
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
116 |
verbose: Boolean = false, |
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
117 |
list_files: Boolean = false, |
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
118 |
check_keywords: Set[String] = Set.empty, |
62631 | 119 |
tree: Sessions.Tree): Deps = |
48710
5b51ccdc8623
prefer static Build.session_content for loaded theories -- discontinued incremental protocol;
wenzelm
parents:
48708
diff
changeset
|
120 |
Deps((Map.empty[String, Session_Content] /: tree.topological_order)( |
48422
9613780a805b
determine source dependencies, relatively to preloaded theories;
wenzelm
parents:
48421
diff
changeset
|
121 |
{ case (deps, (name, info)) => |
56668
56335a8e2e8b
interruptible dependencies, which can take a few seconds;
wenzelm
parents:
56667
diff
changeset
|
122 |
if (progress.stopped) throw Exn.Interrupt() |
56335a8e2e8b
interruptible dependencies, which can take a few seconds;
wenzelm
parents:
56667
diff
changeset
|
123 |
|
54549
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
124 |
try { |
56801
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
125 |
val (loaded_theories0, known_theories0, syntax0) = |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
126 |
info.parent.map(deps(_)) match { |
54549
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
127 |
case None => |
58928
23d0ffd48006
plain value Keywords.keywords, which might be used outside theory for bootstrap purposes;
wenzelm
parents:
58918
diff
changeset
|
128 |
(Set.empty[String], Map.empty[String, Document.Node.Name], |
59736
5c1a0069b9d3
tight span for theory header, which is relevant for error positions (including semantic completion);
wenzelm
parents:
59723
diff
changeset
|
129 |
Thy_Header.bootstrap_syntax) |
56801
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
130 |
case Some(parent) => |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
131 |
(parent.loaded_theories, parent.known_theories, parent.syntax) |
54549
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
132 |
} |
56801
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
133 |
val resources = new Resources(loaded_theories0, known_theories0, syntax0) |
56208 | 134 |
val thy_info = new Thy_Info(resources) |
48422
9613780a805b
determine source dependencies, relatively to preloaded theories;
wenzelm
parents:
48421
diff
changeset
|
135 |
|
54549
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
136 |
if (verbose || list_files) { |
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
137 |
val groups = |
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
138 |
if (info.groups.isEmpty) "" |
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
139 |
else info.groups.mkString(" (", " ", ")") |
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
140 |
progress.echo("Session " + info.chapter + "/" + name + groups) |
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
141 |
} |
48478 | 142 |
|
54549
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
143 |
val thy_deps = |
56801
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
144 |
{ |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
145 |
val root_theories = |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
146 |
info.theories.flatMap({ |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
147 |
case (global, _, thys) => |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
148 |
thys.map(thy => |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
149 |
(resources.node_name( |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
150 |
if (global) "" else name, info.dir + Resources.thy_path(thy)), info.pos)) |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
151 |
}) |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
152 |
val thy_deps = thy_info.dependencies(name, root_theories) |
48422
9613780a805b
determine source dependencies, relatively to preloaded theories;
wenzelm
parents:
48421
diff
changeset
|
153 |
|
56801
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
154 |
thy_deps.errors match { |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
155 |
case Nil => thy_deps |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
156 |
case errs => error(cat_lines(errs)) |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
157 |
} |
54549
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
158 |
} |
51294
0850d43cb355
discontinued obsolete header "files" -- these are loaded explicitly after exploring dependencies;
wenzelm
parents:
51293
diff
changeset
|
159 |
|
56801
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
160 |
val known_theories = |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
161 |
(known_theories0 /: thy_deps.deps)({ case (known, dep) => |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
162 |
val name = dep.name |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
163 |
known.get(name.theory) match { |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
164 |
case Some(name1) if name != name1 => |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
165 |
error("Duplicate theory " + quote(name.node) + " vs. " + quote(name1.node)) |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
166 |
case _ => |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
167 |
known + (name.theory -> name) + (Long_Name.base_name(name.theory) -> name) |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
168 |
} |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
169 |
}) |
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
170 |
|
54549
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
171 |
val loaded_theories = thy_deps.loaded_theories |
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
172 |
val keywords = thy_deps.keywords |
56393
22f533e6a049
more abstract Prover.Syntax, as proposed by Carst Tankink;
wenzelm
parents:
56392
diff
changeset
|
173 |
val syntax = thy_deps.syntax.asInstanceOf[Outer_Syntax] |
48660
730ca503e955
static outer syntax based on session specifications;
wenzelm
parents:
48650
diff
changeset
|
174 |
|
59891
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
175 |
val theory_files = thy_deps.deps.map(dep => Path.explode(dep.name.node)) |
56392 | 176 |
val loaded_files = if (inlined_files) thy_deps.loaded_files else Nil |
48903 | 177 |
|
54549
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
178 |
val all_files = |
59891
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
179 |
(theory_files ::: loaded_files ::: |
56533
cd8b6d849b6a
explicit 'document_files' in session ROOT specifications;
wenzelm
parents:
56464
diff
changeset
|
180 |
info.files.map(file => info.dir + file) ::: |
cd8b6d849b6a
explicit 'document_files' in session ROOT specifications;
wenzelm
parents:
56464
diff
changeset
|
181 |
info.document_files.map(file => info.dir + file._1 + file._2)).map(_.expand) |
54549
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
182 |
|
56824
5ae68f53b7c2
discontinued adhoc check (see also ea8343187225);
wenzelm
parents:
56801
diff
changeset
|
183 |
if (list_files) |
54549
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
184 |
progress.echo(cat_lines(all_files.map(_.implode).sorted.map(" " + _))) |
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
185 |
|
59895 | 186 |
if (check_keywords.nonEmpty) |
187 |
Check_Keywords.check_keywords(progress, syntax.keywords, check_keywords, theory_files) |
|
59891
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
188 |
|
54549
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
189 |
val sources = all_files.map(p => (p, SHA1.digest(p.file))) |
48903 | 190 |
|
60077 | 191 |
val session_graph = |
192 |
Present.session_graph(info.parent getOrElse "", loaded_theories0, thy_deps.deps) |
|
59444 | 193 |
|
56801
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
194 |
val content = |
59444 | 195 |
Session_Content(loaded_theories, known_theories, keywords, syntax, |
196 |
sources, session_graph) |
|
56801
8dd9df88f647
some support for session-qualified theories: allow to refer to resources via qualified name instead of odd file-system path;
wenzelm
parents:
56782
diff
changeset
|
197 |
deps + (name -> content) |
54549
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
198 |
} |
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
199 |
catch { |
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
200 |
case ERROR(msg) => |
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
201 |
cat_error(msg, "The error(s) above occurred in session " + |
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
202 |
quote(name) + Position.here(info.pos)) |
2a3053472ec3
actually expose errors of cumulative theory dependencies;
wenzelm
parents:
54514
diff
changeset
|
203 |
} |
48423
0ccf143a2a69
maintain set of source digests, including relevant parts of session entry;
wenzelm
parents:
48422
diff
changeset
|
204 |
})) |
48422
9613780a805b
determine source dependencies, relatively to preloaded theories;
wenzelm
parents:
48421
diff
changeset
|
205 |
|
52439
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
206 |
def session_dependencies( |
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
207 |
options: Options, |
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
208 |
inlined_files: Boolean, |
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
209 |
dirs: List[Path], |
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
210 |
sessions: List[String]): Deps = |
48710
5b51ccdc8623
prefer static Build.session_content for loaded theories -- discontinued incremental protocol;
wenzelm
parents:
48708
diff
changeset
|
211 |
{ |
62635 | 212 |
val (_, tree) = Sessions.load(options, dirs = dirs).selection(sessions = sessions) |
59891
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
213 |
dependencies(inlined_files = inlined_files, tree = tree) |
48710
5b51ccdc8623
prefer static Build.session_content for loaded theories -- discontinued incremental protocol;
wenzelm
parents:
48708
diff
changeset
|
214 |
} |
5b51ccdc8623
prefer static Build.session_content for loaded theories -- discontinued incremental protocol;
wenzelm
parents:
48708
diff
changeset
|
215 |
|
52439
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
216 |
def session_content( |
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
217 |
options: Options, |
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
218 |
inlined_files: Boolean, |
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
219 |
dirs: List[Path], |
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
220 |
session: String): Session_Content = |
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
221 |
{ |
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
222 |
session_dependencies(options, inlined_files, dirs, List(session))(session) |
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
223 |
} |
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
224 |
|
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
225 |
def outer_syntax(options: Options, session: String): Outer_Syntax = |
4cf3f6153eb8
improved "isabelle keywords" and "isabelle update_keywords" based on Isabelle/Scala, without requiring to build sessions first;
wenzelm
parents:
52115
diff
changeset
|
226 |
session_content(options, false, Nil, session).syntax |
48710
5b51ccdc8623
prefer static Build.session_content for loaded theories -- discontinued incremental protocol;
wenzelm
parents:
48708
diff
changeset
|
227 |
|
48422
9613780a805b
determine source dependencies, relatively to preloaded theories;
wenzelm
parents:
48421
diff
changeset
|
228 |
|
48424 | 229 |
/* jobs */ |
48341 | 230 |
|
62637
0189fe0f6452
support for Poly/ML heap hierarchy, which saves a lot of disk space;
wenzelm
parents:
62636
diff
changeset
|
231 |
private class Job(progress: Progress, name: String, val info: Sessions.Info, tree: Sessions.Tree, |
62633 | 232 |
store: Sessions.Store, do_output: Boolean, verbose: Boolean, |
233 |
session_graph: Graph_Display.Graph, command_timings: List[Properties.T]) |
|
48418 | 234 |
{ |
62633 | 235 |
val output = store.output_dir + Path.basic(name) |
48674 | 236 |
def output_path: Option[Path] = if (do_output) Some(output) else None |
62630 | 237 |
def output_save_state: String = |
238 |
if (do_output) |
|
62638 | 239 |
"PolyML.SaveState.saveChild (" + ML_Syntax.print_string0(File.platform_path(output)) + |
62637
0189fe0f6452
support for Poly/ML heap hierarchy, which saves a lot of disk space;
wenzelm
parents:
62636
diff
changeset
|
240 |
", List.length (PolyML.SaveState.showHierarchy ()))" |
62630 | 241 |
else "" |
62633 | 242 |
output.file.delete |
48674 | 243 |
|
244 |
private val parent = info.parent.getOrElse("") |
|
48418 | 245 |
|
59445 | 246 |
private val graph_file = Isabelle_System.tmp_file("session_graph", "pdf") |
247 |
try { isabelle.graphview.Graph_File.write(info.options, graph_file, session_graph) } |
|
248 |
catch { case ERROR(_) => /*error should be exposed in ML*/ } |
|
249 |
||
62610
4c89504c76fb
more uniform signature for various process invocations;
wenzelm
parents:
62603
diff
changeset
|
250 |
private val env = |
4c89504c76fb
more uniform signature for various process invocations;
wenzelm
parents:
62603
diff
changeset
|
251 |
Isabelle_System.settings() + |
4c89504c76fb
more uniform signature for various process invocations;
wenzelm
parents:
62603
diff
changeset
|
252 |
("ISABELLE_ML_DEBUGGER" -> info.options.bool("ML_debugger").toString) |
48698
2585042b1a30
pass Isabelle/Scala system options into ML process of Isar tty or build jobs;
wenzelm
parents:
48684
diff
changeset
|
253 |
|
62573 | 254 |
private val future_result: Future[Process_Result] = |
61559
313eca3fa847
more direct task future implementation, with proper cancel operation;
wenzelm
parents:
61556
diff
changeset
|
255 |
Future.thread("build") { |
62573 | 256 |
val process = |
62631 | 257 |
if (Sessions.is_pure(name)) { |
62573 | 258 |
val eval = |
259 |
"Command_Line.tool0 (fn () => (Session.finish (); Options.reset_default ();" + |
|
62630 | 260 |
" Session.shutdown (); ML_Heap.share_common_data (); " + output_save_state + "));" |
62573 | 261 |
ML_Process(info.options, "RAW_ML_SYSTEM", List("--use", "ROOT.ML", "--eval", eval), |
62637
0189fe0f6452
support for Poly/ML heap hierarchy, which saves a lot of disk space;
wenzelm
parents:
62636
diff
changeset
|
262 |
cwd = info.dir.file, env = env, tree = Some(tree), store = store) |
62573 | 263 |
} |
264 |
else { |
|
62603 | 265 |
val args_file = Isabelle_System.tmp_file("build") |
266 |
File.write(args_file, YXML.string_of_body( |
|
62599
f35858c831e5
clarified session build options: already provided by ML_Process;
wenzelm
parents:
62596
diff
changeset
|
267 |
{ |
f35858c831e5
clarified session build options: already provided by ML_Process;
wenzelm
parents:
62596
diff
changeset
|
268 |
val theories = info.theories.map(x => (x._2, x._3)) |
f35858c831e5
clarified session build options: already provided by ML_Process;
wenzelm
parents:
62596
diff
changeset
|
269 |
import XML.Encode._ |
62630 | 270 |
pair(list(pair(string, int)), pair(list(properties), pair(bool, pair(bool, |
62599
f35858c831e5
clarified session build options: already provided by ML_Process;
wenzelm
parents:
62596
diff
changeset
|
271 |
pair(Path.encode, pair(list(pair(Path.encode, Path.encode)), pair(string, |
f35858c831e5
clarified session build options: already provided by ML_Process;
wenzelm
parents:
62596
diff
changeset
|
272 |
pair(string, pair(string, pair(string, |
f35858c831e5
clarified session build options: already provided by ML_Process;
wenzelm
parents:
62596
diff
changeset
|
273 |
list(pair(Options.encode, list(Path.encode)))))))))))))( |
62630 | 274 |
(Symbol.codes, (command_timings, (do_output, (verbose, |
62633 | 275 |
(store.browser_info, (info.document_files, (File.standard_path(graph_file), |
62599
f35858c831e5
clarified session build options: already provided by ML_Process;
wenzelm
parents:
62596
diff
changeset
|
276 |
(parent, (info.chapter, (name, |
f35858c831e5
clarified session build options: already provided by ML_Process;
wenzelm
parents:
62596
diff
changeset
|
277 |
theories))))))))))) |
f35858c831e5
clarified session build options: already provided by ML_Process;
wenzelm
parents:
62596
diff
changeset
|
278 |
})) |
62630 | 279 |
val eval = |
280 |
"Command_Line.tool0 (fn () => (" + |
|
62638 | 281 |
"Build.build " + ML_Syntax.print_string0(File.standard_path(args_file)) + |
62630 | 282 |
(if (do_output) "; ML_Heap.share_common_data (); " + output_save_state |
283 |
else "") + "));" |
|
62633 | 284 |
ML_Process(info.options, parent, List("--eval", eval), cwd = info.dir.file, |
62637
0189fe0f6452
support for Poly/ML heap hierarchy, which saves a lot of disk space;
wenzelm
parents:
62636
diff
changeset
|
285 |
env = env, tree = Some(tree), store = store, cleanup = () => args_file.delete) |
62573 | 286 |
} |
287 |
process.result( |
|
51962
016cb7d8f297
limit build process output, to avoid bombing Isabelle/Scala process by ill-behaved jobs (e.g. Containers in AFP/9025435b29cf);
wenzelm
parents:
51663
diff
changeset
|
288 |
progress_stdout = (line: String) => |
50847 | 289 |
Library.try_unprefix("\floading_theory = ", line) match { |
290 |
case Some(theory) => progress.theory(name, theory) |
|
291 |
case None => |
|
51962
016cb7d8f297
limit build process output, to avoid bombing Isabelle/Scala process by ill-behaved jobs (e.g. Containers in AFP/9025435b29cf);
wenzelm
parents:
51663
diff
changeset
|
292 |
}, |
016cb7d8f297
limit build process output, to avoid bombing Isabelle/Scala process by ill-behaved jobs (e.g. Containers in AFP/9025435b29cf);
wenzelm
parents:
51663
diff
changeset
|
293 |
progress_limit = |
016cb7d8f297
limit build process output, to avoid bombing Isabelle/Scala process by ill-behaved jobs (e.g. Containers in AFP/9025435b29cf);
wenzelm
parents:
51663
diff
changeset
|
294 |
info.options.int("process_output_limit") match { |
016cb7d8f297
limit build process output, to avoid bombing Isabelle/Scala process by ill-behaved jobs (e.g. Containers in AFP/9025435b29cf);
wenzelm
parents:
51663
diff
changeset
|
295 |
case 0 => None |
016cb7d8f297
limit build process output, to avoid bombing Isabelle/Scala process by ill-behaved jobs (e.g. Containers in AFP/9025435b29cf);
wenzelm
parents:
51663
diff
changeset
|
296 |
case m => Some(m * 1000000L) |
56871
d06ff36b4fa7
expose interrupts more like ML version, but not in managed bash processes of Build;
wenzelm
parents:
56861
diff
changeset
|
297 |
}, |
d06ff36b4fa7
expose interrupts more like ML version, but not in managed bash processes of Build;
wenzelm
parents:
56861
diff
changeset
|
298 |
strict = false) |
50845 | 299 |
} |
48674 | 300 |
|
62572 | 301 |
def terminate: Unit = future_result.cancel |
302 |
def is_finished: Boolean = future_result.is_finished |
|
48674 | 303 |
|
62569 | 304 |
@volatile private var was_timeout = false |
56779 | 305 |
private val timeout_request: Option[Event_Timer.Request] = |
306 |
{ |
|
62569 | 307 |
if (info.timeout > Time.zero) |
308 |
Some(Event_Timer.request(Time.now() + info.timeout) { terminate; was_timeout = true }) |
|
48674 | 309 |
else None |
56779 | 310 |
} |
48674 | 311 |
|
62400 | 312 |
def join: Process_Result = |
50845 | 313 |
{ |
62572 | 314 |
val result = future_result.join |
50845 | 315 |
|
62631 | 316 |
if (result.ok && !Sessions.is_pure(name)) |
62633 | 317 |
Present.finish(progress, store.browser_info, graph_file, info, name) |
61372 | 318 |
|
59445 | 319 |
graph_file.delete |
56779 | 320 |
timeout_request.foreach(_.cancel) |
48674 | 321 |
|
62572 | 322 |
if (result.interrupted) { |
323 |
if (was_timeout) result.error(Output.error_text("Timeout")).was_timeout |
|
324 |
else result.error(Output.error_text("Interrupt")) |
|
52063
fd533ac64390
timeout counts as regular error, with rc = 1 (cf. special Exn.Interrupt vs. regular TimeLimit.TimeOut in Isabelle/ML);
wenzelm
parents:
51987
diff
changeset
|
325 |
} |
62572 | 326 |
else result |
48674 | 327 |
} |
48364 | 328 |
} |
329 |
||
48424 | 330 |
|
51045
630c0895d9d1
more efficient inlined properties, especially relevant for voluminous tasks trace;
wenzelm
parents:
50982
diff
changeset
|
331 |
/* inlined properties (YXML) */ |
50946
8ad3e376f63e
tuned signature (again) -- keep Properties more generic;
wenzelm
parents:
50893
diff
changeset
|
332 |
|
8ad3e376f63e
tuned signature (again) -- keep Properties more generic;
wenzelm
parents:
50893
diff
changeset
|
333 |
object Props |
8ad3e376f63e
tuned signature (again) -- keep Properties more generic;
wenzelm
parents:
50893
diff
changeset
|
334 |
{ |
51045
630c0895d9d1
more efficient inlined properties, especially relevant for voluminous tasks trace;
wenzelm
parents:
50982
diff
changeset
|
335 |
def parse(text: String): Properties.T = XML.Decode.properties(YXML.parse_body(text)) |
50946
8ad3e376f63e
tuned signature (again) -- keep Properties more generic;
wenzelm
parents:
50893
diff
changeset
|
336 |
|
8ad3e376f63e
tuned signature (again) -- keep Properties more generic;
wenzelm
parents:
50893
diff
changeset
|
337 |
def parse_lines(prefix: String, lines: List[String]): List[Properties.T] = |
8ad3e376f63e
tuned signature (again) -- keep Properties more generic;
wenzelm
parents:
50893
diff
changeset
|
338 |
for (line <- lines; s <- Library.try_unprefix(prefix, line)) yield parse(s) |
8ad3e376f63e
tuned signature (again) -- keep Properties more generic;
wenzelm
parents:
50893
diff
changeset
|
339 |
|
8ad3e376f63e
tuned signature (again) -- keep Properties more generic;
wenzelm
parents:
50893
diff
changeset
|
340 |
def find_parse_line(prefix: String, lines: List[String]): Option[Properties.T] = |
8ad3e376f63e
tuned signature (again) -- keep Properties more generic;
wenzelm
parents:
50893
diff
changeset
|
341 |
lines.find(_.startsWith(prefix)).map(line => parse(line.substring(prefix.length))) |
8ad3e376f63e
tuned signature (again) -- keep Properties more generic;
wenzelm
parents:
50893
diff
changeset
|
342 |
} |
8ad3e376f63e
tuned signature (again) -- keep Properties more generic;
wenzelm
parents:
50893
diff
changeset
|
343 |
|
8ad3e376f63e
tuned signature (again) -- keep Properties more generic;
wenzelm
parents:
50893
diff
changeset
|
344 |
|
50777 | 345 |
/* log files */ |
48504
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
346 |
|
50982 | 347 |
private val SESSION_NAME = "\fSession.name = " |
50707
5b2bf7611662
maintain session index on Scala side, for more determistic results;
wenzelm
parents:
50686
diff
changeset
|
348 |
|
50975 | 349 |
sealed case class Log_Info( |
51220 | 350 |
name: String, |
351 |
stats: List[Properties.T], |
|
352 |
tasks: List[Properties.T], |
|
353 |
command_timings: List[Properties.T], |
|
354 |
session_timing: Properties.T) |
|
50777 | 355 |
|
51220 | 356 |
def parse_log(full_stats: Boolean, text: String): Log_Info = |
50777 | 357 |
{ |
358 |
val lines = split_lines(text) |
|
51223 | 359 |
val xml_cache = new XML.Cache() |
360 |
def parse_lines(prfx: String): List[Properties.T] = |
|
51663 | 361 |
Props.parse_lines(prfx, lines).map(xml_cache.props(_)) |
51223 | 362 |
|
50982 | 363 |
val name = |
364 |
lines.find(_.startsWith(SESSION_NAME)).map(_.substring(SESSION_NAME.length)) getOrElse "" |
|
51223 | 365 |
val stats = if (full_stats) parse_lines("\fML_statistics = ") else Nil |
366 |
val tasks = if (full_stats) parse_lines("\ftask_statistics = ") else Nil |
|
367 |
val command_timings = parse_lines("\fcommand_timing = ") |
|
51220 | 368 |
val session_timing = Props.find_parse_line("\fTiming = ", lines) getOrElse Nil |
369 |
Log_Info(name, stats, tasks, command_timings, session_timing) |
|
50777 | 370 |
} |
371 |
||
372 |
||
373 |
/* sources and heaps */ |
|
374 |
||
62628 | 375 |
private val SOURCES = "sources: " |
376 |
private val INPUT_HEAP = "input_heap: " |
|
377 |
private val OUTPUT_HEAP = "output_heap: " |
|
378 |
private val LOG_START = "log:" |
|
379 |
private val line_prefixes = List(SOURCES, INPUT_HEAP, OUTPUT_HEAP, LOG_START) |
|
48504
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
380 |
|
62628 | 381 |
private def sources_stamp(digests: List[SHA1.Digest]): String = |
382 |
digests.map(_.toString).sorted.mkString(SOURCES, " ", "") |
|
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
383 |
|
62628 | 384 |
private def read_stamps(path: Path): Option[(String, List[String], List[String])] = |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
385 |
if (path.is_file) { |
62628 | 386 |
val stream = new GZIPInputStream(new BufferedInputStream(new FileInputStream(path.file))) |
50203 | 387 |
val reader = new BufferedReader(new InputStreamReader(stream, UTF8.charset)) |
62628 | 388 |
val lines = |
389 |
{ |
|
390 |
val lines = new mutable.ListBuffer[String] |
|
391 |
try { |
|
392 |
var finished = false |
|
393 |
while (!finished) { |
|
394 |
val line = reader.readLine |
|
395 |
if (line != null && line_prefixes.exists(line.startsWith(_))) |
|
396 |
lines += line |
|
397 |
else finished = true |
|
398 |
} |
|
399 |
} |
|
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
400 |
finally { reader.close } |
62628 | 401 |
lines.toList |
402 |
} |
|
403 |
||
404 |
if (!lines.isEmpty && lines.last.startsWith(LOG_START)) { |
|
405 |
lines.find(_.startsWith(SOURCES)).map(s => |
|
406 |
(s, lines.filter(_.startsWith(INPUT_HEAP)), lines.filter(_.startsWith(OUTPUT_HEAP)))) |
|
407 |
} |
|
48504
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
408 |
else None |
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
409 |
} |
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
410 |
else None |
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
411 |
|
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
412 |
|
62631 | 413 |
|
62641 | 414 |
/** build with results **/ |
48424 | 415 |
|
62641 | 416 |
class Results private [Build](results: Map[String, Option[Process_Result]]) |
62403 | 417 |
{ |
418 |
def sessions: Set[String] = results.keySet |
|
419 |
def cancelled(name: String): Boolean = results(name).isEmpty |
|
62406 | 420 |
def apply(name: String): Process_Result = results(name).getOrElse(Process_Result(1)) |
62403 | 421 |
val rc = (0 /: results.iterator.map({ case (_, Some(r)) => r.rc case (_, None) => 1 }))(_ max _) |
62641 | 422 |
def ok: Boolean = rc == 0 |
62406 | 423 |
|
424 |
override def toString: String = rc.toString |
|
62403 | 425 |
} |
426 |
||
62641 | 427 |
def build( |
50404
898cac1dad5e
avoid startup within GUI thread -- it is only required later for dialog;
wenzelm
parents:
50367
diff
changeset
|
428 |
options: Options, |
52115 | 429 |
progress: Progress = Ignore_Progress, |
49131 | 430 |
requirements: Boolean = false, |
48509 | 431 |
all_sessions: Boolean = false, |
48511
37999ee01156
remove old output heaps, to ensure that result is valid wrt. check_stamps;
wenzelm
parents:
48509
diff
changeset
|
432 |
build_heap: Boolean = false, |
48595 | 433 |
clean_build: Boolean = false, |
56890 | 434 |
dirs: List[Path] = Nil, |
435 |
select_dirs: List[Path] = Nil, |
|
60106 | 436 |
exclude_session_groups: List[String] = Nil, |
48509 | 437 |
session_groups: List[String] = Nil, |
438 |
max_jobs: Int = 1, |
|
48903 | 439 |
list_files: Boolean = false, |
59891
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
440 |
check_keywords: Set[String] = Set.empty, |
48509 | 441 |
no_build: Boolean = false, |
442 |
system_mode: Boolean = false, |
|
443 |
verbose: Boolean = false, |
|
59892
2a616319c171
added isabelle build option -x, to exclude sessions;
wenzelm
parents:
59891
diff
changeset
|
444 |
exclude_sessions: List[String] = Nil, |
62641 | 445 |
sessions: List[String] = Nil): Results = |
48341 | 446 |
{ |
51220 | 447 |
/* session tree and dependencies */ |
448 |
||
62635 | 449 |
val full_tree = Sessions.load(options.int("completion_limit") = 0, dirs, select_dirs) |
49131 | 450 |
val (selected, selected_tree) = |
60106 | 451 |
full_tree.selection(requirements, all_sessions, |
452 |
exclude_session_groups, exclude_sessions, session_groups, sessions) |
|
49131 | 453 |
|
59891
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
454 |
val deps = dependencies(progress, true, verbose, list_files, check_keywords, selected_tree) |
48368 | 455 |
|
62628 | 456 |
def session_sources_stamp(name: String): String = |
62631 | 457 |
sources_stamp(selected_tree(name).meta_digest :: deps.sources(name)) |
48504
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
458 |
|
62632 | 459 |
val store = Sessions.store(system_mode) |
51220 | 460 |
|
461 |
||
462 |
/* queue with scheduling information */ |
|
463 |
||
51230
19192615911e
option parallel_proofs_reuse_timing controls reuse of log information -- since it is not always beneficial for performance;
wenzelm
parents:
51229
diff
changeset
|
464 |
def load_timings(name: String): (List[Properties.T], Double) = |
51221 | 465 |
{ |
466 |
val (path, text) = |
|
62632 | 467 |
store.find_log_gz(name) match { |
468 |
case Some(path) => (path, File.read_gzip(path)) |
|
51221 | 469 |
case None => |
62632 | 470 |
store.find_log(name) match { |
471 |
case Some(path) => (path, File.read(path)) |
|
51221 | 472 |
case None => (Path.current, "") |
473 |
} |
|
474 |
} |
|
51244
d8ca566b22b3
more robust load_timings: ignore XML.Decode errors as well;
wenzelm
parents:
51230
diff
changeset
|
475 |
|
d8ca566b22b3
more robust load_timings: ignore XML.Decode errors as well;
wenzelm
parents:
51230
diff
changeset
|
476 |
def ignore_error(msg: String): (List[Properties.T], Double) = |
d8ca566b22b3
more robust load_timings: ignore XML.Decode errors as well;
wenzelm
parents:
51230
diff
changeset
|
477 |
{ |
56782
433cf57550fa
more systematic Isabelle output, like in classic Isabelle/ML (without markup);
wenzelm
parents:
56780
diff
changeset
|
478 |
Output.warning("Ignoring bad log file: " + path + (if (msg == "") "" else "\n" + msg)) |
51244
d8ca566b22b3
more robust load_timings: ignore XML.Decode errors as well;
wenzelm
parents:
51230
diff
changeset
|
479 |
(Nil, 0.0) |
d8ca566b22b3
more robust load_timings: ignore XML.Decode errors as well;
wenzelm
parents:
51230
diff
changeset
|
480 |
} |
d8ca566b22b3
more robust load_timings: ignore XML.Decode errors as well;
wenzelm
parents:
51230
diff
changeset
|
481 |
|
51221 | 482 |
try { |
483 |
val info = parse_log(false, text) |
|
484 |
val session_timing = Markup.Elapsed.unapply(info.session_timing) getOrElse 0.0 |
|
485 |
(info.command_timings, session_timing) |
|
51220 | 486 |
} |
51221 | 487 |
catch { |
51244
d8ca566b22b3
more robust load_timings: ignore XML.Decode errors as well;
wenzelm
parents:
51230
diff
changeset
|
488 |
case ERROR(msg) => ignore_error(msg) |
51986
5fdca5bfc0b4
more robust load_timings: ignore JVM errors such as java.lang.OutOfMemoryError;
wenzelm
parents:
51983
diff
changeset
|
489 |
case exn: java.lang.Error => ignore_error(Exn.message(exn)) |
51987 | 490 |
case _: XML.Error => ignore_error("") |
51221 | 491 |
} |
492 |
} |
|
51220 | 493 |
|
51230
19192615911e
option parallel_proofs_reuse_timing controls reuse of log information -- since it is not always beneficial for performance;
wenzelm
parents:
51229
diff
changeset
|
494 |
val queue = Queue(selected_tree, load_timings) |
51220 | 495 |
|
496 |
||
497 |
/* main build process */ |
|
498 |
||
62632 | 499 |
store.prepare_output() |
48373 | 500 |
|
48595 | 501 |
// optional cleanup |
502 |
if (clean_build) { |
|
49131 | 503 |
for (name <- full_tree.graph.all_succs(selected)) { |
48595 | 504 |
val files = |
62632 | 505 |
List(Path.basic(name), Sessions.log(name), Sessions.log_gz(name)). |
506 |
map(store.output_dir + _).filter(_.is_file) |
|
59319 | 507 |
if (files.nonEmpty) progress.echo("Cleaning " + name + " ...") |
50366 | 508 |
if (!files.forall(p => p.file.delete)) progress.echo(name + " FAILED to delete") |
48595 | 509 |
} |
510 |
} |
|
511 |
||
48425 | 512 |
// scheduler loop |
62636 | 513 |
case class Result(current: Boolean, heap_stamp: Option[String], process: Option[Process_Result]) |
62402 | 514 |
{ |
515 |
def ok: Boolean = |
|
516 |
process match { |
|
517 |
case None => false |
|
518 |
case Some(res) => res.rc == 0 |
|
519 |
} |
|
520 |
} |
|
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
521 |
|
56837
5a598f1eecfd
more robust interrupt handling for Scala_Console, which uses JVM Thread.interrupt instead of POSIX SIGINT;
wenzelm
parents:
56831
diff
changeset
|
522 |
def sleep() |
5a598f1eecfd
more robust interrupt handling for Scala_Console, which uses JVM Thread.interrupt instead of POSIX SIGINT;
wenzelm
parents:
56831
diff
changeset
|
523 |
{ |
5a598f1eecfd
more robust interrupt handling for Scala_Console, which uses JVM Thread.interrupt instead of POSIX SIGINT;
wenzelm
parents:
56831
diff
changeset
|
524 |
try { Thread.sleep(500) } |
56861 | 525 |
catch { case Exn.Interrupt() => Exn.Interrupt.impose() } |
56837
5a598f1eecfd
more robust interrupt handling for Scala_Console, which uses JVM Thread.interrupt instead of POSIX SIGINT;
wenzelm
parents:
56831
diff
changeset
|
526 |
} |
50366 | 527 |
|
48425 | 528 |
@tailrec def loop( |
48676
3ef82491cdd6
clarified Session_Tree (with proper integrity check) vs. Queue (with provision for alternative ordering);
wenzelm
parents:
48675
diff
changeset
|
529 |
pending: Queue, |
62628 | 530 |
running: Map[String, (List[String], Job)], |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
531 |
results: Map[String, Result]): Map[String, Result] = |
48425 | 532 |
{ |
533 |
if (pending.is_empty) results |
|
51253 | 534 |
else { |
535 |
if (progress.stopped) |
|
536 |
for ((_, (_, job)) <- running) job.terminate |
|
537 |
||
48674 | 538 |
running.find({ case (_, (_, job)) => job.is_finished }) match { |
62628 | 539 |
case Some((name, (input_heaps, job))) => |
50367 | 540 |
//{{{ finish job |
48424 | 541 |
|
62401 | 542 |
val process_result = job.join |
62573 | 543 |
process_result.err_lines.foreach(progress.echo(_)) |
544 |
if (process_result.ok) |
|
545 |
progress.echo("Finished " + name + " (" + process_result.timing.message_resources + ")") |
|
48373 | 546 |
|
62404
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
547 |
val process_result_tail = |
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
548 |
{ |
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
549 |
val lines = process_result.out_lines.filterNot(_.startsWith("\f")) |
62409
e391528eff3b
proper option process_output_tail, more generous default;
wenzelm
parents:
62406
diff
changeset
|
550 |
val tail = job.info.options.int("process_output_tail") |
e391528eff3b
proper option process_output_tail, more generous default;
wenzelm
parents:
62406
diff
changeset
|
551 |
val lines1 = if (tail == 0) lines else lines.drop(lines.length - tail max 0) |
62632 | 552 |
process_result.copy( |
553 |
out_lines = |
|
554 |
"(see also " + (store.output_dir + Sessions.log(name)).file.toString + ")" :: |
|
555 |
lines1) |
|
62404
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
556 |
} |
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
557 |
|
62636 | 558 |
val heap_stamp = |
62401 | 559 |
if (process_result.ok) { |
62632 | 560 |
(store.output_dir + Sessions.log(name)).file.delete |
62636 | 561 |
val heap_stamp = |
562 |
for (path <- job.output_path; stamp <- File.time_stamp(path)) |
|
563 |
yield stamp |
|
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
564 |
|
62632 | 565 |
File.write_gzip(store.output_dir + Sessions.log_gz(name), |
62628 | 566 |
Library.terminate_lines( |
567 |
session_sources_stamp(name) :: |
|
568 |
input_heaps.map(INPUT_HEAP + _) ::: |
|
62636 | 569 |
heap_stamp.toList.map(OUTPUT_HEAP + _) ::: |
62628 | 570 |
List(LOG_START) ::: process_result.out_lines)) |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
571 |
|
62636 | 572 |
heap_stamp |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
573 |
} |
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
574 |
else { |
62632 | 575 |
(store.output_dir + Path.basic(name)).file.delete |
576 |
(store.output_dir + Sessions.log_gz(name)).file.delete |
|
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
577 |
|
62632 | 578 |
File.write(store.output_dir + Sessions.log(name), |
579 |
Library.terminate_lines(process_result.out_lines)) |
|
50366 | 580 |
progress.echo(name + " FAILED") |
62404
13a0f537e232
retain tail out_lines as printed, but not the whole log content;
wenzelm
parents:
62403
diff
changeset
|
581 |
if (!process_result.interrupted) progress.echo(process_result_tail.out) |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
582 |
|
62636 | 583 |
None |
48639
675988e64bf9
store parent heap stamp as well -- needs to be propagated through the build hierarchy;
wenzelm
parents:
48626
diff
changeset
|
584 |
} |
50707
5b2bf7611662
maintain session index on Scala side, for more determistic results;
wenzelm
parents:
50686
diff
changeset
|
585 |
loop(pending - name, running - name, |
62636 | 586 |
results + (name -> Result(false, heap_stamp, Some(process_result_tail)))) |
50367 | 587 |
//}}} |
60215 | 588 |
case None if running.size < (max_jobs max 1) => |
50367 | 589 |
//{{{ check/start next job |
48547 | 590 |
pending.dequeue(running.isDefinedAt(_)) match { |
591 |
case Some((name, info)) => |
|
62628 | 592 |
val ancestor_results = selected_tree.ancestors(name).map(results(_)) |
62636 | 593 |
val ancestor_heaps = ancestor_results.flatMap(_.heap_stamp) |
62628 | 594 |
|
62631 | 595 |
val do_output = build_heap || Sessions.is_pure(name) || queue.is_inner(name) |
48547 | 596 |
|
62636 | 597 |
val (current, heap_stamp) = |
48547 | 598 |
{ |
62632 | 599 |
store.find(name) match { |
62636 | 600 |
case Some((log_gz, heap_stamp)) => |
62632 | 601 |
read_stamps(log_gz) match { |
62628 | 602 |
case Some((sources, input_heaps, output_heaps)) => |
603 |
val current = |
|
604 |
sources == session_sources_stamp(name) && |
|
605 |
input_heaps == ancestor_heaps.map(INPUT_HEAP + _) && |
|
62636 | 606 |
output_heaps == heap_stamp.toList.map(OUTPUT_HEAP + _) && |
607 |
!(do_output && heap_stamp.isEmpty) |
|
608 |
(current, heap_stamp) |
|
609 |
case None => (false, None) |
|
48547 | 610 |
} |
62636 | 611 |
case None => (false, None) |
48504
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
612 |
} |
48547 | 613 |
} |
62628 | 614 |
val all_current = current && ancestor_results.forall(_.current) |
48528
784c6f63d79c
proper all_current, which regards parent status as well;
wenzelm
parents:
48511
diff
changeset
|
615 |
|
48547 | 616 |
if (all_current) |
62402 | 617 |
loop(pending - name, running, |
62636 | 618 |
results + (name -> Result(true, heap_stamp, Some(Process_Result(0))))) |
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
619 |
else if (no_build) { |
50366 | 620 |
if (verbose) progress.echo("Skipping " + name + " ...") |
62402 | 621 |
loop(pending - name, running, |
62636 | 622 |
results + (name -> Result(false, heap_stamp, Some(Process_Result(1))))) |
48678
ff27af15530c
queue ordering by descending outdegree and timeout;
wenzelm
parents:
48676
diff
changeset
|
623 |
} |
62628 | 624 |
else if (ancestor_results.forall(_.ok) && !progress.stopped) { |
50366 | 625 |
progress.echo((if (do_output) "Building " else "Running ") + name + " ...") |
51220 | 626 |
val job = |
62637
0189fe0f6452
support for Poly/ML heap hierarchy, which saves a lot of disk space;
wenzelm
parents:
62636
diff
changeset
|
627 |
new Job(progress, name, info, selected_tree, store, do_output, verbose, |
59445 | 628 |
deps(name).session_graph, queue.command_timings(name)) |
62628 | 629 |
loop(pending, running + (name -> (ancestor_heaps, job)), results) |
48547 | 630 |
} |
631 |
else { |
|
50366 | 632 |
progress.echo(name + " CANCELLED") |
62636 | 633 |
loop(pending - name, running, results + (name -> Result(false, heap_stamp, None))) |
48547 | 634 |
} |
635 |
case None => sleep(); loop(pending, running, results) |
|
48425 | 636 |
} |
50367 | 637 |
///}}} |
48425 | 638 |
case None => sleep(); loop(pending, running, results) |
48373 | 639 |
} |
51253 | 640 |
} |
48425 | 641 |
} |
642 |
||
51220 | 643 |
|
644 |
/* build results */ |
|
645 |
||
62641 | 646 |
val results0 = |
48583 | 647 |
if (deps.is_empty) { |
56782
433cf57550fa
more systematic Isabelle output, like in classic Isabelle/ML (without markup);
wenzelm
parents:
56780
diff
changeset
|
648 |
progress.echo(Output.warning_text("Nothing to build")) |
50707
5b2bf7611662
maintain session index on Scala side, for more determistic results;
wenzelm
parents:
50686
diff
changeset
|
649 |
Map.empty[String, Result] |
48583 | 650 |
} |
651 |
else loop(queue, Map.empty, Map.empty) |
|
652 |
||
62641 | 653 |
val results = |
654 |
new Results((for ((name, result) <- results0.iterator) yield (name, result.process)).toMap) |
|
655 |
||
656 |
if (results.rc != 0 && (verbose || !no_build)) { |
|
657 |
val unfinished = |
|
658 |
(for { |
|
659 |
name <- results.sessions.iterator |
|
660 |
if !results(name).ok |
|
661 |
} yield name).toList.sorted |
|
662 |
progress.echo("Unfinished session(s): " + commas(unfinished)) |
|
663 |
} |
|
664 |
||
51418
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
665 |
|
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
666 |
/* global browser info */ |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
667 |
|
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
668 |
if (!no_build) { |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
669 |
val browser_chapters = |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
670 |
(for { |
62641 | 671 |
(name, result) <- results0.iterator |
62631 | 672 |
if result.ok && !Sessions.is_pure(name) |
51418
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
673 |
info = full_tree(name) |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
674 |
if info.options.bool("browser_info") |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
675 |
} yield (info.chapter, (name, info.description))).toList.groupBy(_._1). |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
676 |
map({ case (chapter, es) => (chapter, es.map(_._2)) }).filterNot(_._2.isEmpty) |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
677 |
|
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
678 |
for ((chapter, entries) <- browser_chapters) |
62632 | 679 |
Present.update_chapter_index(store.browser_info, chapter, entries) |
51418
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
680 |
|
62632 | 681 |
if (browser_chapters.nonEmpty) Present.make_global_index(store.browser_info) |
51418
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
682 |
} |
7b8ce8403340
more accurate handling of global browser info at the very end (without races), subject to no_build and info.browser_info;
wenzelm
parents:
51402
diff
changeset
|
683 |
|
62641 | 684 |
results |
48341 | 685 |
} |
686 |
||
687 |
||
48346
e2382bede914
more general support for Isabelle/Scala command line tools;
wenzelm
parents:
48344
diff
changeset
|
688 |
/* command line entry point */ |
48341 | 689 |
|
690 |
def main(args: Array[String]) |
|
691 |
{ |
|
48346
e2382bede914
more general support for Isabelle/Scala command line tools;
wenzelm
parents:
48344
diff
changeset
|
692 |
Command_Line.tool { |
62590 | 693 |
def show_settings(): String = |
694 |
cat_lines(List( |
|
695 |
"ISABELLE_BUILD_OPTIONS=" + |
|
696 |
quote(Isabelle_System.getenv("ISABELLE_BUILD_OPTIONS")), |
|
697 |
"ISABELLE_BUILD_JAVA_OPTIONS=" + |
|
62596 | 698 |
quote(Isabelle_System.getenv("ISABELLE_BUILD_JAVA_OPTIONS")), |
62590 | 699 |
"", |
700 |
"ML_PLATFORM=" + quote(Isabelle_System.getenv("ML_PLATFORM")), |
|
701 |
"ML_HOME=" + quote(Isabelle_System.getenv("ML_HOME")), |
|
702 |
"ML_SYSTEM=" + quote(Isabelle_System.getenv("ML_SYSTEM")), |
|
703 |
"ML_OPTIONS=" + quote(Isabelle_System.getenv("ML_OPTIONS")))) |
|
704 |
||
705 |
val build_options = Word.explode(Isabelle_System.getenv("ISABELLE_BUILD_OPTIONS")) |
|
706 |
||
707 |
var select_dirs: List[Path] = Nil |
|
708 |
var requirements = false |
|
709 |
var exclude_session_groups: List[String] = Nil |
|
710 |
var all_sessions = false |
|
711 |
var build_heap = false |
|
712 |
var clean_build = false |
|
713 |
var dirs: List[Path] = Nil |
|
714 |
var session_groups: List[String] = Nil |
|
715 |
var max_jobs = 1 |
|
716 |
var check_keywords: Set[String] = Set.empty |
|
717 |
var list_files = false |
|
718 |
var no_build = false |
|
719 |
var options = (Options.init() /: build_options)(_ + _) |
|
720 |
var system_mode = false |
|
721 |
var verbose = false |
|
722 |
var exclude_sessions: List[String] = Nil |
|
723 |
||
724 |
val getopts = Getopts(""" |
|
725 |
Usage: isabelle build [OPTIONS] [SESSIONS ...] |
|
726 |
||
727 |
Options are: |
|
728 |
-D DIR include session directory and select its sessions |
|
729 |
-R operate on requirements of selected sessions |
|
730 |
-X NAME exclude sessions from group NAME and all descendants |
|
731 |
-a select all sessions |
|
732 |
-b build heap images |
|
733 |
-c clean build |
|
734 |
-d DIR include session directory |
|
735 |
-g NAME select session group NAME |
|
736 |
-j INT maximum number of parallel jobs (default 1) |
|
737 |
-k KEYWORD check theory sources for conflicts with proposed keywords |
|
738 |
-l list session source files |
|
739 |
-n no build -- test dependencies only |
|
740 |
-o OPTION override Isabelle system OPTION (via NAME=VAL or NAME) |
|
741 |
-s system build mode: produce output in ISABELLE_HOME |
|
742 |
-v verbose |
|
743 |
-x NAME exclude session NAME and all descendants |
|
744 |
||
62596 | 745 |
Build and manage Isabelle sessions, depending on implicit settings: |
746 |
||
62590 | 747 |
""" + Library.prefix_lines(" ", show_settings()), |
748 |
"D:" -> (arg => select_dirs = select_dirs ::: List(Path.explode(arg))), |
|
749 |
"R" -> (_ => requirements = true), |
|
750 |
"X:" -> (arg => exclude_session_groups = exclude_session_groups ::: List(arg)), |
|
751 |
"a" -> (_ => all_sessions = true), |
|
752 |
"b" -> (_ => build_heap = true), |
|
753 |
"c" -> (_ => clean_build = true), |
|
754 |
"d:" -> (arg => dirs = dirs ::: List(Path.explode(arg))), |
|
755 |
"g:" -> (arg => session_groups = session_groups ::: List(arg)), |
|
756 |
"j:" -> (arg => max_jobs = Properties.Value.Int.parse(arg)), |
|
757 |
"k:" -> (arg => check_keywords = check_keywords + arg), |
|
758 |
"l" -> (_ => list_files = true), |
|
759 |
"n" -> (_ => no_build = true), |
|
760 |
"o:" -> (arg => options = options + arg), |
|
761 |
"s" -> (_ => system_mode = true), |
|
762 |
"v" -> (_ => verbose = true), |
|
763 |
"x:" -> (arg => exclude_sessions = exclude_sessions ::: List(arg))) |
|
764 |
||
765 |
val sessions = getopts(args) |
|
766 |
||
767 |
val progress = new Console_Progress(verbose) |
|
768 |
||
62595 | 769 |
if (verbose) { |
62590 | 770 |
progress.echo( |
771 |
Library.trim_line( |
|
772 |
Isabelle_System.bash( |
|
62596 | 773 |
"""echo "Started at $(date) ($ML_IDENTIFIER on $(hostname))" """).out) + "\n") |
774 |
progress.echo(show_settings() + "\n") |
|
48341 | 775 |
} |
62590 | 776 |
|
62595 | 777 |
val start_time = Time.now() |
62590 | 778 |
val results = |
779 |
progress.interrupt_handler { |
|
62641 | 780 |
build(options, progress, requirements, all_sessions, build_heap, clean_build, |
62590 | 781 |
dirs, select_dirs, exclude_session_groups, session_groups, max_jobs, list_files, |
782 |
check_keywords, no_build, system_mode, verbose, exclude_sessions, sessions) |
|
783 |
} |
|
62595 | 784 |
val elapsed_time = Time.now() - start_time |
62590 | 785 |
|
62595 | 786 |
if (verbose) { |
62596 | 787 |
progress.echo("\n" + |
62590 | 788 |
Library.trim_line( |
62596 | 789 |
Isabelle_System.bash("""echo -n "Finished at "; date""").out)) |
62595 | 790 |
} |
62590 | 791 |
|
62595 | 792 |
val total_timing = |
793 |
(Timing.zero /: results.sessions.iterator.map(a => results(a).timing))(_ + _). |
|
794 |
copy(elapsed = elapsed_time) |
|
795 |
progress.echo(total_timing.message_resources) |
|
62590 | 796 |
|
797 |
results.rc |
|
48346
e2382bede914
more general support for Isabelle/Scala command line tools;
wenzelm
parents:
48344
diff
changeset
|
798 |
} |
48341 | 799 |
} |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
800 |
|
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
801 |
|
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
802 |
/* PIDE protocol */ |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
803 |
|
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
804 |
def build_theories( |
59369
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
805 |
session: Session, master_dir: Path, theories: List[(Options, List[Path])]): Promise[XML.Body] = |
59367 | 806 |
session.get_protocol_handler(classOf[Handler].getName) match { |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
807 |
case Some(handler: Handler) => handler.build_theories(session, master_dir, theories) |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
808 |
case _ => error("Cannot invoke build_theories: bad protocol handler") |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
809 |
} |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
810 |
|
59367 | 811 |
class Handler(progress: Progress, session_name: String) extends Session.Protocol_Handler |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
812 |
{ |
59369
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
813 |
private val pending = Synchronized(Map.empty[String, Promise[XML.Body]]) |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
814 |
|
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
815 |
def build_theories( |
59369
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
816 |
session: Session, master_dir: Path, theories: List[(Options, List[Path])]): Promise[XML.Body] = |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
817 |
{ |
59369
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
818 |
val promise = Future.promise[XML.Body] |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
819 |
val id = Document_ID.make().toString |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
820 |
pending.change(promises => promises + (id -> promise)) |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
821 |
session.build_theories(id, master_dir, theories) |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
822 |
promise |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
823 |
} |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
824 |
|
59367 | 825 |
private def loading_theory(prover: Prover, msg: Prover.Protocol_Output): Boolean = |
826 |
msg.properties match { |
|
827 |
case Markup.Loading_Theory(name) => progress.theory(session_name, name); true |
|
828 |
case _ => false |
|
829 |
} |
|
830 |
||
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
831 |
private def build_theories_result(prover: Prover, msg: Prover.Protocol_Output): Boolean = |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
832 |
msg.properties match { |
59369
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
833 |
case Markup.Build_Theories_Result(id) => |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
834 |
pending.change_result(promises => |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
835 |
promises.get(id) match { |
59369
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
836 |
case Some(promise) => |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
837 |
val error_message = |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
838 |
try { YXML.parse_body(Symbol.decode(msg.text)) } |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
839 |
catch { case exn: Throwable => List(XML.Text(Exn.message(exn))) } |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
840 |
promise.fulfill(error_message) |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
841 |
(true, promises - id) |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
842 |
case None => |
7090199d3f78
more informative build_theories_result: cumulative Runtime.exn_message;
wenzelm
parents:
59367
diff
changeset
|
843 |
(false, promises) |
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
844 |
}) |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
845 |
case _ => false |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
846 |
} |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
847 |
|
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
848 |
override def stop(prover: Prover): Unit = |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
849 |
pending.change(promises => { for ((_, promise) <- promises) promise.cancel; Map.empty }) |
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
850 |
|
59367 | 851 |
val functions = |
852 |
Map( |
|
853 |
Markup.BUILD_THEORIES_RESULT -> build_theories_result _, |
|
854 |
Markup.LOADING_THEORY -> loading_theory _) |
|
59366
e94df7f6b608
clarified build_theories: proper protocol handler;
wenzelm
parents:
59319
diff
changeset
|
855 |
} |
48276 | 856 |
} |