author | wenzelm |
Wed, 08 Mar 2023 13:33:18 +0100 | |
changeset 77579 | 69d3547206db |
parent 77563 | cbb49fe8e5a2 |
child 77628 | a538dab533ef |
permissions | -rw-r--r-- |
50686 | 1 |
/* Title: Pure/Tools/build.scala |
48276 | 2 |
Author: Makarius |
57923 | 3 |
Options: :folding=explicit: |
48276 | 4 |
|
77553 | 5 |
Command-line tools to build and manage Isabelle sessions. |
48276 | 6 |
*/ |
7 |
||
8 |
package isabelle |
|
9 |
||
77553 | 10 |
import scala.collection.mutable |
11 |
import scala.util.matching.Regex |
|
12 |
||
48276 | 13 |
|
75393 | 14 |
object Build { |
77553 | 15 |
/** "isabelle build" **/ |
16 |
||
77330
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
17 |
/* results */ |
48424 | 18 |
|
77311 | 19 |
object Results { |
20 |
def apply(context: Build_Process.Context, results: Map[String, Process_Result]): Results = |
|
77453 | 21 |
new Results(context.store, context.build_deps, results) |
77311 | 22 |
} |
23 |
||
24 |
class Results private( |
|
76206
769a7cd5a16a
clarified signature: re-use store/cache from build results;
wenzelm
parents:
76202
diff
changeset
|
25 |
val store: Sessions.Store, |
76209 | 26 |
val deps: Sessions.Deps, |
77253
792dad9cb04f
clarified data structure: absorb Option[Process_Result] into Process_Result, e.g. to simplify database storage;
wenzelm
parents:
77252
diff
changeset
|
27 |
results: Map[String, Process_Result] |
76198
fb4215da4919
clarified presentation_sessions: work with partial results;
wenzelm
parents:
76197
diff
changeset
|
28 |
) { |
76206
769a7cd5a16a
clarified signature: re-use store/cache from build results;
wenzelm
parents:
76202
diff
changeset
|
29 |
def cache: Term.Cache = store.cache |
769a7cd5a16a
clarified signature: re-use store/cache from build results;
wenzelm
parents:
76202
diff
changeset
|
30 |
|
77311 | 31 |
def sessions_ok: List[String] = |
32 |
(for { |
|
33 |
name <- deps.sessions_structure.build_topological_order.iterator |
|
34 |
result <- results.get(name) if result.ok |
|
35 |
} yield name).toList |
|
36 |
||
77250
22016642d6af
clarified data structure: use static info from deps, not dynamic results;
wenzelm
parents:
77246
diff
changeset
|
37 |
def info(name: String): Sessions.Info = deps.sessions_structure(name) |
62403 | 38 |
def sessions: Set[String] = results.keySet |
77253
792dad9cb04f
clarified data structure: absorb Option[Process_Result] into Process_Result, e.g. to simplify database storage;
wenzelm
parents:
77252
diff
changeset
|
39 |
def cancelled(name: String): Boolean = !results(name).defined |
792dad9cb04f
clarified data structure: absorb Option[Process_Result] into Process_Result, e.g. to simplify database storage;
wenzelm
parents:
77252
diff
changeset
|
40 |
def apply(name: String): Process_Result = results(name).strict |
792dad9cb04f
clarified data structure: absorb Option[Process_Result] into Process_Result, e.g. to simplify database storage;
wenzelm
parents:
77252
diff
changeset
|
41 |
val rc: Int = results.valuesIterator.map(_.strict.rc).foldLeft(Process_Result.RC.ok)(_ max _) |
74306 | 42 |
def ok: Boolean = rc == Process_Result.RC.ok |
62406 | 43 |
|
76202 | 44 |
def unfinished: List[String] = sessions.iterator.filterNot(apply(_).ok).toList.sorted |
76199 | 45 |
|
62406 | 46 |
override def toString: String = rc.toString |
62403 | 47 |
} |
48 |
||
77330
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
49 |
|
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
50 |
/* engine */ |
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
51 |
|
77411
149cc77f7348
clafified signature: simplify object-oriented reuse;
wenzelm
parents:
77384
diff
changeset
|
52 |
class Engine(val name: String) extends Isabelle_System.Service { |
77330
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
53 |
override def toString: String = name |
77505 | 54 |
def init(build_context: Build_Process.Context, build_progress: Progress): Build_Process = |
55 |
new Build_Process(build_context, build_progress) |
|
77330
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
56 |
} |
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
57 |
|
77411
149cc77f7348
clafified signature: simplify object-oriented reuse;
wenzelm
parents:
77384
diff
changeset
|
58 |
class Default_Engine extends Engine("") { override def toString: String = "<default>" } |
77330
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
59 |
|
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
60 |
lazy val engines: List[Engine] = |
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
61 |
Isabelle_System.make_services(classOf[Engine]) |
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
62 |
|
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
63 |
def get_engine(name: String): Engine = |
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
64 |
engines.find(_.name == name).getOrElse(error("Bad build engine " + quote(name))) |
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
65 |
|
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
66 |
|
77558 | 67 |
/* options */ |
68 |
||
69 |
def hostname(options: Options): String = |
|
70 |
Isabelle_System.hostname(options.string("build_hostname")) |
|
77330
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
71 |
|
77559
4ad322ee6025
clarified signature: support all arguments of Sessions.store();
wenzelm
parents:
77558
diff
changeset
|
72 |
def build_init(options: Options, cache: Term.Cache = Term.Cache.make()): Sessions.Store = { |
77557 | 73 |
val build_options = |
74 |
options + |
|
75 |
"completion_limit=0" + |
|
76 |
"editor_tracing_messages=0" + |
|
77 |
("pide_reports=" + options.bool("build_pide_reports")) |
|
77559
4ad322ee6025
clarified signature: support all arguments of Sessions.store();
wenzelm
parents:
77558
diff
changeset
|
78 |
val store = Sessions.store(build_options, cache = cache) |
77557 | 79 |
|
80 |
Isabelle_Fonts.init() |
|
81 |
||
82 |
store |
|
83 |
} |
|
84 |
||
77558 | 85 |
|
86 |
/* build */ |
|
87 |
||
62641 | 88 |
def build( |
50404
898cac1dad5e
avoid startup within GUI thread -- it is only required later for dialog;
wenzelm
parents:
50367
diff
changeset
|
89 |
options: Options, |
71981 | 90 |
selection: Sessions.Selection = Sessions.Selection.empty, |
75942
603852abed8f
clarified names: Browser_Info.Config vs. Browser_Info.Context;
wenzelm
parents:
75941
diff
changeset
|
91 |
browser_info: Browser_Info.Config = Browser_Info.Config.none, |
71726
a5fda30edae2
clarified signature: more uniform treatment of stopped/interrupted state;
wenzelm
parents:
71718
diff
changeset
|
92 |
progress: Progress = new Progress, |
65832
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
93 |
check_unknown_files: Boolean = false, |
48511
37999ee01156
remove old output heaps, to ensure that result is valid wrt. check_stamps;
wenzelm
parents:
48509
diff
changeset
|
94 |
build_heap: Boolean = false, |
48595 | 95 |
clean_build: Boolean = false, |
56890 | 96 |
dirs: List[Path] = Nil, |
97 |
select_dirs: List[Path] = Nil, |
|
66968
9991671c98aa
allow to augment session context via explicit session infos;
wenzelm
parents:
66962
diff
changeset
|
98 |
infos: List[Sessions.Info] = Nil, |
64265 | 99 |
numa_shuffling: Boolean = false, |
48509 | 100 |
max_jobs: Int = 1, |
48903 | 101 |
list_files: Boolean = false, |
59891
9ce697050455
added isabelle build option -k, for fast off-line checking of theory sources;
wenzelm
parents:
59811
diff
changeset
|
102 |
check_keywords: Set[String] = Set.empty, |
66841 | 103 |
fresh_build: Boolean = false, |
48509 | 104 |
no_build: Boolean = false, |
66745 | 105 |
soft_build: Boolean = false, |
73805
b73777a0c076
allow build session setup, e.g. for protocol handlers;
wenzelm
parents:
73804
diff
changeset
|
106 |
export_files: Boolean = false, |
76927
da13da82f6f9
treat update_options as part of Sessions.Info meta_digest, for proper re-build of updated sessions;
wenzelm
parents:
76919
diff
changeset
|
107 |
augment_options: String => List[Options.Spec] = _ => Nil, |
77559
4ad322ee6025
clarified signature: support all arguments of Sessions.store();
wenzelm
parents:
77558
diff
changeset
|
108 |
session_setup: (String, Session) => Unit = (_, _) => (), |
4ad322ee6025
clarified signature: support all arguments of Sessions.store();
wenzelm
parents:
77558
diff
changeset
|
109 |
cache: Term.Cache = Term.Cache.make() |
75393 | 110 |
): Results = { |
77559
4ad322ee6025
clarified signature: support all arguments of Sessions.store();
wenzelm
parents:
77558
diff
changeset
|
111 |
val store = build_init(options, cache = cache) |
77557 | 112 |
val build_options = store.options |
69369
6ecc85955e04
prefer "Isabelle DejaVu Sans", even for headless batch-build (session_graph.pdf);
wenzelm
parents:
68957
diff
changeset
|
113 |
|
66745 | 114 |
|
115 |
/* session selection and dependencies */ |
|
65422 | 116 |
|
66961 | 117 |
val full_sessions = |
76927
da13da82f6f9
treat update_options as part of Sessions.Info meta_digest, for proper re-build of updated sessions;
wenzelm
parents:
76919
diff
changeset
|
118 |
Sessions.load_structure(build_options, dirs = dirs, select_dirs = select_dirs, infos = infos, |
da13da82f6f9
treat update_options as part of Sessions.Info meta_digest, for proper re-build of updated sessions;
wenzelm
parents:
76919
diff
changeset
|
119 |
augment_options = augment_options) |
73012
238ddf525da4
clarified HTML presentation, e.g. avoid bulky jobs like HOL or HOL-Analysis in applications;
wenzelm
parents:
72993
diff
changeset
|
120 |
val full_sessions_selection = full_sessions.imports_selection(selection) |
238ddf525da4
clarified HTML presentation, e.g. avoid bulky jobs like HOL or HOL-Analysis in applications;
wenzelm
parents:
72993
diff
changeset
|
121 |
|
75948 | 122 |
val build_deps = { |
66745 | 123 |
val deps0 = |
77521
5642de4d225d
clarified signature: manage "verbose" flag via "progress";
wenzelm
parents:
77505
diff
changeset
|
124 |
Sessions.deps(full_sessions.selection(selection), progress = progress, inlined_files = true, |
66962
e1bde71bace6
clarified signature: global_theories is always required;
wenzelm
parents:
66961
diff
changeset
|
125 |
list_files = list_files, check_keywords = check_keywords).check_errors |
66745 | 126 |
|
66841 | 127 |
if (soft_build && !fresh_build) { |
66745 | 128 |
val outdated = |
68204 | 129 |
deps0.sessions_structure.build_topological_order.flatMap(name => |
72634
5cea0993ee4f
clarified access to single database server vs. collection of database files;
wenzelm
parents:
72624
diff
changeset
|
130 |
store.try_open_database(name) match { |
68214 | 131 |
case Some(db) => |
68216 | 132 |
using(db)(store.read_build(_, name)) match { |
66745 | 133 |
case Some(build) |
77204 | 134 |
if build.ok && build.sources == deps0.sources_shasum(name) => None |
66745 | 135 |
case _ => Some(name) |
136 |
} |
|
137 |
case None => Some(name) |
|
138 |
}) |
|
68204 | 139 |
|
140 |
Sessions.deps(full_sessions.selection(Sessions.Selection(sessions = outdated)), |
|
70671
cb1776c8e216
clarified signature: retain global session information, unaffected by later restriction;
wenzelm
parents:
70509
diff
changeset
|
141 |
progress = progress, inlined_files = true).check_errors |
66745 | 142 |
} |
68204 | 143 |
else deps0 |
66745 | 144 |
} |
145 |
||
146 |
||
147 |
/* check unknown files */ |
|
48504
21dfd6c04482
actually check source vs. target stamps, based on information from log files;
wenzelm
parents:
48494
diff
changeset
|
148 |
|
65832
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
149 |
if (check_unknown_files) { |
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
150 |
val source_files = |
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
151 |
(for { |
75948 | 152 |
(_, base) <- build_deps.session_bases.iterator |
75741 | 153 |
(path, _) <- base.session_sources.iterator |
65832
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
154 |
} yield path).toList |
76670
b04d45bebbc5
discontinued somewhat pointless dependency: avoid illusion of extra accuracy (see also 09fb749d1a1e and 0f750a6dc754);
wenzelm
parents:
76669
diff
changeset
|
155 |
Mercurial.check_files(source_files)._2 match { |
b04d45bebbc5
discontinued somewhat pointless dependency: avoid illusion of extra accuracy (see also 09fb749d1a1e and 0f750a6dc754);
wenzelm
parents:
76669
diff
changeset
|
156 |
case Nil => |
b04d45bebbc5
discontinued somewhat pointless dependency: avoid illusion of extra accuracy (see also 09fb749d1a1e and 0f750a6dc754);
wenzelm
parents:
76669
diff
changeset
|
157 |
case unknown_files => |
b04d45bebbc5
discontinued somewhat pointless dependency: avoid illusion of extra accuracy (see also 09fb749d1a1e and 0f750a6dc754);
wenzelm
parents:
76669
diff
changeset
|
158 |
progress.echo_warning("Unknown files (not part of the underlying Mercurial repository):" + |
76883 | 159 |
unknown_files.map(File.standard_path).sorted.mkString("\n ", "\n ", "")) |
65832
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
160 |
} |
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
161 |
} |
2fb85623c386
implicitly check for unknown files (not part of a Mercurial repository);
wenzelm
parents:
65831
diff
changeset
|
162 |
|
51220 | 163 |
|
77257 | 164 |
/* build process and results */ |
51220 | 165 |
|
77315
f34559b24277
clarified signature: move all parameters into Build_Process.Context;
wenzelm
parents:
77311
diff
changeset
|
166 |
val build_context = |
f34559b24277
clarified signature: move all parameters into Build_Process.Context;
wenzelm
parents:
77311
diff
changeset
|
167 |
Build_Process.Context(store, build_deps, progress = progress, |
77558 | 168 |
hostname = hostname(build_options), build_heap = build_heap, |
169 |
numa_shuffling = numa_shuffling, max_jobs = max_jobs, fresh_build = fresh_build, |
|
77579
69d3547206db
clarified signature: prefer Build_Process.Context for parameters;
wenzelm
parents:
77563
diff
changeset
|
170 |
no_build = no_build, session_setup = session_setup, master = true) |
77254
8d34f53871b4
clarified signature: make dynamic Queue from static Context;
wenzelm
parents:
77253
diff
changeset
|
171 |
|
77535 | 172 |
store.prepare_output() |
77536
7c7f1473e51a
clarified database content and prepare/init stages;
wenzelm
parents:
77535
diff
changeset
|
173 |
build_context.prepare_database() |
48373 | 174 |
|
48595 | 175 |
if (clean_build) { |
73012
238ddf525da4
clarified HTML presentation, e.g. avoid bulky jobs like HOL or HOL-Analysis in applications;
wenzelm
parents:
72993
diff
changeset
|
176 |
for (name <- full_sessions.imports_descendants(full_sessions_selection)) { |
68220
8fc4e3d1df86
clarified store.clean_output: cleanup user_output_dir even in system_mode;
wenzelm
parents:
68219
diff
changeset
|
177 |
val (relevant, ok) = store.clean_output(name) |
8fc4e3d1df86
clarified store.clean_output: cleanup user_output_dir even in system_mode;
wenzelm
parents:
68219
diff
changeset
|
178 |
if (relevant) { |
8fc4e3d1df86
clarified store.clean_output: cleanup user_output_dir even in system_mode;
wenzelm
parents:
68219
diff
changeset
|
179 |
if (ok) progress.echo("Cleaned " + name) |
8fc4e3d1df86
clarified store.clean_output: cleanup user_output_dir even in system_mode;
wenzelm
parents:
68219
diff
changeset
|
180 |
else progress.echo(name + " FAILED to clean") |
8fc4e3d1df86
clarified store.clean_output: cleanup user_output_dir even in system_mode;
wenzelm
parents:
68219
diff
changeset
|
181 |
} |
48595 | 182 |
} |
183 |
} |
|
184 |
||
77311 | 185 |
val results = |
186 |
Isabelle_Thread.uninterruptible { |
|
77330
47eb96592aa2
support alternative build engines, via system option "build_engine";
wenzelm
parents:
77326
diff
changeset
|
187 |
val engine = get_engine(build_options.string("build_engine")) |
77505 | 188 |
using(engine.init(build_context, progress)) { build_process => |
77579
69d3547206db
clarified signature: prefer Build_Process.Context for parameters;
wenzelm
parents:
77563
diff
changeset
|
189 |
val res = build_process.run() |
77372 | 190 |
Results(build_context, res) |
191 |
} |
|
77311 | 192 |
} |
62641 | 193 |
|
69811
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
194 |
if (export_files) { |
76196 | 195 |
for (name <- full_sessions_selection.iterator if results(name).ok) { |
69811
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
196 |
val info = results.info(name) |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
197 |
if (info.export_files.nonEmpty) { |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
198 |
progress.echo("Exporting " + info.name + " ...") |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
199 |
for ((dir, prune, pats) <- info.export_files) { |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
200 |
Export.export_files(store, name, info.dir + dir, |
77521
5642de4d225d
clarified signature: manage "verbose" flag via "progress";
wenzelm
parents:
77505
diff
changeset
|
201 |
progress = if (progress.verbose) progress else new Progress, |
69811
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
202 |
export_prune = prune, |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
203 |
export_patterns = pats) |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
204 |
} |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
205 |
} |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
206 |
} |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
207 |
} |
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
208 |
|
76901 | 209 |
val presentation_sessions = |
210 |
results.sessions_ok.filter(name => browser_info.enabled(results.info(name))) |
|
76230 | 211 |
if (presentation_sessions.nonEmpty && !progress.stopped) { |
212 |
Browser_Info.build(browser_info, results.store, results.deps, presentation_sessions, |
|
77521
5642de4d225d
clarified signature: manage "verbose" flag via "progress";
wenzelm
parents:
77505
diff
changeset
|
213 |
progress = progress) |
76198
fb4215da4919
clarified presentation_sessions: work with partial results;
wenzelm
parents:
76197
diff
changeset
|
214 |
} |
fb4215da4919
clarified presentation_sessions: work with partial results;
wenzelm
parents:
76197
diff
changeset
|
215 |
|
77521
5642de4d225d
clarified signature: manage "verbose" flag via "progress";
wenzelm
parents:
77505
diff
changeset
|
216 |
if (!results.ok && (progress.verbose || !no_build)) { |
76199 | 217 |
progress.echo("Unfinished session(s): " + commas(results.unfinished)) |
62641 | 218 |
} |
219 |
||
220 |
results |
|
48341 | 221 |
} |
222 |
||
223 |
||
77555 | 224 |
/* build logic image */ |
225 |
||
226 |
def build_logic(options: Options, logic: String, |
|
227 |
progress: Progress = new Progress, |
|
228 |
build_heap: Boolean = false, |
|
229 |
dirs: List[Path] = Nil, |
|
230 |
fresh: Boolean = false, |
|
231 |
strict: Boolean = false |
|
232 |
): Int = { |
|
233 |
val selection = Sessions.Selection.session(logic) |
|
234 |
val rc = |
|
235 |
if (!fresh && build(options, selection = selection, |
|
236 |
build_heap = build_heap, no_build = true, dirs = dirs).ok) Process_Result.RC.ok |
|
237 |
else { |
|
238 |
progress.echo("Build started for Isabelle/" + logic + " ...") |
|
239 |
build(options, selection = selection, progress = progress, |
|
240 |
build_heap = build_heap, fresh_build = fresh, dirs = dirs).rc |
|
241 |
} |
|
242 |
if (strict && rc != Process_Result.RC.ok) error("Failed to build Isabelle/" + logic) else rc |
|
243 |
} |
|
244 |
||
245 |
||
77553 | 246 |
/* command-line wrapper */ |
48341 | 247 |
|
77553 | 248 |
val isabelle_tool1 = Isabelle_Tool("build", "build and manage Isabelle sessions", |
75394 | 249 |
Scala_Project.here, |
250 |
{ args => |
|
251 |
val build_options = Word.explode(Isabelle_System.getenv("ISABELLE_BUILD_OPTIONS")) |
|
62590 | 252 |
|
75394 | 253 |
var base_sessions: List[String] = Nil |
254 |
var select_dirs: List[Path] = Nil |
|
255 |
var numa_shuffling = false |
|
75942
603852abed8f
clarified names: Browser_Info.Config vs. Browser_Info.Context;
wenzelm
parents:
75941
diff
changeset
|
256 |
var browser_info = Browser_Info.Config.none |
75394 | 257 |
var requirements = false |
258 |
var soft_build = false |
|
259 |
var exclude_session_groups: List[String] = Nil |
|
260 |
var all_sessions = false |
|
261 |
var build_heap = false |
|
262 |
var clean_build = false |
|
263 |
var dirs: List[Path] = Nil |
|
264 |
var export_files = false |
|
265 |
var fresh_build = false |
|
266 |
var session_groups: List[String] = Nil |
|
267 |
var max_jobs = 1 |
|
268 |
var check_keywords: Set[String] = Set.empty |
|
269 |
var list_files = false |
|
270 |
var no_build = false |
|
271 |
var options = Options.init(opts = build_options) |
|
272 |
var verbose = false |
|
273 |
var exclude_sessions: List[String] = Nil |
|
62590 | 274 |
|
75394 | 275 |
val getopts = Getopts(""" |
62590 | 276 |
Usage: isabelle build [OPTIONS] [SESSIONS ...] |
277 |
||
278 |
Options are: |
|
66737
2edc0c42c883
option -B for "isabelle build" and "isabelle imports";
wenzelm
parents:
66736
diff
changeset
|
279 |
-B NAME include session NAME and all descendants |
62590 | 280 |
-D DIR include session directory and select its sessions |
64265 | 281 |
-N cyclic shuffling of NUMA CPU nodes (performance tuning) |
72648 | 282 |
-P DIR enable HTML/PDF presentation in directory (":" for default) |
71807 | 283 |
-R refer to requirements of selected sessions |
66745 | 284 |
-S soft build: only observe changes of sources, not heap images |
62590 | 285 |
-X NAME exclude sessions from group NAME and all descendants |
286 |
-a select all sessions |
|
287 |
-b build heap images |
|
288 |
-c clean build |
|
289 |
-d DIR include session directory |
|
69811
18f61ce86425
clarified 'export_files' in session ROOT: require explicit "isabelle build -e";
wenzelm
parents:
69777
diff
changeset
|
290 |
-e export files from session specification into file-system |
66841 | 291 |
-f fresh build |
62590 | 292 |
-g NAME select session group NAME |
293 |
-j INT maximum number of parallel jobs (default 1) |
|
294 |
-k KEYWORD check theory sources for conflicts with proposed keywords |
|
295 |
-l list session source files |
|
77554
4465d9dff448
clarified terminology of "session build database", while "build database" is the one underlying Build_Process;
wenzelm
parents:
77553
diff
changeset
|
296 |
-n no build -- take existing session build databases |
62590 | 297 |
-o OPTION override Isabelle system OPTION (via NAME=VAL or NAME) |
298 |
-v verbose |
|
299 |
-x NAME exclude session NAME and all descendants |
|
300 |
||
62596 | 301 |
Build and manage Isabelle sessions, depending on implicit settings: |
302 |
||
73736 | 303 |
""" + Library.indent_lines(2, Build_Log.Settings.show()) + "\n", |
75394 | 304 |
"B:" -> (arg => base_sessions = base_sessions ::: List(arg)), |
305 |
"D:" -> (arg => select_dirs = select_dirs ::: List(Path.explode(arg))), |
|
306 |
"N" -> (_ => numa_shuffling = true), |
|
75942
603852abed8f
clarified names: Browser_Info.Config vs. Browser_Info.Context;
wenzelm
parents:
75941
diff
changeset
|
307 |
"P:" -> (arg => browser_info = Browser_Info.Config.make(arg)), |
75394 | 308 |
"R" -> (_ => requirements = true), |
309 |
"S" -> (_ => soft_build = true), |
|
310 |
"X:" -> (arg => exclude_session_groups = exclude_session_groups ::: List(arg)), |
|
311 |
"a" -> (_ => all_sessions = true), |
|
312 |
"b" -> (_ => build_heap = true), |
|
313 |
"c" -> (_ => clean_build = true), |
|
314 |
"d:" -> (arg => dirs = dirs ::: List(Path.explode(arg))), |
|
315 |
"e" -> (_ => export_files = true), |
|
316 |
"f" -> (_ => fresh_build = true), |
|
317 |
"g:" -> (arg => session_groups = session_groups ::: List(arg)), |
|
318 |
"j:" -> (arg => max_jobs = Value.Int.parse(arg)), |
|
319 |
"k:" -> (arg => check_keywords = check_keywords + arg), |
|
320 |
"l" -> (_ => list_files = true), |
|
321 |
"n" -> (_ => no_build = true), |
|
322 |
"o:" -> (arg => options = options + arg), |
|
323 |
"v" -> (_ => verbose = true), |
|
324 |
"x:" -> (arg => exclude_sessions = exclude_sessions ::: List(arg))) |
|
62590 | 325 |
|
75394 | 326 |
val sessions = getopts(args) |
62590 | 327 |
|
75394 | 328 |
val progress = new Console_Progress(verbose = verbose) |
62590 | 329 |
|
75394 | 330 |
val start_date = Date.now() |
64140 | 331 |
|
77521
5642de4d225d
clarified signature: manage "verbose" flag via "progress";
wenzelm
parents:
77505
diff
changeset
|
332 |
progress.echo( |
5642de4d225d
clarified signature: manage "verbose" flag via "progress";
wenzelm
parents:
77505
diff
changeset
|
333 |
"Started at " + Build_Log.print_date(start_date) + |
77558 | 334 |
" (" + Isabelle_System.getenv("ML_IDENTIFIER") + " on " + hostname(options) +")", |
77521
5642de4d225d
clarified signature: manage "verbose" flag via "progress";
wenzelm
parents:
77505
diff
changeset
|
335 |
verbose = true) |
5642de4d225d
clarified signature: manage "verbose" flag via "progress";
wenzelm
parents:
77505
diff
changeset
|
336 |
progress.echo(Build_Log.Settings.show() + "\n", verbose = true) |
62590 | 337 |
|
75394 | 338 |
val results = |
339 |
progress.interrupt_handler { |
|
340 |
build(options, |
|
341 |
selection = Sessions.Selection( |
|
342 |
requirements = requirements, |
|
343 |
all_sessions = all_sessions, |
|
344 |
base_sessions = base_sessions, |
|
345 |
exclude_session_groups = exclude_session_groups, |
|
346 |
exclude_sessions = exclude_sessions, |
|
347 |
session_groups = session_groups, |
|
348 |
sessions = sessions), |
|
75942
603852abed8f
clarified names: Browser_Info.Config vs. Browser_Info.Context;
wenzelm
parents:
75941
diff
changeset
|
349 |
browser_info = browser_info, |
75394 | 350 |
progress = progress, |
351 |
check_unknown_files = Mercurial.is_repository(Path.ISABELLE_HOME), |
|
352 |
build_heap = build_heap, |
|
353 |
clean_build = clean_build, |
|
354 |
dirs = dirs, |
|
355 |
select_dirs = select_dirs, |
|
77477 | 356 |
numa_shuffling = Host.numa_check(progress, numa_shuffling), |
75394 | 357 |
max_jobs = max_jobs, |
358 |
list_files = list_files, |
|
359 |
check_keywords = check_keywords, |
|
360 |
fresh_build = fresh_build, |
|
361 |
no_build = no_build, |
|
362 |
soft_build = soft_build, |
|
363 |
export_files = export_files) |
|
364 |
} |
|
77545 | 365 |
val stop_date = Date.now() |
366 |
val elapsed_time = stop_date.time - start_date.time |
|
75394 | 367 |
|
77545 | 368 |
progress.echo("\nFinished at " + Build_Log.print_date(stop_date), verbose = true) |
62590 | 369 |
|
75394 | 370 |
val total_timing = |
371 |
results.sessions.iterator.map(a => results(a).timing).foldLeft(Timing.zero)(_ + _). |
|
372 |
copy(elapsed = elapsed_time) |
|
373 |
progress.echo(total_timing.message_resources) |
|
62590 | 374 |
|
75394 | 375 |
sys.exit(results.rc) |
376 |
}) |
|
68305 | 377 |
|
378 |
||
77553 | 379 |
|
77557 | 380 |
/** "isabelle build_worker" **/ |
381 |
||
382 |
/* build_worker */ |
|
383 |
||
384 |
def build_worker( |
|
385 |
options: Options, |
|
386 |
build_uuid: String, |
|
387 |
progress: Progress = new Progress, |
|
388 |
dirs: List[Path] = Nil, |
|
389 |
infos: List[Sessions.Info] = Nil, |
|
390 |
numa_shuffling: Boolean = false, |
|
391 |
max_jobs: Int = 1, |
|
77559
4ad322ee6025
clarified signature: support all arguments of Sessions.store();
wenzelm
parents:
77558
diff
changeset
|
392 |
session_setup: (String, Session) => Unit = (_, _) => (), |
4ad322ee6025
clarified signature: support all arguments of Sessions.store();
wenzelm
parents:
77558
diff
changeset
|
393 |
cache: Term.Cache = Term.Cache.make() |
77557 | 394 |
): Results = { |
77559
4ad322ee6025
clarified signature: support all arguments of Sessions.store();
wenzelm
parents:
77558
diff
changeset
|
395 |
val store = build_init(options, cache = cache) |
77557 | 396 |
val build_options = store.options |
397 |
||
398 |
progress.echo("build worker for " + build_uuid) |
|
399 |
progress.echo_warning("FIXME") |
|
400 |
??? |
|
401 |
} |
|
402 |
||
403 |
||
404 |
/* command-line wrapper */ |
|
405 |
||
406 |
val isabelle_tool2 = Isabelle_Tool("build_worker", "external worker for session build process", |
|
407 |
Scala_Project.here, |
|
408 |
{ args => |
|
409 |
val build_options = Word.explode(Isabelle_System.getenv("ISABELLE_BUILD_OPTIONS")) |
|
410 |
||
411 |
var numa_shuffling = false |
|
412 |
var dirs: List[Path] = Nil |
|
413 |
var max_jobs = 1 |
|
414 |
var options = Options.init(opts = build_options) |
|
415 |
var verbose = false |
|
416 |
var build_uuid = "" |
|
417 |
||
418 |
val getopts = Getopts(""" |
|
419 |
Usage: isabelle build_worker [OPTIONS] ...] |
|
420 |
||
421 |
Options are: |
|
422 |
-N cyclic shuffling of NUMA CPU nodes (performance tuning) |
|
423 |
-U UUID Universally Unique Identifier of the build process |
|
424 |
-d DIR include session directory |
|
425 |
-j INT maximum number of parallel jobs (default 1) |
|
426 |
-o OPTION override Isabelle system OPTION (via NAME=VAL or NAME) |
|
427 |
-v verbose |
|
428 |
||
429 |
Run as external worker for session build process, as identified via |
|
430 |
option -U UUID. |
|
431 |
""" + Library.indent_lines(2, Build_Log.Settings.show()) + "\n", |
|
432 |
"N" -> (_ => numa_shuffling = true), |
|
433 |
"U:" -> (arg => build_uuid = arg), |
|
434 |
"d:" -> (arg => dirs = dirs ::: List(Path.explode(arg))), |
|
435 |
"j:" -> (arg => max_jobs = Value.Int.parse(arg)), |
|
436 |
"o:" -> (arg => options = options + arg), |
|
437 |
"v" -> (_ => verbose = true)) |
|
438 |
||
439 |
val more_args = getopts(args) |
|
440 |
if (more_args.nonEmpty) getopts.usage() |
|
441 |
||
442 |
if (build_uuid.isEmpty) error("Missing UUID for build process (option -U)") |
|
443 |
||
444 |
val progress = new Console_Progress(verbose = verbose) |
|
445 |
||
446 |
val results = |
|
447 |
progress.interrupt_handler { |
|
448 |
build_worker(options, build_uuid, |
|
449 |
progress = progress, |
|
450 |
dirs = dirs, |
|
451 |
numa_shuffling = Host.numa_check(progress, numa_shuffling), |
|
452 |
max_jobs = max_jobs) |
|
453 |
} |
|
454 |
||
455 |
sys.exit(results.rc) |
|
456 |
}) |
|
457 |
||
458 |
||
459 |
||
77563 | 460 |
/** "isabelle build_log" **/ |
77553 | 461 |
|
462 |
/* theory markup/messages from session database */ |
|
463 |
||
464 |
def read_theory( |
|
465 |
theory_context: Export.Theory_Context, |
|
466 |
unicode_symbols: Boolean = false |
|
467 |
): Option[Document.Snapshot] = { |
|
468 |
def decode_bytes(bytes: Bytes): String = |
|
469 |
Symbol.output(unicode_symbols, UTF8.decode_permissive(bytes)) |
|
470 |
||
471 |
def read(name: String): Export.Entry = theory_context(name, permissive = true) |
|
472 |
||
473 |
def read_xml(name: String): XML.Body = |
|
474 |
YXML.parse_body(decode_bytes(read(name).bytes), cache = theory_context.cache) |
|
475 |
||
476 |
def read_source_file(name: String): Sessions.Source_File = |
|
477 |
theory_context.session_context.source_file(name) |
|
478 |
||
479 |
for { |
|
480 |
id <- theory_context.document_id() |
|
481 |
(thy_file, blobs_files) <- theory_context.files(permissive = true) |
|
482 |
} |
|
483 |
yield { |
|
484 |
val master_dir = |
|
485 |
Path.explode(Url.strip_base_name(thy_file).getOrElse( |
|
486 |
error("Cannot determine theory master directory: " + quote(thy_file)))) |
|
487 |
||
488 |
val blobs = |
|
489 |
blobs_files.map { name => |
|
490 |
val path = Path.explode(name) |
|
491 |
val src_path = File.relative_path(master_dir, path).getOrElse(path) |
|
492 |
||
493 |
val file = read_source_file(name) |
|
494 |
val bytes = file.bytes |
|
495 |
val text = decode_bytes(bytes) |
|
496 |
val chunk = Symbol.Text_Chunk(text) |
|
497 |
||
498 |
Command.Blob(Document.Node.Name(name), src_path, Some((file.digest, chunk))) -> |
|
499 |
Document.Blobs.Item(bytes, text, chunk, changed = false) |
|
500 |
} |
|
501 |
||
502 |
val thy_source = decode_bytes(read_source_file(thy_file).bytes) |
|
503 |
val thy_xml = read_xml(Export.MARKUP) |
|
504 |
val blobs_xml = |
|
505 |
for (i <- (1 to blobs.length).toList) |
|
506 |
yield read_xml(Export.MARKUP + i) |
|
507 |
||
508 |
val markups_index = Command.Markup_Index.make(blobs.map(_._1)) |
|
509 |
val markups = |
|
510 |
Command.Markups.make( |
|
511 |
for ((index, xml) <- markups_index.zip(thy_xml :: blobs_xml)) |
|
512 |
yield index -> Markup_Tree.from_XML(xml)) |
|
513 |
||
514 |
val results = |
|
515 |
Command.Results.make( |
|
516 |
for (elem @ XML.Elem(Markup(_, Markup.Serial(i)), _) <- read_xml(Export.MESSAGES)) |
|
517 |
yield i -> elem) |
|
518 |
||
519 |
val command = |
|
520 |
Command.unparsed(thy_source, theory = true, id = id, |
|
521 |
node_name = Document.Node.Name(thy_file, theory = theory_context.theory), |
|
522 |
blobs_info = Command.Blobs_Info.make(blobs), |
|
523 |
markups = markups, results = results) |
|
524 |
||
525 |
val doc_blobs = Document.Blobs.make(blobs) |
|
526 |
||
527 |
Document.State.init.snippet(command, doc_blobs) |
|
528 |
} |
|
529 |
} |
|
530 |
||
531 |
||
532 |
/* print messages */ |
|
533 |
||
534 |
def print_log( |
|
535 |
options: Options, |
|
536 |
sessions: List[String], |
|
537 |
theories: List[String] = Nil, |
|
538 |
message_head: List[Regex] = Nil, |
|
539 |
message_body: List[Regex] = Nil, |
|
540 |
progress: Progress = new Progress, |
|
541 |
margin: Double = Pretty.default_margin, |
|
542 |
breakgain: Double = Pretty.default_breakgain, |
|
543 |
metric: Pretty.Metric = Symbol.Metric, |
|
544 |
unicode_symbols: Boolean = false |
|
545 |
): Unit = { |
|
546 |
val store = Sessions.store(options) |
|
547 |
val session = new Session(options, Resources.bootstrap) |
|
548 |
||
549 |
def check(filter: List[Regex], make_string: => String): Boolean = |
|
550 |
filter.isEmpty || { |
|
551 |
val s = Output.clean_yxml(make_string) |
|
552 |
filter.forall(r => r.findFirstIn(Output.clean_yxml(s)).nonEmpty) |
|
553 |
} |
|
554 |
||
555 |
def print(session_name: String): Unit = { |
|
556 |
using(Export.open_session_context0(store, session_name)) { session_context => |
|
557 |
val result = |
|
558 |
for { |
|
559 |
db <- session_context.session_db() |
|
560 |
theories = store.read_theories(db, session_name) |
|
561 |
errors = store.read_errors(db, session_name) |
|
562 |
info <- store.read_build(db, session_name) |
|
563 |
} yield (theories, errors, info.return_code) |
|
564 |
result match { |
|
565 |
case None => store.error_database(session_name) |
|
566 |
case Some((used_theories, errors, rc)) => |
|
567 |
theories.filterNot(used_theories.toSet) match { |
|
568 |
case Nil => |
|
569 |
case bad => error("Unknown theories " + commas_quote(bad)) |
|
570 |
} |
|
571 |
val print_theories = |
|
572 |
if (theories.isEmpty) used_theories else used_theories.filter(theories.toSet) |
|
573 |
||
574 |
for (thy <- print_theories) { |
|
575 |
val thy_heading = "\nTheory " + quote(thy) + " (in " + session_name + ")" + ":" |
|
576 |
||
577 |
Build_Job.read_theory(session_context.theory(thy), unicode_symbols = unicode_symbols) match { |
|
578 |
case None => progress.echo(thy_heading + " MISSING") |
|
579 |
case Some(snapshot) => |
|
580 |
val rendering = new Rendering(snapshot, options, session) |
|
581 |
val messages = |
|
582 |
rendering.text_messages(Text.Range.full) |
|
583 |
.filter(message => progress.verbose || Protocol.is_exported(message.info)) |
|
584 |
if (messages.nonEmpty) { |
|
585 |
val line_document = Line.Document(snapshot.node.source) |
|
586 |
val buffer = new mutable.ListBuffer[String] |
|
587 |
for (Text.Info(range, elem) <- messages) { |
|
588 |
val line = line_document.position(range.start).line1 |
|
589 |
val pos = Position.Line_File(line, snapshot.node_name.node) |
|
590 |
def message_text: String = |
|
591 |
Protocol.message_text(elem, heading = true, pos = pos, |
|
592 |
margin = margin, breakgain = breakgain, metric = metric) |
|
593 |
val ok = |
|
594 |
check(message_head, Protocol.message_heading(elem, pos)) && |
|
595 |
check(message_body, XML.content(Pretty.unformatted(List(elem)))) |
|
596 |
if (ok) buffer += message_text |
|
597 |
} |
|
598 |
if (buffer.nonEmpty) { |
|
599 |
progress.echo(thy_heading) |
|
600 |
buffer.foreach(progress.echo(_)) |
|
601 |
} |
|
602 |
} |
|
603 |
} |
|
604 |
} |
|
605 |
||
606 |
if (errors.nonEmpty) { |
|
607 |
val msg = Symbol.output(unicode_symbols, cat_lines(errors)) |
|
608 |
progress.echo("\nBuild errors:\n" + Output.error_message_text(msg)) |
|
609 |
} |
|
610 |
if (rc != Process_Result.RC.ok) { |
|
611 |
progress.echo("\n" + Process_Result.RC.print_long(rc)) |
|
612 |
} |
|
613 |
} |
|
614 |
} |
|
615 |
} |
|
616 |
||
617 |
val errors = new mutable.ListBuffer[String] |
|
618 |
for (session_name <- sessions) { |
|
619 |
Exn.interruptible_capture(print(session_name)) match { |
|
620 |
case Exn.Res(_) => |
|
621 |
case Exn.Exn(exn) => errors += Exn.message(exn) |
|
622 |
} |
|
623 |
} |
|
624 |
if (errors.nonEmpty) error(cat_lines(errors.toList)) |
|
625 |
} |
|
626 |
||
627 |
||
628 |
/* command-line wrapper */ |
|
629 |
||
77563 | 630 |
val isabelle_tool3 = Isabelle_Tool("build_log", "print messages from session build database", |
77553 | 631 |
Scala_Project.here, |
632 |
{ args => |
|
633 |
/* arguments */ |
|
634 |
||
635 |
var message_head = List.empty[Regex] |
|
636 |
var message_body = List.empty[Regex] |
|
637 |
var unicode_symbols = false |
|
638 |
var theories: List[String] = Nil |
|
639 |
var margin = Pretty.default_margin |
|
640 |
var options = Options.init() |
|
641 |
var verbose = false |
|
642 |
||
643 |
val getopts = Getopts(""" |
|
77563 | 644 |
Usage: isabelle build_log [OPTIONS] [SESSIONS ...] |
77553 | 645 |
|
646 |
Options are: |
|
647 |
-H REGEX filter messages by matching against head |
|
648 |
-M REGEX filter messages by matching against body |
|
649 |
-T NAME restrict to given theories (multiple options possible) |
|
650 |
-U output Unicode symbols |
|
651 |
-m MARGIN margin for pretty printing (default: """ + margin + """) |
|
652 |
-o OPTION override Isabelle system OPTION (via NAME=VAL or NAME) |
|
653 |
-v print all messages, including information etc. |
|
654 |
||
77554
4465d9dff448
clarified terminology of "session build database", while "build database" is the one underlying Build_Process;
wenzelm
parents:
77553
diff
changeset
|
655 |
Print messages from the session build database of the given sessions, |
4465d9dff448
clarified terminology of "session build database", while "build database" is the one underlying Build_Process;
wenzelm
parents:
77553
diff
changeset
|
656 |
without any checks against current sources nor session structure: results |
4465d9dff448
clarified terminology of "session build database", while "build database" is the one underlying Build_Process;
wenzelm
parents:
77553
diff
changeset
|
657 |
from old sessions or failed builds can be printed as well. |
77553 | 658 |
|
659 |
Multiple options -H and -M are conjunctive: all given patterns need to |
|
660 |
match. Patterns match any substring, but ^ or $ may be used to match the |
|
661 |
start or end explicitly. |
|
662 |
""", |
|
663 |
"H:" -> (arg => message_head = message_head ::: List(arg.r)), |
|
664 |
"M:" -> (arg => message_body = message_body ::: List(arg.r)), |
|
665 |
"T:" -> (arg => theories = theories ::: List(arg)), |
|
666 |
"U" -> (_ => unicode_symbols = true), |
|
667 |
"m:" -> (arg => margin = Value.Double.parse(arg)), |
|
668 |
"o:" -> (arg => options = options + arg), |
|
669 |
"v" -> (_ => verbose = true)) |
|
670 |
||
671 |
val sessions = getopts(args) |
|
672 |
||
673 |
val progress = new Console_Progress(verbose = verbose) |
|
674 |
||
675 |
if (sessions.isEmpty) progress.echo_warning("No sessions to print") |
|
676 |
else { |
|
677 |
print_log(options, sessions, theories = theories, message_head = message_head, |
|
678 |
message_body = message_body, margin = margin, progress = progress, |
|
679 |
unicode_symbols = unicode_symbols) |
|
680 |
} |
|
681 |
}) |
|
48276 | 682 |
} |