author | wenzelm |
Sat, 29 Dec 2018 14:58:51 +0100 | |
changeset 69535 | e3a9680d9ed8 |
parent 69534 | 913970ae2324 |
child 69536 | 892b68f932f9 |
permissions | -rw-r--r-- |
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
1 |
/* Title: Pure/Tools/dump.scala |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
2 |
Author: Makarius |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
3 |
|
68348 | 4 |
Dump cumulative PIDE session database. |
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
5 |
*/ |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
6 |
|
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
7 |
package isabelle |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
8 |
|
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
9 |
|
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
10 |
object Dump |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
11 |
{ |
68316 | 12 |
/* aspects */ |
13 |
||
14 |
sealed case class Aspect_Args( |
|
68355 | 15 |
options: Options, |
16 |
progress: Progress, |
|
68365 | 17 |
deps: Sessions.Deps, |
68355 | 18 |
output_dir: Path, |
68929 | 19 |
snapshot: Document.Snapshot, |
69534 | 20 |
status: Document_Status.Node_Status) |
68319 | 21 |
{ |
68365 | 22 |
def write(file_name: Path, bytes: Bytes) |
68319 | 23 |
{ |
68929 | 24 |
val path = output_dir + Path.basic(snapshot.node_name.theory) + file_name |
68319 | 25 |
Isabelle_System.mkdirs(path.dir) |
26 |
Bytes.write(path, bytes) |
|
27 |
} |
|
28 |
||
68365 | 29 |
def write(file_name: Path, text: String): Unit = |
30 |
write(file_name, Bytes(text)) |
|
68332 | 31 |
|
68365 | 32 |
def write(file_name: Path, body: XML.Body): Unit = |
33 |
write(file_name, Symbol.encode(YXML.string_of_body(body))) |
|
68319 | 34 |
} |
68316 | 35 |
|
68347 | 36 |
sealed case class Aspect(name: String, description: String, operation: Aspect_Args => Unit, |
37 |
options: List[String] = Nil) |
|
68345 | 38 |
{ |
39 |
override def toString: String = name |
|
40 |
} |
|
68316 | 41 |
|
69521 | 42 |
val known_aspects: List[Aspect] = |
68316 | 43 |
List( |
68365 | 44 |
Aspect("markup", "PIDE markup (YXML format)", |
45 |
{ case args => |
|
46 |
args.write(Path.explode("markup.yxml"), |
|
47 |
args.snapshot.markup_to_XML(Text.Range.full, Markup.Elements.full)) |
|
48 |
}), |
|
68319 | 49 |
Aspect("messages", "output messages (YXML format)", |
50 |
{ case args => |
|
68365 | 51 |
args.write(Path.explode("messages.yxml"), |
52 |
args.snapshot.messages.iterator.map(_._1).toList) |
|
68347 | 53 |
}), |
54 |
Aspect("latex", "generated LaTeX source", |
|
55 |
{ case args => |
|
68365 | 56 |
for (entry <- args.snapshot.exports if entry.name == "document.tex") |
57 |
args.write(Path.explode(entry.name), entry.uncompressed()) |
|
69533
1d2e6a4e073f
clarified options: ensure consolidated Node_Status and thus percentage = 100% for progress;
wenzelm
parents:
69532
diff
changeset
|
58 |
}, options = List("export_document")), |
68347 | 59 |
Aspect("theory", "foundational theory content", |
60 |
{ case args => |
|
61 |
for { |
|
68365 | 62 |
entry <- args.snapshot.exports |
68347 | 63 |
if entry.name.startsWith(Export_Theory.export_prefix) |
68365 | 64 |
} args.write(Path.explode(entry.name), entry.uncompressed()) |
69533
1d2e6a4e073f
clarified options: ensure consolidated Node_Status and thus percentage = 100% for progress;
wenzelm
parents:
69532
diff
changeset
|
65 |
}, options = List("export_theory")) |
68345 | 66 |
).sortBy(_.name) |
68316 | 67 |
|
68 |
def show_aspects: String = |
|
68345 | 69 |
cat_lines(known_aspects.map(aspect => aspect.name + " - " + aspect.description)) |
68316 | 70 |
|
71 |
def the_aspect(name: String): Aspect = |
|
72 |
known_aspects.find(aspect => aspect.name == name) getOrElse |
|
73 |
error("Unknown aspect " + quote(name)) |
|
74 |
||
75 |
||
69523 | 76 |
/* session */ |
68316 | 77 |
|
69523 | 78 |
def session(dump_options: Options, logic: String, |
69535 | 79 |
process_theory: (Sessions.Deps, Document.Snapshot, Document_Status.Node_Status) => Unit, |
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
80 |
progress: Progress = No_Progress, |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
81 |
log: Logger = No_Logger, |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
82 |
dirs: List[Path] = Nil, |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
83 |
select_dirs: List[Path] = Nil, |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
84 |
system_mode: Boolean = false, |
69535 | 85 |
selection: Sessions.Selection = Sessions.Selection.empty) |
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
86 |
{ |
68318 | 87 |
/* dependencies */ |
88 |
||
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
89 |
val deps = |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
90 |
Sessions.load_structure(dump_options, dirs = dirs, select_dirs = select_dirs). |
69532 | 91 |
selection_deps(dump_options, selection, progress = progress, |
92 |
uniform_session = true, loading_sessions = true) |
|
68318 | 93 |
|
69535 | 94 |
val use_theories = |
95 |
for { (_, name) <- deps.used_theories_condition(dump_options, progress.echo_warning) } |
|
96 |
yield name.theory |
|
68318 | 97 |
|
69535 | 98 |
|
99 |
/* asynchronous consumer */ |
|
68926 | 100 |
|
101 |
object Consumer |
|
102 |
{ |
|
69535 | 103 |
sealed case class Bad_Theory( |
104 |
name: Document.Node.Name, |
|
105 |
status: Document_Status.Node_Status, |
|
106 |
errors: List[String]) |
|
107 |
||
108 |
private val consumer_bad_theories = Synchronized(List.empty[Bad_Theory]) |
|
68926 | 109 |
|
110 |
private val consumer = |
|
111 |
Consumer_Thread.fork(name = "dump")( |
|
112 |
consume = (args: (Document.Snapshot, Document_Status.Node_Status)) => |
|
113 |
{ |
|
69534 | 114 |
val (snapshot, status) = args |
69535 | 115 |
val name = snapshot.node_name |
116 |
if (status.ok) { |
|
117 |
try { process_theory(deps, snapshot, status) } |
|
118 |
catch { |
|
119 |
case exn: Throwable if !Exn.is_interrupt(exn) => |
|
120 |
val msg = Exn.message(exn) |
|
121 |
progress.echo("FAILED to process theory " + name) |
|
122 |
progress.echo_error_message(msg) |
|
123 |
consumer_bad_theories.change(Bad_Theory(name, status, List(msg)) :: _) |
|
124 |
} |
|
125 |
} |
|
68930 | 126 |
else { |
69535 | 127 |
val msgs = |
128 |
for ((tree, pos) <- snapshot.messages if Protocol.is_error(tree)) |
|
129 |
yield { |
|
130 |
"Error" + Position.here(pos) + ":\n" + |
|
131 |
XML.content(Pretty.formatted(List(tree))) |
|
132 |
} |
|
133 |
progress.echo("FAILED to process theory " + name) |
|
134 |
msgs.foreach(progress.echo_error_message) |
|
135 |
consumer_bad_theories.change(Bad_Theory(name, status, msgs) :: _) |
|
68926 | 136 |
} |
137 |
true |
|
138 |
}) |
|
139 |
||
69534 | 140 |
def apply(snapshot: Document.Snapshot, status: Document_Status.Node_Status): Unit = |
141 |
consumer.send((snapshot, status)) |
|
68926 | 142 |
|
69535 | 143 |
def shutdown(): List[Bad_Theory] = |
68926 | 144 |
{ |
145 |
consumer.shutdown() |
|
69535 | 146 |
consumer_bad_theories.value.reverse |
68926 | 147 |
} |
148 |
} |
|
149 |
||
150 |
||
69523 | 151 |
/* run session */ |
68318 | 152 |
|
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
153 |
val session = |
69532 | 154 |
Headless.start_session(dump_options, logic, progress = progress, log = log, |
155 |
session_dirs = dirs ::: select_dirs, |
|
156 |
include_sessions = deps.sessions_structure.imports_topological_order) |
|
157 |
||
69535 | 158 |
try { |
159 |
val use_theories_result = |
|
160 |
session.use_theories(use_theories, progress = progress, commit = Some(Consumer.apply _)) |
|
68320
1d33697199c1
shutdown ML process before output: Theories_Result is timeless/stateless;
wenzelm
parents:
68319
diff
changeset
|
161 |
|
69535 | 162 |
val bad_theories = Consumer.shutdown() |
163 |
val bad_msgs = |
|
164 |
bad_theories.map(bad => |
|
165 |
Output.clean_yxml( |
|
166 |
"FAILED theory " + bad.name + |
|
167 |
(if (bad.status.consolidated) "" else ": " + bad.status.percentage + "% finished") + |
|
168 |
(if (bad.errors.isEmpty) "" else bad.errors.mkString("\n", "\n", "")))) |
|
68320
1d33697199c1
shutdown ML process before output: Theories_Result is timeless/stateless;
wenzelm
parents:
68319
diff
changeset
|
169 |
|
69535 | 170 |
val pending_msgs = |
171 |
use_theories_result.nodes_pending match { |
|
172 |
case Nil => Nil |
|
173 |
case pending => List("Pending theories: " + commas(pending.map(p => p._1.toString))) |
|
174 |
} |
|
175 |
||
176 |
val errors = bad_msgs ::: pending_msgs |
|
177 |
if (errors.nonEmpty) error(errors.mkString("\n\n")) |
|
69032
90bb4cabe1e8
clarified errors: no result from forced session.stop, check pending theories;
wenzelm
parents:
69026
diff
changeset
|
178 |
} |
69535 | 179 |
finally { session.stop() } |
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
180 |
} |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
181 |
|
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
182 |
|
69523 | 183 |
/* dump */ |
184 |
||
185 |
val default_output_dir: Path = Path.explode("dump") |
|
186 |
||
187 |
def make_options(options: Options, aspects: List[Aspect] = Nil): Options = |
|
188 |
{ |
|
189 |
val options0 = if (NUMA.enabled) NUMA.policy_options(options) else options |
|
190 |
val options1 = |
|
191 |
options0 + "completion_limit=0" + "ML_statistics=false" + |
|
69533
1d2e6a4e073f
clarified options: ensure consolidated Node_Status and thus percentage = 100% for progress;
wenzelm
parents:
69532
diff
changeset
|
192 |
"parallel_proofs=0" + "editor_tracing_messages=0" + "editor_presentation" |
69523 | 193 |
(options1 /: aspects)({ case (opts, aspect) => (opts /: aspect.options)(_ + _) }) |
194 |
} |
|
195 |
||
196 |
def dump(options: Options, logic: String, |
|
197 |
aspects: List[Aspect] = Nil, |
|
198 |
progress: Progress = No_Progress, |
|
199 |
log: Logger = No_Logger, |
|
200 |
dirs: List[Path] = Nil, |
|
201 |
select_dirs: List[Path] = Nil, |
|
202 |
output_dir: Path = default_output_dir, |
|
203 |
system_mode: Boolean = false, |
|
69535 | 204 |
selection: Sessions.Selection = Sessions.Selection.empty) |
69523 | 205 |
{ |
206 |
if (Build.build_logic(options, logic, build_heap = true, progress = progress, |
|
207 |
dirs = dirs ::: select_dirs, system_mode = system_mode) != 0) error(logic + " FAILED") |
|
208 |
||
209 |
val dump_options = make_options(options, aspects) |
|
210 |
||
69535 | 211 |
def process_theory( |
69523 | 212 |
deps: Sessions.Deps, |
213 |
snapshot: Document.Snapshot, |
|
69534 | 214 |
status: Document_Status.Node_Status) |
69523 | 215 |
{ |
69534 | 216 |
val aspect_args = Aspect_Args(dump_options, progress, deps, output_dir, snapshot, status) |
69523 | 217 |
aspects.foreach(_.operation(aspect_args)) |
218 |
} |
|
219 |
||
69535 | 220 |
session(dump_options, logic, process_theory _, |
69523 | 221 |
progress = progress, log = log, dirs = dirs, select_dirs = select_dirs, selection = selection) |
222 |
} |
|
223 |
||
224 |
||
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
225 |
/* Isabelle tool wrapper */ |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
226 |
|
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
227 |
val isabelle_tool = |
68348 | 228 |
Isabelle_Tool("dump", "dump cumulative PIDE session database", args => |
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
229 |
{ |
68345 | 230 |
var aspects: List[Aspect] = known_aspects |
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
231 |
var base_sessions: List[String] = Nil |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
232 |
var select_dirs: List[Path] = Nil |
68316 | 233 |
var output_dir = default_output_dir |
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
234 |
var requirements = false |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
235 |
var exclude_session_groups: List[String] = Nil |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
236 |
var all_sessions = false |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
237 |
var dirs: List[Path] = Nil |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
238 |
var session_groups: List[String] = Nil |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
239 |
var logic = Isabelle_System.getenv("ISABELLE_LOGIC") |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
240 |
var options = Options.init() |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
241 |
var system_mode = false |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
242 |
var verbose = false |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
243 |
var exclude_sessions: List[String] = Nil |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
244 |
|
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
245 |
val getopts = Getopts(""" |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
246 |
Usage: isabelle dump [OPTIONS] [SESSIONS ...] |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
247 |
|
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
248 |
Options are: |
68345 | 249 |
-A NAMES dump named aspects (default: """ + known_aspects.mkString("\"", ",", "\"") + """) |
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
250 |
-B NAME include session NAME and all descendants |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
251 |
-D DIR include session directory and select its sessions |
68316 | 252 |
-O DIR output directory for dumped files (default: """ + default_output_dir + """) |
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
253 |
-R operate on requirements of selected sessions |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
254 |
-X NAME exclude sessions from group NAME and all descendants |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
255 |
-a select all sessions |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
256 |
-d DIR include session directory |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
257 |
-g NAME select session group NAME |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
258 |
-l NAME logic session name (default ISABELLE_LOGIC=""" + quote(logic) + """) |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
259 |
-o OPTION override Isabelle system OPTION (via NAME=VAL or NAME) |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
260 |
-s system build mode for logic image |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
261 |
-v verbose |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
262 |
-x NAME exclude session NAME and all descendants |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
263 |
|
68348 | 264 |
Dump cumulative PIDE session database, with the following aspects: |
68316 | 265 |
|
266 |
""" + Library.prefix_lines(" ", show_aspects) + "\n", |
|
267 |
"A:" -> (arg => aspects = Library.distinct(space_explode(',', arg)).map(the_aspect(_))), |
|
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
268 |
"B:" -> (arg => base_sessions = base_sessions ::: List(arg)), |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
269 |
"D:" -> (arg => select_dirs = select_dirs ::: List(Path.explode(arg))), |
68316 | 270 |
"O:" -> (arg => output_dir = Path.explode(arg)), |
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
271 |
"R" -> (_ => requirements = true), |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
272 |
"X:" -> (arg => exclude_session_groups = exclude_session_groups ::: List(arg)), |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
273 |
"a" -> (_ => all_sessions = true), |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
274 |
"d:" -> (arg => dirs = dirs ::: List(Path.explode(arg))), |
68741 | 275 |
"g:" -> (arg => session_groups = session_groups ::: List(arg)), |
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
276 |
"l:" -> (arg => logic = arg), |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
277 |
"o:" -> (arg => options = options + arg), |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
278 |
"s" -> (_ => system_mode = true), |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
279 |
"v" -> (_ => verbose = true), |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
280 |
"x:" -> (arg => exclude_sessions = exclude_sessions ::: List(arg))) |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
281 |
|
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
282 |
val sessions = getopts(args) |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
283 |
|
68951
a7b1fe2d30ad
more uniform Progress, with theory() for batch-build and theory_percentage() for PIDE session;
wenzelm
parents:
68948
diff
changeset
|
284 |
val progress = new Console_Progress(verbose = verbose) |
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
285 |
|
69535 | 286 |
progress.interrupt_handler { |
287 |
dump(options, logic, |
|
288 |
aspects = aspects, |
|
289 |
progress = progress, |
|
290 |
dirs = dirs, |
|
291 |
select_dirs = select_dirs, |
|
292 |
output_dir = output_dir, |
|
293 |
selection = Sessions.Selection( |
|
294 |
requirements = requirements, |
|
295 |
all_sessions = all_sessions, |
|
296 |
base_sessions = base_sessions, |
|
297 |
exclude_session_groups = exclude_session_groups, |
|
298 |
exclude_sessions = exclude_sessions, |
|
299 |
session_groups = session_groups, |
|
300 |
sessions = sessions)) |
|
301 |
} |
|
68308
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
302 |
}) |
119fc05f6b00
support to dump build database produced by PIDE session;
wenzelm
parents:
diff
changeset
|
303 |
} |