290 |
290 |
291 private def echo(msg: String) { java.lang.System.out.println(msg) } |
291 private def echo(msg: String) { java.lang.System.out.println(msg) } |
292 private def sleep(): Unit = Thread.sleep(500) |
292 private def sleep(): Unit = Thread.sleep(500) |
293 |
293 |
294 |
294 |
295 /* dependencies */ |
295 /* source dependencies */ |
296 |
296 |
297 sealed case class Node( |
297 sealed case class Node( |
298 loaded_theories: Set[String], |
298 loaded_theories: Set[String], |
299 sources: List[(Path, SHA1.Digest)]) |
299 sources: List[(Path, SHA1.Digest)]) |
300 |
300 |
301 sealed case class Deps(deps: Map[String, Node]) |
301 sealed case class Deps(deps: Map[String, Node]) |
302 { |
302 { |
303 def sources(name: String): List[(Path, SHA1.Digest)] = deps(name).sources |
303 def sources(name: String): List[SHA1.Digest] = deps(name).sources.map(_._2) |
304 } |
304 } |
305 |
|
306 private def read_deps(file: JFile): List[String] = |
|
307 if (file.isFile) { |
|
308 val stream = new GZIPInputStream (new BufferedInputStream(new FileInputStream(file))) |
|
309 val reader = new BufferedReader(new InputStreamReader(stream, Standard_System.charset)) |
|
310 try { |
|
311 List(reader.readLine).filter(_.startsWith("sources:")) ::: |
|
312 List(reader.readLine).filter(_.startsWith("heap:")) |
|
313 } |
|
314 finally { reader.close } |
|
315 } |
|
316 else Nil |
|
317 |
305 |
318 def dependencies(verbose: Boolean, queue: Session.Queue): Deps = |
306 def dependencies(verbose: Boolean, queue: Session.Queue): Deps = |
319 Deps((Map.empty[String, Node] /: queue.topological_order)( |
307 Deps((Map.empty[String, Node] /: queue.topological_order)( |
320 { case (deps, (name, info)) => |
308 { case (deps, (name, info)) => |
321 val preloaded = |
309 val preloaded = |
428 } |
416 } |
429 new Job(cwd, env, script, YXML.string_of_body(args_xml), output_path) |
417 new Job(cwd, env, script, YXML.string_of_body(args_xml), output_path) |
430 } |
418 } |
431 |
419 |
432 |
420 |
|
421 /* log files and corresponding heaps */ |
|
422 |
|
423 val LOG = Path.explode("log") |
|
424 def log(name: String): Path = LOG + Path.basic(name) |
|
425 def log_gz(name: String): Path = log(name).ext("gz") |
|
426 |
|
427 def sources_stamp(digests: List[SHA1.Digest]): String = |
|
428 digests.map(_.toString).sorted.mkString("sources: ", " ", "") |
|
429 |
|
430 def heap_stamp(output: Option[Path]): String = |
|
431 { |
|
432 "heap: " + |
|
433 (output match { |
|
434 case Some(path) => |
|
435 val file = path.file |
|
436 if (file.isFile) file.length.toString + " " + file.lastModified.toString |
|
437 else "-" |
|
438 case None => "-" |
|
439 }) |
|
440 } |
|
441 |
|
442 def check_stamps(dir: Path, name: String): Option[(String, Boolean)] = |
|
443 { |
|
444 val file = (dir + log_gz(name)).file |
|
445 if (file.isFile) { |
|
446 val stream = new GZIPInputStream (new BufferedInputStream(new FileInputStream(file))) |
|
447 val reader = new BufferedReader(new InputStreamReader(stream, Standard_System.charset)) |
|
448 val (s, h) = try { (reader.readLine, reader.readLine) } finally { reader.close } |
|
449 if (s != null && s.startsWith("sources: ") && h != null && h.startsWith("heap: ") && |
|
450 h == heap_stamp(Some(dir + Path.basic(name)))) Some((s, h != "heap: -")) |
|
451 else None |
|
452 } |
|
453 else None |
|
454 } |
|
455 |
|
456 |
433 /* build */ |
457 /* build */ |
434 |
458 |
435 def build(all_sessions: Boolean, build_images: Boolean, max_jobs: Int, |
459 def build(all_sessions: Boolean, build_images: Boolean, max_jobs: Int, |
436 no_build: Boolean, system_mode: Boolean, timing: Boolean, verbose: Boolean, |
460 no_build: Boolean, system_mode: Boolean, timing: Boolean, verbose: Boolean, |
437 more_dirs: List[Path], more_options: List[String], sessions: List[String]): Int = |
461 more_dirs: List[Path], more_options: List[String], sessions: List[String]): Int = |
438 { |
462 { |
439 val options = (Options.init() /: more_options)(_.define_simple(_)) |
463 val options = (Options.init() /: more_options)(_.define_simple(_)) |
440 val queue = find_sessions(options, all_sessions, sessions, more_dirs) |
464 val queue = find_sessions(options, all_sessions, sessions, more_dirs) |
441 val deps = dependencies(verbose, queue) |
465 val deps = dependencies(verbose, queue) |
442 |
466 |
443 val (output_dir, browser_info) = |
467 def make_stamp(name: String): String = |
444 if (system_mode) (Path.explode("~~/heaps/$ML_IDENTIFIER"), Path.explode("~~/browser_info")) |
468 sources_stamp(queue(name).digest :: deps.sources(name)) |
445 else (Path.explode("$ISABELLE_OUTPUT"), Path.explode("$ISABELLE_BROWSER_INFO")) |
469 |
|
470 val (input_dirs, output_dir, browser_info) = |
|
471 if (system_mode) { |
|
472 val output_dir = Path.explode("~~/heaps/$ML_IDENTIFIER") |
|
473 (List(output_dir), output_dir, Path.explode("~~/browser_info")) |
|
474 } |
|
475 else { |
|
476 val output_dir = Path.explode("$ISABELLE_OUTPUT") |
|
477 (output_dir :: Isabelle_System.find_logics_dirs(), output_dir, |
|
478 Path.explode("$ISABELLE_BROWSER_INFO")) |
|
479 } |
446 |
480 |
447 // prepare log dir |
481 // prepare log dir |
448 val log_dir = output_dir + Path.explode("log") |
482 (output_dir + LOG).file.mkdirs() |
449 log_dir.file.mkdirs() |
|
450 |
483 |
451 // scheduler loop |
484 // scheduler loop |
452 @tailrec def loop( |
485 @tailrec def loop( |
453 pending: Session.Queue, |
486 pending: Session.Queue, |
454 running: Map[String, Job], |
487 running: Map[String, Job], |
455 results: Map[String, Int]): Map[String, Int] = |
488 results: Map[String, Int]): Map[String, Int] = |
456 { |
489 { |
457 if (pending.is_empty) results |
490 if (pending.is_empty) results |
458 else if (running.exists({ case (_, job) => job.is_finished })) { |
491 else if (running.exists({ case (_, job) => job.is_finished })) |
|
492 { // finish job |
459 val (name, job) = running.find({ case (_, job) => job.is_finished }).get |
493 val (name, job) = running.find({ case (_, job) => job.is_finished }).get |
460 |
494 |
461 val (out, err, rc) = job.join |
495 val (out, err, rc) = job.join |
462 echo(Library.trim_line(err)) |
496 echo(Library.trim_line(err)) |
463 |
497 |
464 val log = log_dir + Path.basic(name) |
|
465 if (rc == 0) { |
498 if (rc == 0) { |
466 val sources = |
499 val sources = make_stamp(name) |
467 (queue(name).digest :: deps.sources(name).map(_._2)).map(_.toString).sorted |
500 val heap = heap_stamp(job.output_path) |
468 .mkString("sources: ", " ", "\n") |
501 File.write_gzip(output_dir + log_gz(name), sources + "\n" + heap + "\n" + out) |
469 val heap = |
|
470 job.output_path.map(_.file) match { |
|
471 case Some(file) => |
|
472 "heap: " + file.length.toString + " " + file.lastModified.toString + "\n" |
|
473 case None => "" |
|
474 } |
|
475 File.write_gzip(log.ext("gz"), sources + heap + out) |
|
476 } |
502 } |
477 else { |
503 else { |
478 File.write(log, out) |
504 File.write(output_dir + log(name), out) |
479 echo(name + " FAILED") |
505 echo(name + " FAILED") |
480 echo("(see also " + log.file + ")") |
506 echo("(see also " + log(name).file.toString + ")") |
481 val lines = split_lines(out) |
507 val lines = split_lines(out) |
482 val tail = lines.drop(lines.length - 20 max 0) |
508 val tail = lines.drop(lines.length - 20 max 0) |
483 echo("\n" + cat_lines(tail)) |
509 echo("\n" + cat_lines(tail)) |
484 } |
510 } |
485 loop(pending - name, running - name, results + (name -> rc)) |
511 loop(pending - name, running - name, results + (name -> rc)) |
486 } |
512 } |
487 else if (running.size < (max_jobs max 1)) { |
513 else if (running.size < (max_jobs max 1)) |
|
514 { // check/start next job |
488 pending.dequeue(running.isDefinedAt(_)) match { |
515 pending.dequeue(running.isDefinedAt(_)) match { |
489 case Some((name, info)) => |
516 case Some((name, info)) => |
490 if (no_build) { |
517 val output = |
491 loop(pending - name, running, results + (name -> 0)) |
518 if (build_images || queue.is_inner(name)) |
|
519 Some(output_dir + Path.basic(name)) |
|
520 else None |
|
521 |
|
522 val current = |
|
523 { |
|
524 input_dirs.find(dir => (dir + log_gz(name)).file.isFile) match { |
|
525 case Some(dir) => |
|
526 check_stamps(dir, name) match { |
|
527 case Some((s, h)) => s == make_stamp(name) && (h || output.isEmpty) |
|
528 case None => false |
|
529 } |
|
530 case None => false |
|
531 } |
492 } |
532 } |
|
533 if (current || no_build) |
|
534 loop(pending - name, running, results + (name -> (if (current) 0 else 1))) |
493 else if (info.parent.map(results(_)).forall(_ == 0)) { |
535 else if (info.parent.map(results(_)).forall(_ == 0)) { |
494 val output = |
|
495 if (build_images || queue.is_inner(name)) |
|
496 Some(output_dir + Path.basic(name)) |
|
497 else None |
|
498 echo((if (output.isDefined) "Building " else "Running ") + name + " ...") |
536 echo((if (output.isDefined) "Building " else "Running ") + name + " ...") |
499 val job = start_job(name, info, output, info.options, timing, verbose, browser_info) |
537 val job = start_job(name, info, output, info.options, timing, verbose, browser_info) |
500 loop(pending, running + (name -> job), results) |
538 loop(pending, running + (name -> job), results) |
501 } |
539 } |
502 else { |
540 else { |