src/Pure/System/build.scala
changeset 48504 21dfd6c04482
parent 48494 00eb5be9e76b
child 48505 d9e43ea3a045
equal deleted inserted replaced
48503:f26b6b364c2c 48504:21dfd6c04482
   290 
   290 
   291   private def echo(msg: String) { java.lang.System.out.println(msg) }
   291   private def echo(msg: String) { java.lang.System.out.println(msg) }
   292   private def sleep(): Unit = Thread.sleep(500)
   292   private def sleep(): Unit = Thread.sleep(500)
   293 
   293 
   294 
   294 
   295   /* dependencies */
   295   /* source dependencies */
   296 
   296 
   297   sealed case class Node(
   297   sealed case class Node(
   298     loaded_theories: Set[String],
   298     loaded_theories: Set[String],
   299     sources: List[(Path, SHA1.Digest)])
   299     sources: List[(Path, SHA1.Digest)])
   300 
   300 
   301   sealed case class Deps(deps: Map[String, Node])
   301   sealed case class Deps(deps: Map[String, Node])
   302   {
   302   {
   303     def sources(name: String): List[(Path, SHA1.Digest)] = deps(name).sources
   303     def sources(name: String): List[SHA1.Digest] = deps(name).sources.map(_._2)
   304   }
   304   }
   305 
       
   306   private def read_deps(file: JFile): List[String] =
       
   307     if (file.isFile) {
       
   308       val stream = new GZIPInputStream (new BufferedInputStream(new FileInputStream(file)))
       
   309       val reader = new BufferedReader(new InputStreamReader(stream, Standard_System.charset))
       
   310       try {
       
   311         List(reader.readLine).filter(_.startsWith("sources:")) :::
       
   312         List(reader.readLine).filter(_.startsWith("heap:"))
       
   313       }
       
   314       finally { reader.close }
       
   315     }
       
   316     else Nil
       
   317 
   305 
   318   def dependencies(verbose: Boolean, queue: Session.Queue): Deps =
   306   def dependencies(verbose: Boolean, queue: Session.Queue): Deps =
   319     Deps((Map.empty[String, Node] /: queue.topological_order)(
   307     Deps((Map.empty[String, Node] /: queue.topological_order)(
   320       { case (deps, (name, info)) =>
   308       { case (deps, (name, info)) =>
   321           val preloaded =
   309           val preloaded =
   428     }
   416     }
   429     new Job(cwd, env, script, YXML.string_of_body(args_xml), output_path)
   417     new Job(cwd, env, script, YXML.string_of_body(args_xml), output_path)
   430   }
   418   }
   431 
   419 
   432 
   420 
       
   421   /* log files and corresponding heaps */
       
   422 
       
   423   val LOG = Path.explode("log")
       
   424   def log(name: String): Path = LOG + Path.basic(name)
       
   425   def log_gz(name: String): Path = log(name).ext("gz")
       
   426 
       
   427   def sources_stamp(digests: List[SHA1.Digest]): String =
       
   428     digests.map(_.toString).sorted.mkString("sources: ", " ", "")
       
   429 
       
   430   def heap_stamp(output: Option[Path]): String =
       
   431   {
       
   432     "heap: " +
       
   433       (output match {
       
   434         case Some(path) =>
       
   435           val file = path.file
       
   436           if (file.isFile) file.length.toString + " " + file.lastModified.toString
       
   437           else "-"
       
   438         case None => "-"
       
   439       })
       
   440   }
       
   441 
       
   442   def check_stamps(dir: Path, name: String): Option[(String, Boolean)] =
       
   443   {
       
   444     val file = (dir + log_gz(name)).file
       
   445     if (file.isFile) {
       
   446       val stream = new GZIPInputStream (new BufferedInputStream(new FileInputStream(file)))
       
   447       val reader = new BufferedReader(new InputStreamReader(stream, Standard_System.charset))
       
   448       val (s, h) = try { (reader.readLine, reader.readLine) } finally { reader.close }
       
   449       if (s != null && s.startsWith("sources: ") && h != null && h.startsWith("heap: ") &&
       
   450           h == heap_stamp(Some(dir + Path.basic(name)))) Some((s, h != "heap: -"))
       
   451       else None
       
   452     }
       
   453     else None
       
   454   }
       
   455 
       
   456 
   433   /* build */
   457   /* build */
   434 
   458 
   435   def build(all_sessions: Boolean, build_images: Boolean, max_jobs: Int,
   459   def build(all_sessions: Boolean, build_images: Boolean, max_jobs: Int,
   436     no_build: Boolean, system_mode: Boolean, timing: Boolean, verbose: Boolean,
   460     no_build: Boolean, system_mode: Boolean, timing: Boolean, verbose: Boolean,
   437     more_dirs: List[Path], more_options: List[String], sessions: List[String]): Int =
   461     more_dirs: List[Path], more_options: List[String], sessions: List[String]): Int =
   438   {
   462   {
   439     val options = (Options.init() /: more_options)(_.define_simple(_))
   463     val options = (Options.init() /: more_options)(_.define_simple(_))
   440     val queue = find_sessions(options, all_sessions, sessions, more_dirs)
   464     val queue = find_sessions(options, all_sessions, sessions, more_dirs)
   441     val deps = dependencies(verbose, queue)
   465     val deps = dependencies(verbose, queue)
   442 
   466 
   443     val (output_dir, browser_info) =
   467     def make_stamp(name: String): String =
   444       if (system_mode) (Path.explode("~~/heaps/$ML_IDENTIFIER"), Path.explode("~~/browser_info"))
   468       sources_stamp(queue(name).digest :: deps.sources(name))
   445       else (Path.explode("$ISABELLE_OUTPUT"), Path.explode("$ISABELLE_BROWSER_INFO"))
   469 
       
   470     val (input_dirs, output_dir, browser_info) =
       
   471       if (system_mode) {
       
   472         val output_dir = Path.explode("~~/heaps/$ML_IDENTIFIER")
       
   473         (List(output_dir), output_dir, Path.explode("~~/browser_info"))
       
   474       }
       
   475       else {
       
   476         val output_dir = Path.explode("$ISABELLE_OUTPUT")
       
   477         (output_dir :: Isabelle_System.find_logics_dirs(), output_dir,
       
   478          Path.explode("$ISABELLE_BROWSER_INFO"))
       
   479       }
   446 
   480 
   447     // prepare log dir
   481     // prepare log dir
   448     val log_dir = output_dir + Path.explode("log")
   482     (output_dir + LOG).file.mkdirs()
   449     log_dir.file.mkdirs()
       
   450 
   483 
   451     // scheduler loop
   484     // scheduler loop
   452     @tailrec def loop(
   485     @tailrec def loop(
   453       pending: Session.Queue,
   486       pending: Session.Queue,
   454       running: Map[String, Job],
   487       running: Map[String, Job],
   455       results: Map[String, Int]): Map[String, Int] =
   488       results: Map[String, Int]): Map[String, Int] =
   456     {
   489     {
   457       if (pending.is_empty) results
   490       if (pending.is_empty) results
   458       else if (running.exists({ case (_, job) => job.is_finished })) {
   491       else if (running.exists({ case (_, job) => job.is_finished }))
       
   492       { // finish job
   459         val (name, job) = running.find({ case (_, job) => job.is_finished }).get
   493         val (name, job) = running.find({ case (_, job) => job.is_finished }).get
   460 
   494 
   461         val (out, err, rc) = job.join
   495         val (out, err, rc) = job.join
   462         echo(Library.trim_line(err))
   496         echo(Library.trim_line(err))
   463 
   497 
   464         val log = log_dir + Path.basic(name)
       
   465         if (rc == 0) {
   498         if (rc == 0) {
   466           val sources =
   499           val sources = make_stamp(name)
   467             (queue(name).digest :: deps.sources(name).map(_._2)).map(_.toString).sorted
   500           val heap = heap_stamp(job.output_path)
   468               .mkString("sources: ", " ", "\n")
   501           File.write_gzip(output_dir + log_gz(name), sources + "\n" + heap + "\n" + out)
   469           val heap =
       
   470             job.output_path.map(_.file) match {
       
   471               case Some(file) =>
       
   472                 "heap: " + file.length.toString + " " + file.lastModified.toString + "\n"
       
   473               case None => ""
       
   474             }
       
   475           File.write_gzip(log.ext("gz"), sources + heap + out)
       
   476         }
   502         }
   477         else {
   503         else {
   478           File.write(log, out)
   504           File.write(output_dir + log(name), out)
   479           echo(name + " FAILED")
   505           echo(name + " FAILED")
   480           echo("(see also " + log.file + ")")
   506           echo("(see also " + log(name).file.toString + ")")
   481           val lines = split_lines(out)
   507           val lines = split_lines(out)
   482           val tail = lines.drop(lines.length - 20 max 0)
   508           val tail = lines.drop(lines.length - 20 max 0)
   483           echo("\n" + cat_lines(tail))
   509           echo("\n" + cat_lines(tail))
   484         }
   510         }
   485         loop(pending - name, running - name, results + (name -> rc))
   511         loop(pending - name, running - name, results + (name -> rc))
   486       }
   512       }
   487       else if (running.size < (max_jobs max 1)) {
   513       else if (running.size < (max_jobs max 1))
       
   514       { // check/start next job
   488         pending.dequeue(running.isDefinedAt(_)) match {
   515         pending.dequeue(running.isDefinedAt(_)) match {
   489           case Some((name, info)) =>
   516           case Some((name, info)) =>
   490             if (no_build) {
   517             val output =
   491               loop(pending - name, running, results + (name -> 0))
   518               if (build_images || queue.is_inner(name))
       
   519                 Some(output_dir + Path.basic(name))
       
   520               else None
       
   521 
       
   522             val current =
       
   523             {
       
   524               input_dirs.find(dir => (dir + log_gz(name)).file.isFile) match {
       
   525                 case Some(dir) =>
       
   526                   check_stamps(dir, name) match {
       
   527                     case Some((s, h)) => s == make_stamp(name) && (h || output.isEmpty)
       
   528                     case None => false
       
   529                   }
       
   530                 case None => false
       
   531               }
   492             }
   532             }
       
   533             if (current || no_build)
       
   534               loop(pending - name, running, results + (name -> (if (current) 0 else 1)))
   493             else if (info.parent.map(results(_)).forall(_ == 0)) {
   535             else if (info.parent.map(results(_)).forall(_ == 0)) {
   494               val output =
       
   495                 if (build_images || queue.is_inner(name))
       
   496                   Some(output_dir + Path.basic(name))
       
   497                 else None
       
   498               echo((if (output.isDefined) "Building " else "Running ") + name + " ...")
   536               echo((if (output.isDefined) "Building " else "Running ") + name + " ...")
   499               val job = start_job(name, info, output, info.options, timing, verbose, browser_info)
   537               val job = start_job(name, info, output, info.options, timing, verbose, browser_info)
   500               loop(pending, running + (name -> job), results)
   538               loop(pending, running + (name -> job), results)
   501             }
   539             }
   502             else {
   540             else {
   509       else { sleep(); loop(pending, running, results) }
   547       else { sleep(); loop(pending, running, results) }
   510     }
   548     }
   511 
   549 
   512     val results = loop(queue, Map.empty, Map.empty)
   550     val results = loop(queue, Map.empty, Map.empty)
   513     val rc = (0 /: results)({ case (rc1, (_, rc2)) => rc1 max rc2 })
   551     val rc = (0 /: results)({ case (rc1, (_, rc2)) => rc1 max rc2 })
   514     if (rc != 0) {
   552     if (rc != 0 && (verbose || !no_build)) {
   515       val unfinished = (for ((name, r) <- results.iterator if r != 0) yield name).toList.sorted
   553       val unfinished = (for ((name, r) <- results.iterator if r != 0) yield name).toList.sorted
   516       echo("Unfinished session(s): " + commas(unfinished))
   554       echo("Unfinished session(s): " + commas(unfinished))
   517     }
   555     }
   518     rc
   556     rc
   519   }
   557   }