src/Pure/Admin/isabelle_cronjob.scala
changeset 64231 dbc8294c75d3
parent 64220 e7cbf81ec4b7
child 64232 367d83d6030e
     1.1 --- a/src/Pure/Admin/isabelle_cronjob.scala	Sat Oct 15 20:51:41 2016 +0200
     1.2 +++ b/src/Pure/Admin/isabelle_cronjob.scala	Sat Oct 15 21:02:39 2016 +0200
     1.3 @@ -20,8 +20,9 @@
     1.4    val current_log = main_dir + Path.explode("run/main.log")  // owned by log service
     1.5    val cumulative_log = main_dir + Path.explode("log/main.log")  // owned by log service
     1.6  
     1.7 -  val isabelle_repos = main_dir + Path.explode("isabelle-build_history")
     1.8 -  val afp_repos = main_dir + Path.explode("AFP-build_history")
     1.9 +  val isabelle_repos = main_dir + Path.explode("isabelle")
    1.10 +  val isabelle_repos_test = main_dir + Path.explode("isabelle-test")
    1.11 +  val afp_repos = main_dir + Path.explode("AFP")
    1.12  
    1.13    val release_snapshot = Path.explode("~/html-data/release_snapshot")
    1.14  
    1.15 @@ -34,8 +35,10 @@
    1.16    private val isabelle_identify =
    1.17      Logger_Task("isabelle_identify", logger =>
    1.18        {
    1.19 -        val isabelle_id = Mercurial.repository(isabelle_repos).pull_id()
    1.20 -        val afp_id = Mercurial.repository(afp_repos).pull_id()
    1.21 +        val isabelle_id = Mercurial.repository(isabelle_repos).identify(options = "-i")
    1.22 +        val afp_id =
    1.23 +          Mercurial.setup_repository(
    1.24 +            logger.cronjob_options.string("afp_repos"), afp_repos).pull_id()
    1.25  
    1.26          File.write(logger.log_dir + Build_Log.log_filename("isabelle_identify", logger.start_date),
    1.27            terminate_lines(
    1.28 @@ -53,8 +56,8 @@
    1.29        {
    1.30          for {
    1.31            (result, log_path) <-
    1.32 -            Build_History.build_history(Mercurial.repository(isabelle_repos),
    1.33 -              rev = "build_history_base", fresh = true, build_args = List("FOL"))
    1.34 +            Build_History.build_history(Mercurial.repository(isabelle_repos_test),
    1.35 +              rev = "build_history_base", fresh = true, build_args = List("HOL"))
    1.36          } {
    1.37            result.check
    1.38            File.copy(log_path, logger.log_dir + log_path.base)
    1.39 @@ -83,12 +86,48 @@
    1.40          }))
    1.41  
    1.42  
    1.43 +  /* remote build_history */
    1.44 +
    1.45 +  private sealed case class Remote_Build(
    1.46 +    host: String,
    1.47 +    user: String = "",
    1.48 +    port: Int = SSH.default_port,
    1.49 +    shared_home: Boolean = false,
    1.50 +    options: String = "",
    1.51 +    args: String = "-a")
    1.52 +
    1.53 +  private val remote_builds =
    1.54 +    List(
    1.55 +      Remote_Build("lxbroy10", options = "-m32 -M4", shared_home = true),
    1.56 +      Remote_Build("macbroy2", options = "-m32 -M4"))
    1.57 +
    1.58 +  private def remote_build_history(rev: String, r: Remote_Build): Logger_Task =
    1.59 +    Logger_Task("build_history-" + r.host, logger =>
    1.60 +      {
    1.61 +        using(logger.ssh_context.open_session(host = r.host, user = r.user, port = r.port))(
    1.62 +          session =>
    1.63 +            {
    1.64 +              val results =
    1.65 +                Build_History.remote_build_history(session,
    1.66 +                  isabelle_repos,
    1.67 +                  isabelle_repos.ext(r.host),
    1.68 +                  isabelle_repos_source = logger.cronjob_options.string("isabelle_repos"),
    1.69 +                  self_update = !r.shared_home,
    1.70 +                  options = r.options + " -f -r " + File.bash_string(rev),
    1.71 +                  args = r.args)
    1.72 +              for ((log, bytes) <- results)
    1.73 +                Bytes.write(logger.log_dir + Path.explode(log), bytes)
    1.74 +            })
    1.75 +      })
    1.76 +
    1.77 +
    1.78  
    1.79    /** task logging **/
    1.80  
    1.81    sealed case class Logger_Task(name: String = "", body: Logger => Unit)
    1.82  
    1.83 -  class Log_Service private[Isabelle_Cronjob](progress: Progress)
    1.84 +  class Log_Service private[Isabelle_Cronjob](
    1.85 +    progress: Progress, val cronjob_options: Options, val ssh_context: SSH)
    1.86    {
    1.87      current_log.file.delete
    1.88  
    1.89 @@ -134,6 +173,9 @@
    1.90    class Logger private[Isabelle_Cronjob](
    1.91      val log_service: Log_Service, val start_date: Date, val task_name: String)
    1.92    {
    1.93 +    def cronjob_options: Options = log_service.cronjob_options
    1.94 +    def ssh_context: SSH = log_service.ssh_context
    1.95 +
    1.96      def log(date: Date, msg: String): Unit = log_service.log(date, task_name, msg)
    1.97  
    1.98      def log_end(end_date: Date, err: Option[String])
    1.99 @@ -161,8 +203,6 @@
   1.100  
   1.101    /** cronjob **/
   1.102  
   1.103 -  def init_options(): Options = Options.load(Path.explode("~~/Admin/cronjob/cronjob.options"))
   1.104 -
   1.105    def cronjob(progress: Progress, exclude_task: Set[String])
   1.106    {
   1.107      /* soft lock */
   1.108 @@ -180,7 +220,9 @@
   1.109  
   1.110      /* log service */
   1.111  
   1.112 -    val log_service = new Log_Service(progress)
   1.113 +    val cronjob_options = Options.load(Path.explode("~~/Admin/cronjob/cronjob.options"))
   1.114 +    val ssh_context = SSH.init(Options.init())
   1.115 +    val log_service = new Log_Service(progress, cronjob_options, ssh_context)
   1.116  
   1.117      def run(start_date: Date, task: Logger_Task) { log_service.run_task(start_date, task) }
   1.118  
   1.119 @@ -216,9 +258,13 @@
   1.120      val main_start_date = Date.now()
   1.121      File.write(main_state_file, main_start_date + " " + log_service.hostname)
   1.122  
   1.123 +    val rev = Mercurial.repository(isabelle_repos).identify(options = "-i")
   1.124 +
   1.125      run(main_start_date,
   1.126        Logger_Task("isabelle_cronjob", _ =>
   1.127 -        run_now(SEQ(isabelle_identify, build_history_base, build_release))))
   1.128 +        run_now(
   1.129 +          SEQ(isabelle_identify, build_history_base, build_release,
   1.130 +            PAR(remote_builds.map(remote_build_history(rev, _)):_*)))))
   1.131  
   1.132      log_service.shutdown()
   1.133