# HG changeset patch # User wenzelm # Date 1476558159 -7200 # Node ID dbc8294c75d313b8a4158d38c89102cefcb65db7 # Parent 13a97c1d7d22f23d1ebeebed2096f95ac98fa252 added remote_build_history tasks: parallel on several remote hosts; isabelle_identify: use self repos for robustness; more logger context: options and ssh; setup repository clones on demand; clarified target repositories; diff -r 13a97c1d7d22 -r dbc8294c75d3 src/Pure/Admin/build_history.scala --- a/src/Pure/Admin/build_history.scala Sat Oct 15 20:51:41 2016 +0200 +++ b/src/Pure/Admin/build_history.scala Sat Oct 15 21:02:39 2016 +0200 @@ -342,18 +342,14 @@ { using(session.sftp())(sftp => { - val isabelle_admin = (isabelle_repos_self + Path.explode("Admin")).implode + val isabelle_admin = sftp.path(isabelle_repos_self + Path.explode("Admin")) /* prepare repository clones */ - val ssh = Some(session) - val isabelle_hg = - if (sftp.stat(isabelle_repos_self.implode).nonEmpty) - Mercurial.repository(isabelle_repos_self, ssh = ssh) - else - Mercurial.clone_repository(isabelle_repos_source, isabelle_repos_self, ssh = ssh) + Mercurial.setup_repository( + isabelle_repos_source, isabelle_repos_self, ssh = Some(session)) if (self_update) { isabelle_hg.pull() @@ -361,8 +357,8 @@ session.execute(File.bash_string(isabelle_admin + "/build") + " jars_fresh").check } - if (sftp.stat(isabelle_repos_other.implode).isEmpty) - Mercurial.clone_repository(isabelle_repos_self.implode, isabelle_repos_other, ssh = ssh) + Mercurial.setup_repository( + sftp.path(isabelle_repos_self), isabelle_repos_other, ssh = Some(session)) /* Admin/build_history */ @@ -370,7 +366,7 @@ val result = session.execute( File.bash_string(isabelle_admin + "/build_history") + " " + options + " " + - File.bash_path(isabelle_repos_other) + " " + args, + File.bash_string(sftp.path(isabelle_repos_other)) + " " + args, progress_stderr = progress.echo(_)) result.check.out_lines.map(log => (Path.explode(log).base.implode, sftp.read_bytes(log))) diff -r 13a97c1d7d22 -r dbc8294c75d3 src/Pure/Admin/isabelle_cronjob.scala --- a/src/Pure/Admin/isabelle_cronjob.scala Sat Oct 15 20:51:41 2016 +0200 +++ b/src/Pure/Admin/isabelle_cronjob.scala Sat Oct 15 21:02:39 2016 +0200 @@ -20,8 +20,9 @@ val current_log = main_dir + Path.explode("run/main.log") // owned by log service val cumulative_log = main_dir + Path.explode("log/main.log") // owned by log service - val isabelle_repos = main_dir + Path.explode("isabelle-build_history") - val afp_repos = main_dir + Path.explode("AFP-build_history") + val isabelle_repos = main_dir + Path.explode("isabelle") + val isabelle_repos_test = main_dir + Path.explode("isabelle-test") + val afp_repos = main_dir + Path.explode("AFP") val release_snapshot = Path.explode("~/html-data/release_snapshot") @@ -34,8 +35,10 @@ private val isabelle_identify = Logger_Task("isabelle_identify", logger => { - val isabelle_id = Mercurial.repository(isabelle_repos).pull_id() - val afp_id = Mercurial.repository(afp_repos).pull_id() + val isabelle_id = Mercurial.repository(isabelle_repos).identify(options = "-i") + val afp_id = + Mercurial.setup_repository( + logger.cronjob_options.string("afp_repos"), afp_repos).pull_id() File.write(logger.log_dir + Build_Log.log_filename("isabelle_identify", logger.start_date), terminate_lines( @@ -53,8 +56,8 @@ { for { (result, log_path) <- - Build_History.build_history(Mercurial.repository(isabelle_repos), - rev = "build_history_base", fresh = true, build_args = List("FOL")) + Build_History.build_history(Mercurial.repository(isabelle_repos_test), + rev = "build_history_base", fresh = true, build_args = List("HOL")) } { result.check File.copy(log_path, logger.log_dir + log_path.base) @@ -83,12 +86,48 @@ })) + /* remote build_history */ + + private sealed case class Remote_Build( + host: String, + user: String = "", + port: Int = SSH.default_port, + shared_home: Boolean = false, + options: String = "", + args: String = "-a") + + private val remote_builds = + List( + Remote_Build("lxbroy10", options = "-m32 -M4", shared_home = true), + Remote_Build("macbroy2", options = "-m32 -M4")) + + private def remote_build_history(rev: String, r: Remote_Build): Logger_Task = + Logger_Task("build_history-" + r.host, logger => + { + using(logger.ssh_context.open_session(host = r.host, user = r.user, port = r.port))( + session => + { + val results = + Build_History.remote_build_history(session, + isabelle_repos, + isabelle_repos.ext(r.host), + isabelle_repos_source = logger.cronjob_options.string("isabelle_repos"), + self_update = !r.shared_home, + options = r.options + " -f -r " + File.bash_string(rev), + args = r.args) + for ((log, bytes) <- results) + Bytes.write(logger.log_dir + Path.explode(log), bytes) + }) + }) + + /** task logging **/ sealed case class Logger_Task(name: String = "", body: Logger => Unit) - class Log_Service private[Isabelle_Cronjob](progress: Progress) + class Log_Service private[Isabelle_Cronjob]( + progress: Progress, val cronjob_options: Options, val ssh_context: SSH) { current_log.file.delete @@ -134,6 +173,9 @@ class Logger private[Isabelle_Cronjob]( val log_service: Log_Service, val start_date: Date, val task_name: String) { + def cronjob_options: Options = log_service.cronjob_options + def ssh_context: SSH = log_service.ssh_context + def log(date: Date, msg: String): Unit = log_service.log(date, task_name, msg) def log_end(end_date: Date, err: Option[String]) @@ -161,8 +203,6 @@ /** cronjob **/ - def init_options(): Options = Options.load(Path.explode("~~/Admin/cronjob/cronjob.options")) - def cronjob(progress: Progress, exclude_task: Set[String]) { /* soft lock */ @@ -180,7 +220,9 @@ /* log service */ - val log_service = new Log_Service(progress) + val cronjob_options = Options.load(Path.explode("~~/Admin/cronjob/cronjob.options")) + val ssh_context = SSH.init(Options.init()) + val log_service = new Log_Service(progress, cronjob_options, ssh_context) def run(start_date: Date, task: Logger_Task) { log_service.run_task(start_date, task) } @@ -216,9 +258,13 @@ val main_start_date = Date.now() File.write(main_state_file, main_start_date + " " + log_service.hostname) + val rev = Mercurial.repository(isabelle_repos).identify(options = "-i") + run(main_start_date, Logger_Task("isabelle_cronjob", _ => - run_now(SEQ(isabelle_identify, build_history_base, build_release)))) + run_now( + SEQ(isabelle_identify, build_history_base, build_release, + PAR(remote_builds.map(remote_build_history(rev, _)):_*))))) log_service.shutdown()