author | wenzelm |
Sun, 07 May 2017 23:18:23 +0200 | |
changeset 65764 | 1af6d544c2a3 |
parent 65747 | 5a3052b2095f |
child 65768 | b8da621a3297 |
permissions | -rw-r--r-- |
64148 | 1 |
/* Title: Pure/Admin/isabelle_cronjob.scala |
2 |
Author: Makarius |
|
3 |
||
4 |
Main entry point for administrative cronjob at TUM. |
|
5 |
*/ |
|
6 |
||
7 |
package isabelle |
|
8 |
||
9 |
||
64170 | 10 |
import scala.annotation.tailrec |
11 |
import scala.collection.mutable |
|
12 |
||
13 |
||
64148 | 14 |
object Isabelle_Cronjob |
15 |
{ |
|
64194
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
16 |
/* file-system state: owned by main cronjob */ |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
17 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
18 |
val main_dir = Path.explode("~/cronjob") |
64194
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
19 |
val main_state_file = main_dir + Path.explode("run/main.state") |
64219 | 20 |
val current_log = main_dir + Path.explode("run/main.log") // owned by log service |
21 |
val cumulative_log = main_dir + Path.explode("log/main.log") // owned by log service |
|
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
22 |
|
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
23 |
val isabelle_repos = main_dir + Path.explode("isabelle") |
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
24 |
val isabelle_repos_test = main_dir + Path.explode("isabelle-test") |
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
25 |
val afp_repos = main_dir + Path.explode("AFP") |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
26 |
|
64236
358f9ff08681
discontinued somewhat pointless cronjob.options -- compile-time constants are sufficient;
wenzelm
parents:
64235
diff
changeset
|
27 |
val isabelle_dev_source = "http://isabelle.in.tum.de/repos/isabelle" |
358f9ff08681
discontinued somewhat pointless cronjob.options -- compile-time constants are sufficient;
wenzelm
parents:
64235
diff
changeset
|
28 |
val isabelle_release_source = "http://bitbucket.org/isabelle_project/isabelle-release" |
358f9ff08681
discontinued somewhat pointless cronjob.options -- compile-time constants are sufficient;
wenzelm
parents:
64235
diff
changeset
|
29 |
val afp_source = "https://bitbucket.org/isa-afp/afp-devel" |
358f9ff08681
discontinued somewhat pointless cronjob.options -- compile-time constants are sufficient;
wenzelm
parents:
64235
diff
changeset
|
30 |
|
65710 | 31 |
val devel_dir = Path.explode("~/html-data/devel") |
65746 | 32 |
val release_snapshot_dir = devel_dir + Path.explode("release_snapshot") |
33 |
val build_log_db = devel_dir + Path.explode("build_log.db") |
|
34 |
val build_status_dir = devel_dir + Path.explode("build_status") |
|
65707
353b965378cf
produce build_log_snapshot, based on small prefix into pull_date history;
wenzelm
parents:
65694
diff
changeset
|
35 |
|
65736 | 36 |
val jenkins_jobs = "identify" :: Jenkins.build_log_jobs |
65666
45d0692bb019
incremental download of Jenkins log files, which are also added to database;
wenzelm
parents:
65647
diff
changeset
|
37 |
|
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
38 |
|
64192 | 39 |
|
40 |
/** particular tasks **/ |
|
41 |
||
64405
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
42 |
/* identify Isabelle + AFP repository snapshots and build release */ |
64192 | 43 |
|
64405
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
44 |
private val build_release = |
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
45 |
Logger_Task("build_release", logger => |
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
46 |
Isabelle_System.with_tmp_dir("isadist")(base_dir => |
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
47 |
{ |
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
48 |
val rev = Mercurial.repository(isabelle_repos).id() |
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
49 |
val afp_rev = Mercurial.setup_repository(afp_source, afp_repos).id() |
64192 | 50 |
|
64405
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
51 |
File.write(logger.log_dir + Build_Log.log_filename("isabelle_identify", logger.start_date), |
65674
23897f5d885d
approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents:
65670
diff
changeset
|
52 |
Build_Log.Identify.content(logger.start_date, Some(rev), Some(afp_rev))) |
64405
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
53 |
|
65746 | 54 |
val new_snapshot = release_snapshot_dir.ext("new") |
55 |
val old_snapshot = release_snapshot_dir.ext("old") |
|
64405
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
56 |
|
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
57 |
Isabelle_System.rm_tree(new_snapshot) |
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
58 |
Isabelle_System.rm_tree(old_snapshot) |
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
59 |
|
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
60 |
Build_Release.build_release(base_dir, rev = rev, afp_rev = afp_rev, |
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
61 |
parallel_jobs = 4, remote_mac = "macbroy31", website = Some(new_snapshot)) |
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
62 |
|
65746 | 63 |
if (release_snapshot_dir.is_dir) File.move(release_snapshot_dir, old_snapshot) |
64 |
File.move(new_snapshot, release_snapshot_dir) |
|
64405
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
65 |
Isabelle_System.rm_tree(old_snapshot) |
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
66 |
})) |
64192 | 67 |
|
68 |
||
64194
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
69 |
/* integrity test of build_history vs. build_history_base */ |
64193 | 70 |
|
64194
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
71 |
private val build_history_base = |
64193 | 72 |
Logger_Task("build_history_base", logger => |
73 |
{ |
|
64251 | 74 |
val hg = |
75 |
Mercurial.setup_repository( |
|
76 |
File.standard_path(isabelle_repos), isabelle_repos_test) |
|
64194
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
77 |
for { |
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
78 |
(result, log_path) <- |
64251 | 79 |
Build_History.build_history( |
80 |
hg, rev = "build_history_base", fresh = true, build_args = List("HOL")) |
|
64194
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
81 |
} { |
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
82 |
result.check |
64482
43f6c28ff496
clarified File.move: target directory like File.copy;
wenzelm
parents:
64481
diff
changeset
|
83 |
File.move(log_path, logger.log_dir + log_path.base) |
64194
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
84 |
} |
64193 | 85 |
}) |
86 |
||
87 |
||
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
88 |
/* remote build_history */ |
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
89 |
|
64409
70c87ca55f2c
tuned signature -- more friendly for experimentation;
wenzelm
parents:
64405
diff
changeset
|
90 |
sealed case class Remote_Build( |
65764 | 91 |
description: String, |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
92 |
host: String, |
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
93 |
user: String = "", |
65594 | 94 |
port: Int = 0, |
64294
303976a45afe
shared_home is default for classic isatest home setup;
wenzelm
parents:
64277
diff
changeset
|
95 |
shared_home: Boolean = true, |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
96 |
options: String = "", |
65732 | 97 |
args: String = "", |
98 |
detect: SQL.Source = "") |
|
99 |
{ |
|
65747 | 100 |
def profile: Build_Status.Profile = |
101 |
{ |
|
102 |
val sql = |
|
103 |
Build_Log.Prop.build_engine + " = " + SQL.string(Build_History.engine) + " AND " + |
|
104 |
Build_Log.Prop.build_host + " = " + SQL.string(host) + |
|
105 |
(if (detect == "") "" else " AND " + SQL.enclose(detect)) |
|
65764 | 106 |
Build_Status.Profile(description, sql) |
65747 | 107 |
} |
65732 | 108 |
} |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
109 |
|
65747 | 110 |
private val remote_builds: List[List[Remote_Build]] = |
65732 | 111 |
{ |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
112 |
List( |
65764 | 113 |
List(Remote_Build("Poly/ML test", "lxbroy8", |
65063 | 114 |
options = "-m32 -B -M1x2,2 -t polyml-test -e 'init_component /home/isabelle/contrib/polyml-5.7-20170217'", |
65732 | 115 |
args = "-N -g timing", |
116 |
detect = Build_Log.Prop.build_tags + " = " + SQL.string("polyml-test"))), |
|
65764 | 117 |
List(Remote_Build("Linux A", "lxbroy9", |
65732 | 118 |
options = "-m32 -B -M1x2,2", args = "-N -g timing")), |
65764 | 119 |
List(Remote_Build("Linux B", "lxbroy10", |
65732 | 120 |
options = "-m32 -B -M1x4,2,4,6", args = "-N -g timing")), |
64351 | 121 |
List( |
65764 | 122 |
Remote_Build("Mac OS X Mavericks", "macbroy2", options = "-m32 -M8", args = "-a", |
65732 | 123 |
detect = Build_Log.Prop.build_tags + " IS NULL"), |
65764 | 124 |
Remote_Build("Mac OS X Mavericks, quick_and_dirty", "macbroy2", |
65732 | 125 |
options = "-m32 -M8 -t quick_and_dirty", args = "-a -o quick_and_dirty", |
126 |
detect = Build_Log.Prop.build_tags + " = " + SQL.string("quick_and_dirty")), |
|
65764 | 127 |
Remote_Build("Mac OS X Mavericks, skip_proofs", "macbroy2", |
65732 | 128 |
options = "-m32 -M8 -t skip_proofs", args = "-a -o skip_proofs", |
129 |
detect = Build_Log.Prop.build_tags + " = " + SQL.string("skip_proofs"))), |
|
65764 | 130 |
List(Remote_Build("Mac OS X Yosemite", "macbroy30", options = "-m32 -M2", args = "-a")), |
131 |
List(Remote_Build("Mac OS X Sierra", "macbroy31", options = "-m32 -M2", args = "-a")), |
|
64351 | 132 |
List( |
65764 | 133 |
Remote_Build("Windows", "vmnipkow9", shared_home = false, |
65732 | 134 |
options = "-m32 -M4", args = "-a", |
135 |
detect = Build_Log.Settings.ML_PLATFORM + " = " + SQL.string("x86-windows")), |
|
65764 | 136 |
Remote_Build("Windows", "vmnipkow9", shared_home = false, |
65732 | 137 |
options = "-m64 -M4", args = "-a", |
138 |
detect = Build_Log.Settings.ML_PLATFORM + " = " + SQL.string("x86_64-windows")))) |
|
139 |
} |
|
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
140 |
|
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
141 |
private def remote_build_history(rev: String, r: Remote_Build): Logger_Task = |
64294
303976a45afe
shared_home is default for classic isatest home setup;
wenzelm
parents:
64277
diff
changeset
|
142 |
{ |
303976a45afe
shared_home is default for classic isatest home setup;
wenzelm
parents:
64277
diff
changeset
|
143 |
val task_name = "build_history-" + r.host |
303976a45afe
shared_home is default for classic isatest home setup;
wenzelm
parents:
64277
diff
changeset
|
144 |
Logger_Task(task_name, logger => |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
145 |
{ |
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
146 |
using(logger.ssh_context.open_session(host = r.host, user = r.user, port = r.port))( |
64259 | 147 |
ssh => |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
148 |
{ |
64348
4c253e84ae62
clarified push/pull chain: current ISABELLE_HOME may server as source for changes that are not published on isabelle_repos_source yet (e.g. isabelle-release branch);
wenzelm
parents:
64346
diff
changeset
|
149 |
val self_update = !r.shared_home |
4c253e84ae62
clarified push/pull chain: current ISABELLE_HOME may server as source for changes that are not published on isabelle_repos_source yet (e.g. isabelle-release branch);
wenzelm
parents:
64346
diff
changeset
|
150 |
val push_isabelle_home = self_update && Mercurial.is_repository(Path.explode("~~")) |
4c253e84ae62
clarified push/pull chain: current ISABELLE_HOME may server as source for changes that are not published on isabelle_repos_source yet (e.g. isabelle-release branch);
wenzelm
parents:
64346
diff
changeset
|
151 |
|
64468
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
152 |
val (results, _) = |
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
153 |
Build_History.remote_build_history(ssh, |
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
154 |
isabelle_repos, |
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
155 |
isabelle_repos.ext(r.host), |
64538 | 156 |
isabelle_repos_source = isabelle_dev_source, |
64468
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
157 |
self_update = self_update, |
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
158 |
push_isabelle_home = push_isabelle_home, |
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
159 |
options = |
64479 | 160 |
"-r " + Bash.string(rev) + " -N " + Bash.string(task_name) + " -f " + r.options, |
64468
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
161 |
args = "-o timeout=10800 " + r.args) |
64346 | 162 |
|
64481 | 163 |
for ((log_name, bytes) <- results) { |
164 |
logger.log(Date.now(), log_name) |
|
64468
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
165 |
Bytes.write(logger.log_dir + Path.explode(log_name), bytes) |
64481 | 166 |
} |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
167 |
}) |
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
168 |
}) |
64294
303976a45afe
shared_home is default for classic isatest home setup;
wenzelm
parents:
64277
diff
changeset
|
169 |
} |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
170 |
|
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
171 |
|
65640 | 172 |
/* maintain build_log database */ |
173 |
||
174 |
val database_dirs = |
|
175 |
List(Path.explode("~/log"), Path.explode("~/afp/log"), Path.explode("~/cronjob/log")) |
|
176 |
||
177 |
def database_update(options: Options) |
|
178 |
{ |
|
179 |
val store = Build_Log.store(options) |
|
65670
490649872acc
create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents:
65667
diff
changeset
|
180 |
using(store.open_database())(db => |
65707
353b965378cf
produce build_log_snapshot, based on small prefix into pull_date history;
wenzelm
parents:
65694
diff
changeset
|
181 |
{ |
353b965378cf
produce build_log_snapshot, based on small prefix into pull_date history;
wenzelm
parents:
65694
diff
changeset
|
182 |
store.update_database(db, database_dirs, ml_statistics = true) |
65746 | 183 |
store.snapshot_database(db, build_log_db) |
65707
353b965378cf
produce build_log_snapshot, based on small prefix into pull_date history;
wenzelm
parents:
65694
diff
changeset
|
184 |
}) |
65640 | 185 |
} |
186 |
||
187 |
||
65746 | 188 |
/* present build status */ |
189 |
||
65747 | 190 |
val build_status_profiles: List[Build_Status.Profile] = |
191 |
remote_builds.flatten.map(_.profile) |
|
192 |
||
65746 | 193 |
def build_status(options: Options) |
194 |
{ |
|
195 |
Build_Status.present_data(Build_Status.read_data(options), target_dir = build_status_dir) |
|
196 |
} |
|
197 |
||
198 |
||
64192 | 199 |
|
200 |
/** task logging **/ |
|
64171 | 201 |
|
64193 | 202 |
sealed case class Logger_Task(name: String = "", body: Logger => Unit) |
64154 | 203 |
|
64257 | 204 |
class Log_Service private[Isabelle_Cronjob](progress: Progress, val ssh_context: SSH.Context) |
64171 | 205 |
{ |
64219 | 206 |
current_log.file.delete |
207 |
||
64171 | 208 |
private val thread: Consumer_Thread[String] = |
209 |
Consumer_Thread.fork("cronjob: logger", daemon = true)( |
|
210 |
consume = (text: String) => |
|
64219 | 211 |
{ // critical |
212 |
File.append(current_log, text + "\n") |
|
213 |
File.append(cumulative_log, text + "\n") |
|
64171 | 214 |
progress.echo(text) |
215 |
true |
|
216 |
}) |
|
217 |
||
218 |
def shutdown() { thread.shutdown() } |
|
219 |
||
220 |
val hostname = Isabelle_System.hostname() |
|
221 |
||
222 |
def log(date: Date, task_name: String, msg: String): Unit = |
|
64193 | 223 |
if (task_name != "") |
224 |
thread.send( |
|
225 |
"[" + Build_Log.print_date(date) + ", " + hostname + ", " + task_name + "]: " + msg) |
|
64171 | 226 |
|
227 |
def start_logger(start_date: Date, task_name: String): Logger = |
|
228 |
new Logger(this, start_date, task_name) |
|
64154 | 229 |
|
64171 | 230 |
def run_task(start_date: Date, task: Logger_Task) |
231 |
{ |
|
232 |
val logger = start_logger(start_date, task.name) |
|
233 |
val res = Exn.capture { task.body(logger) } |
|
234 |
val end_date = Date.now() |
|
235 |
val err = |
|
236 |
res match { |
|
237 |
case Exn.Res(_) => None |
|
64295 | 238 |
case Exn.Exn(exn) => |
64410 | 239 |
System.err.println("Exception trace for " + quote(task.name) + ":") |
64397 | 240 |
exn.printStackTrace() |
64295 | 241 |
val first_line = Library.split_lines(Exn.message(exn)).headOption getOrElse "exception" |
242 |
Some(first_line) |
|
64171 | 243 |
} |
244 |
logger.log_end(end_date, err) |
|
245 |
} |
|
246 |
||
247 |
def fork_task(start_date: Date, task: Logger_Task): Task = |
|
248 |
new Task(task.name, run_task(start_date, task)) |
|
249 |
} |
|
250 |
||
251 |
class Logger private[Isabelle_Cronjob]( |
|
252 |
val log_service: Log_Service, val start_date: Date, val task_name: String) |
|
64162 | 253 |
{ |
64257 | 254 |
def ssh_context: SSH.Context = log_service.ssh_context |
65640 | 255 |
def options: Options = ssh_context.options |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
256 |
|
64171 | 257 |
def log(date: Date, msg: String): Unit = log_service.log(date, task_name, msg) |
258 |
||
259 |
def log_end(end_date: Date, err: Option[String]) |
|
260 |
{ |
|
261 |
val elapsed_time = end_date.time - start_date.time |
|
262 |
val msg = |
|
263 |
(if (err.isEmpty) "finished" else "ERROR " + err.get) + |
|
64197 | 264 |
(if (elapsed_time.seconds < 3.0) "" else " (" + elapsed_time.message_hms + " elapsed time)") |
64171 | 265 |
log(end_date, msg) |
266 |
} |
|
267 |
||
64195 | 268 |
val log_dir: Path = main_dir + Build_Log.log_subdir(start_date) |
269 |
||
270 |
Isabelle_System.mkdirs(log_dir) |
|
64171 | 271 |
log(start_date, "started") |
272 |
} |
|
273 |
||
274 |
class Task private[Isabelle_Cronjob](name: String, body: => Unit) |
|
275 |
{ |
|
276 |
private val future: Future[Unit] = Future.thread("cronjob: " + name) { body } |
|
277 |
def is_finished: Boolean = future.is_finished |
|
64162 | 278 |
} |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
279 |
|
64170 | 280 |
|
281 |
||
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
282 |
/** cronjob **/ |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
283 |
|
64187 | 284 |
def cronjob(progress: Progress, exclude_task: Set[String]) |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
285 |
{ |
64171 | 286 |
/* soft lock */ |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
287 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
288 |
val still_running = |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
289 |
try { Some(File.read(main_state_file)) } |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
290 |
catch { case ERROR(_) => None } |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
291 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
292 |
still_running match { |
64170 | 293 |
case None | Some("") => |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
294 |
case Some(running) => |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
295 |
error("Isabelle cronjob appears to be still running: " + running) |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
296 |
} |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
297 |
|
64199 | 298 |
|
299 |
/* log service */ |
|
300 |
||
64257 | 301 |
val log_service = new Log_Service(progress, SSH.init_context(Options.init())) |
64154 | 302 |
|
64199 | 303 |
def run(start_date: Date, task: Logger_Task) { log_service.run_task(start_date, task) } |
304 |
||
305 |
def run_now(task: Logger_Task) { run(Date.now(), task) } |
|
64154 | 306 |
|
307 |
||
64199 | 308 |
/* structured tasks */ |
64184 | 309 |
|
64350
3af8566788e7
remote_builds has PAR-SEQ semantics of old isatest-makedist;
wenzelm
parents:
64348
diff
changeset
|
310 |
def SEQ(tasks: List[Logger_Task]): Logger_Task = Logger_Task(body = _ => |
64193 | 311 |
for (task <- tasks.iterator if !exclude_task(task.name) || task.name == "") |
64199 | 312 |
run_now(task)) |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
313 |
|
64350
3af8566788e7
remote_builds has PAR-SEQ semantics of old isatest-makedist;
wenzelm
parents:
64348
diff
changeset
|
314 |
def PAR(tasks: List[Logger_Task]): Logger_Task = Logger_Task(body = _ => |
64170 | 315 |
{ |
64199 | 316 |
@tailrec def join(running: List[Task]) |
317 |
{ |
|
318 |
running.partition(_.is_finished) match { |
|
319 |
case (Nil, Nil) => |
|
320 |
case (Nil, _ :: _) => Thread.sleep(500); join(running) |
|
321 |
case (_ :: _, remaining) => join(remaining) |
|
322 |
} |
|
64170 | 323 |
} |
64199 | 324 |
val start_date = Date.now() |
325 |
val running = |
|
64350
3af8566788e7
remote_builds has PAR-SEQ semantics of old isatest-makedist;
wenzelm
parents:
64348
diff
changeset
|
326 |
for (task <- tasks if !exclude_task(task.name)) |
64199 | 327 |
yield log_service.fork_task(start_date, task) |
328 |
join(running) |
|
329 |
}) |
|
64193 | 330 |
|
64170 | 331 |
|
332 |
/* main */ |
|
333 |
||
64199 | 334 |
val main_start_date = Date.now() |
335 |
File.write(main_state_file, main_start_date + " " + log_service.hostname) |
|
336 |
||
64232
367d83d6030e
clarified hg.id operation, with explicit tip as default;
wenzelm
parents:
64231
diff
changeset
|
337 |
val rev = Mercurial.repository(isabelle_repos).id() |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
338 |
|
64193 | 339 |
run(main_start_date, |
340 |
Logger_Task("isabelle_cronjob", _ => |
|
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
341 |
run_now( |
64405
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
342 |
SEQ(List(build_release, build_history_base, |
65640 | 343 |
PAR(remote_builds.map(seq => SEQ(seq.map(remote_build_history(rev, _))))), |
65666
45d0692bb019
incremental download of Jenkins log files, which are also added to database;
wenzelm
parents:
65647
diff
changeset
|
344 |
Logger_Task("jenkins_logs", _ => Jenkins.download_logs(jenkins_jobs, main_dir)), |
65746 | 345 |
Logger_Task("build_log_database", logger => database_update(logger.options)), |
346 |
Logger_Task("build_status", logger => build_status(logger.options))))))) |
|
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
347 |
|
64171 | 348 |
log_service.shutdown() |
64170 | 349 |
|
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
350 |
main_state_file.file.delete |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
351 |
} |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
352 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
353 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
354 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
355 |
/** command line entry point **/ |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
356 |
|
64148 | 357 |
def main(args: Array[String]) |
358 |
{ |
|
359 |
Command_Line.tool0 { |
|
360 |
var force = false |
|
361 |
var verbose = false |
|
64187 | 362 |
var exclude_task = Set.empty[String] |
64148 | 363 |
|
364 |
val getopts = Getopts(""" |
|
365 |
Usage: Admin/cronjob/main [OPTIONS] |
|
366 |
||
367 |
Options are: |
|
368 |
-f apply force to do anything |
|
369 |
-v verbose |
|
64187 | 370 |
-x NAME exclude tasks with this name |
64148 | 371 |
""", |
372 |
"f" -> (_ => force = true), |
|
64187 | 373 |
"v" -> (_ => verbose = true), |
374 |
"x:" -> (arg => exclude_task += arg)) |
|
64148 | 375 |
|
376 |
val more_args = getopts(args) |
|
377 |
if (more_args.nonEmpty) getopts.usage() |
|
378 |
||
64909 | 379 |
val progress = if (verbose) new Console_Progress() else No_Progress |
64148 | 380 |
|
64187 | 381 |
if (force) cronjob(progress, exclude_task) |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
382 |
else error("Need to apply force to do anything") |
64148 | 383 |
} |
384 |
} |
|
385 |
} |