author | wenzelm |
Sun, 22 Oct 2017 13:57:25 +0200 | |
changeset 66898 | 8b7c4679c05b |
parent 66897 | d2f983fe3e73 |
child 66902 | f6bc83ffda02 |
permissions | -rw-r--r-- |
64148 | 1 |
/* Title: Pure/Admin/isabelle_cronjob.scala |
2 |
Author: Makarius |
|
3 |
||
4 |
Main entry point for administrative cronjob at TUM. |
|
5 |
*/ |
|
6 |
||
7 |
package isabelle |
|
8 |
||
9 |
||
64170 | 10 |
import scala.annotation.tailrec |
11 |
import scala.collection.mutable |
|
12 |
||
13 |
||
64148 | 14 |
object Isabelle_Cronjob |
15 |
{ |
|
66897 | 16 |
/* global resources: owned by main cronjob */ |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
17 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
18 |
val main_dir = Path.explode("~/cronjob") |
64194
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
19 |
val main_state_file = main_dir + Path.explode("run/main.state") |
64219 | 20 |
val current_log = main_dir + Path.explode("run/main.log") // owned by log service |
21 |
val cumulative_log = main_dir + Path.explode("log/main.log") // owned by log service |
|
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
22 |
|
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
23 |
val isabelle_repos = main_dir + Path.explode("isabelle") |
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
24 |
val isabelle_repos_test = main_dir + Path.explode("isabelle-test") |
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
25 |
val afp_repos = main_dir + Path.explode("AFP") |
64236
358f9ff08681
discontinued somewhat pointless cronjob.options -- compile-time constants are sufficient;
wenzelm
parents:
64235
diff
changeset
|
26 |
|
65736 | 27 |
val jenkins_jobs = "identify" :: Jenkins.build_log_jobs |
65666
45d0692bb019
incremental download of Jenkins log files, which are also added to database;
wenzelm
parents:
65647
diff
changeset
|
28 |
|
66896 | 29 |
val build_log_dirs = |
30 |
List(Path.explode("~/log"), Path.explode("~/afp/log"), Path.explode("~/cronjob/log")) |
|
31 |
||
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
32 |
|
64192 | 33 |
|
34 |
/** particular tasks **/ |
|
35 |
||
64405
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
36 |
/* identify Isabelle + AFP repository snapshots and build release */ |
64192 | 37 |
|
64405
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
38 |
private val build_release = |
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
39 |
Logger_Task("build_release", logger => |
66859 | 40 |
{ |
41 |
Isabelle_Devel.make_index() |
|
65771 | 42 |
|
66859 | 43 |
val rev = Mercurial.repository(isabelle_repos).id() |
44 |
val afp_rev = Mercurial.setup_repository(AFP.repos_source, afp_repos).id() |
|
64192 | 45 |
|
66859 | 46 |
File.write(logger.log_dir + Build_Log.log_filename("isabelle_identify", logger.start_date), |
47 |
Build_Log.Identify.content(logger.start_date, Some(rev), Some(afp_rev))) |
|
64405
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
48 |
|
66859 | 49 |
Isabelle_Devel.release_snapshot(rev = rev, afp_rev = afp_rev, |
50 |
parallel_jobs = 4, remote_mac = "macbroy31") |
|
51 |
}) |
|
64192 | 52 |
|
53 |
||
64194
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
54 |
/* integrity test of build_history vs. build_history_base */ |
64193 | 55 |
|
64194
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
56 |
private val build_history_base = |
64193 | 57 |
Logger_Task("build_history_base", logger => |
58 |
{ |
|
64251 | 59 |
val hg = |
60 |
Mercurial.setup_repository( |
|
61 |
File.standard_path(isabelle_repos), isabelle_repos_test) |
|
64194
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
62 |
for { |
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
63 |
(result, log_path) <- |
66861
f6676691ef8a
partition AFP sessions according to structure, which happens to cut it roughly into equal parts;
wenzelm
parents:
66860
diff
changeset
|
64 |
Build_History.build_history(logger.options, isabelle_repos_test, |
66860
54ae2cc05325
support for AFP in build_history and remote_build_history;
wenzelm
parents:
66859
diff
changeset
|
65 |
rev = "build_history_base", fresh = true, build_args = List("HOL")) |
64194
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
66 |
} { |
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
67 |
result.check |
64482
43f6c28ff496
clarified File.move: target directory like File.copy;
wenzelm
parents:
64481
diff
changeset
|
68 |
File.move(log_path, logger.log_dir + log_path.base) |
64194
b5ada7dcceaa
integrity test of build_history vs. build_history_base;
wenzelm
parents:
64193
diff
changeset
|
69 |
} |
64193 | 70 |
}) |
71 |
||
72 |
||
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
73 |
/* remote build_history */ |
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
74 |
|
66864 | 75 |
sealed case class Item( |
76 |
known: Boolean, |
|
77 |
isabelle_version: String, |
|
78 |
afp_version: Option[String], |
|
79 |
pull_date: Date) |
|
65783
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
80 |
{ |
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
81 |
def unknown: Boolean = !known |
66864 | 82 |
def versions: (String, Option[String]) = (isabelle_version, afp_version) |
83 |
||
84 |
def known_versions(rev: String, afp_rev: Option[String]): Boolean = |
|
85 |
known && rev != "" && isabelle_version == rev && |
|
86 |
(afp_rev.isEmpty || afp_rev.get != "" && afp_version == afp_rev.get) |
|
65783
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
87 |
} |
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
88 |
|
66864 | 89 |
def recent_items(db: SQL.Database, |
90 |
days: Int, rev: String, afp_rev: Option[String], sql: SQL.Source): List[Item] = |
|
65807 | 91 |
{ |
66880
486f4af28db9
more thorough treatment of afp_version and afp_pull_date;
wenzelm
parents:
66879
diff
changeset
|
92 |
val afp = afp_rev.isDefined |
65807 | 93 |
val select = |
66864 | 94 |
Build_Log.Data.select_recent_versions( |
95 |
days = days, rev = rev, afp_rev = afp_rev, sql = "WHERE " + sql) |
|
65807 | 96 |
|
97 |
db.using_statement(select)(stmt => |
|
98 |
stmt.execute_query().iterator(res => |
|
99 |
{ |
|
100 |
val known = res.bool(Build_Log.Data.known) |
|
101 |
val isabelle_version = res.string(Build_Log.Prop.isabelle_version) |
|
66880
486f4af28db9
more thorough treatment of afp_version and afp_pull_date;
wenzelm
parents:
66879
diff
changeset
|
102 |
val afp_version = if (afp) proper_string(res.string(Build_Log.Prop.afp_version)) else None |
486f4af28db9
more thorough treatment of afp_version and afp_pull_date;
wenzelm
parents:
66879
diff
changeset
|
103 |
val pull_date = res.date(Build_Log.Data.pull_date(afp)) |
66864 | 104 |
Item(known, isabelle_version, afp_version, pull_date) |
65807 | 105 |
}).toList) |
106 |
} |
|
107 |
||
108 |
def unknown_runs(items: List[Item]): List[List[Item]] = |
|
109 |
{ |
|
110 |
val (run, rest) = Library.take_prefix[Item](_.unknown, items.dropWhile(_.known)) |
|
111 |
if (run.nonEmpty) run :: unknown_runs(rest) else Nil |
|
112 |
} |
|
113 |
||
64409
70c87ca55f2c
tuned signature -- more friendly for experimentation;
wenzelm
parents:
64405
diff
changeset
|
114 |
sealed case class Remote_Build( |
65764 | 115 |
description: String, |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
116 |
host: String, |
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
117 |
user: String = "", |
65594 | 118 |
port: Int = 0, |
64294
303976a45afe
shared_home is default for classic isatest home setup;
wenzelm
parents:
64277
diff
changeset
|
119 |
shared_home: Boolean = true, |
65820 | 120 |
historic: Boolean = false, |
65789 | 121 |
history: Int = 0, |
65820 | 122 |
history_base: String = "build_history_base", |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
123 |
options: String = "", |
65732 | 124 |
args: String = "", |
66864 | 125 |
afp: Boolean = false, |
65732 | 126 |
detect: SQL.Source = "") |
127 |
{ |
|
65783
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
128 |
def sql: SQL.Source = |
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
129 |
Build_Log.Prop.build_engine + " = " + SQL.string(Build_History.engine) + " AND " + |
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
130 |
Build_Log.Prop.build_host + " = " + SQL.string(host) + |
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
131 |
(if (detect == "") "" else " AND " + SQL.enclose(detect)) |
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
132 |
|
65747 | 133 |
def profile: Build_Status.Profile = |
66880
486f4af28db9
more thorough treatment of afp_version and afp_pull_date;
wenzelm
parents:
66879
diff
changeset
|
134 |
Build_Status.Profile(description, history = history, afp = afp, sql = sql) |
65783
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
135 |
|
66864 | 136 |
def pick( |
137 |
options: Options, |
|
138 |
rev: String = "", |
|
66865 | 139 |
filter: Item => Boolean = _ => true): Option[(String, Option[String])] = |
65747 | 140 |
{ |
66864 | 141 |
val afp_rev = if (afp) Some(Mercurial.repository(afp_repos).id()) else None |
142 |
||
65783
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
143 |
val store = Build_Log.store(options) |
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
144 |
using(store.open_database())(db => |
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
145 |
{ |
66864 | 146 |
def pick_days(days: Int, gap: Int): Option[(String, Option[String])] = |
65810 | 147 |
{ |
66865 | 148 |
val items = recent_items(db, days, rev, afp_rev, sql).filter(filter) |
66007 | 149 |
def runs = unknown_runs(items).filter(run => run.length >= gap) |
65783
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
150 |
|
66864 | 151 |
if (historic || items.exists(_.known_versions(rev, afp_rev))) { |
65810 | 152 |
val longest_run = |
153 |
(List.empty[Item] /: runs)({ case (item1, item2) => |
|
154 |
if (item1.length >= item2.length) item1 else item2 |
|
155 |
}) |
|
156 |
if (longest_run.isEmpty) None |
|
66864 | 157 |
else Some(longest_run(longest_run.length / 2).versions) |
65810 | 158 |
} |
66864 | 159 |
else if (rev != "") Some((rev, afp_rev)) |
160 |
else runs.flatten.headOption.map(_.versions) |
|
65783
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
161 |
} |
65810 | 162 |
|
66007 | 163 |
pick_days(options.int("build_log_history") max history, 2) orElse |
164 |
pick_days(200, 5) orElse |
|
165 |
pick_days(2000, 1) |
|
65783
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
166 |
}) |
65747 | 167 |
} |
65732 | 168 |
} |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
169 |
|
65806 | 170 |
val remote_builds_old: List[Remote_Build] = |
171 |
List( |
|
66762 | 172 |
Remote_Build("Poly/ML 5.7 Linux", "lxbroy8", |
173 |
history_base = "37074e22e8be", |
|
174 |
options = "-m32 -B -M1x2,2 -t polyml-5.7 -i 'init_component /home/isabelle/contrib/polyml-5.7'", |
|
175 |
args = "-N -g timing", |
|
176 |
detect = Build_Log.Prop.build_tags + " = " + SQL.string("polyml-5.7") + " AND " + |
|
177 |
Build_Log.Settings.ML_OPTIONS + " <> " + SQL.string("-H 500")), |
|
178 |
Remote_Build("Poly/ML 5.7 Mac OS X", "macbroy2", |
|
179 |
history_base = "37074e22e8be", |
|
180 |
options = "-m32 -B -M1x4,4 -t polyml-5.7 -i 'init_component /home/isabelle/contrib/polyml-5.7'", |
|
181 |
args = "-a", |
|
182 |
detect = Build_Log.Prop.build_tags + " = " + SQL.string("polyml-5.7")), |
|
65806 | 183 |
Remote_Build("Poly/ML test", "lxbroy8", |
65843
d547173212d2
proper init_settings for init_component (before generated ML_OPTIONS etc.);
wenzelm
parents:
65840
diff
changeset
|
184 |
options = "-m32 -B -M1x2,2 -t polyml-test -i 'init_component /home/isabelle/contrib/polyml-5.7-20170217'", |
65806 | 185 |
args = "-N -g timing", |
186 |
detect = Build_Log.Prop.build_tags + " = " + SQL.string("polyml-test")), |
|
187 |
Remote_Build("Mac OS X 10.8 Mountain Lion", "macbroy30", options = "-m32 -M2", args = "-a", |
|
188 |
detect = Build_Log.Prop.build_start + " < date '2017-03-03'")) |
|
189 |
||
190 |
||
66898
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
191 |
val remote_builds1: List[List[Remote_Build]] = |
65732 | 192 |
{ |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
193 |
List( |
66762 | 194 |
List(Remote_Build("Poly/ML 5.7.1 Linux", "lxbroy8", |
65820 | 195 |
history_base = "37074e22e8be", |
66762 | 196 |
options = "-m32 -B -M1x2,2 -t polyml-5.7.1-pre1 -i 'init_component /home/isabelle/contrib/polyml-test-e7a662f8f9c4'", |
65732 | 197 |
args = "-N -g timing", |
66762 | 198 |
detect = Build_Log.Prop.build_tags + " = " + SQL.string("polyml-5.7.1-pre1"))), |
65764 | 199 |
List(Remote_Build("Linux A", "lxbroy9", |
65732 | 200 |
options = "-m32 -B -M1x2,2", args = "-N -g timing")), |
65820 | 201 |
List(Remote_Build("Linux B", "lxbroy10", historic = true, history = 90, |
65732 | 202 |
options = "-m32 -B -M1x4,2,4,6", args = "-N -g timing")), |
64351 | 203 |
List( |
65902 | 204 |
Remote_Build("Mac OS X 10.9 Mavericks", "macbroy2", |
65906 | 205 |
options = "-m32 -M8" + |
206 |
" -e ISABELLE_GHC=ghc -e ISABELLE_MLTON=mlton -e ISABELLE_OCAML=ocaml" + |
|
207 |
" -e ISABELLE_OCAMLC=ocamlc -e ISABELLE_SMLNJ=/mnt/nfsbroy/home/smlnj/bin/sml", |
|
65902 | 208 |
args = "-a", |
66856 | 209 |
detect = Build_Log.Prop.build_tags.undefined), |
65768 | 210 |
Remote_Build("Mac OS X 10.9 Mavericks, quick_and_dirty", "macbroy2", |
65732 | 211 |
options = "-m32 -M8 -t quick_and_dirty", args = "-a -o quick_and_dirty", |
212 |
detect = Build_Log.Prop.build_tags + " = " + SQL.string("quick_and_dirty")), |
|
65768 | 213 |
Remote_Build("Mac OS X 10.9 Mavericks, skip_proofs", "macbroy2", |
65732 | 214 |
options = "-m32 -M8 -t skip_proofs", args = "-a -o skip_proofs", |
66009 | 215 |
detect = Build_Log.Prop.build_tags + " = " + SQL.string("skip_proofs")), |
66762 | 216 |
Remote_Build("Poly/ML 5.7.1 Mac OS X", "macbroy2", |
66009 | 217 |
history_base = "37074e22e8be", |
66762 | 218 |
options = "-m32 -B -M1x4,4 -t polyml-5.7.1-pre1 -i 'init_component /home/isabelle/contrib/polyml-test-e7a662f8f9c4'", |
66009 | 219 |
args = "-a", |
66762 | 220 |
detect = Build_Log.Prop.build_tags + " = " + SQL.string("polyml-5.7.1-pre1"))), |
65768 | 221 |
List( |
222 |
Remote_Build("Mac OS X 10.12 Sierra", "macbroy30", options = "-m32 -M2", args = "-a", |
|
223 |
detect = Build_Log.Prop.build_start + " > date '2017-03-03'")), |
|
224 |
List(Remote_Build("Mac OS X 10.10 Yosemite", "macbroy31", options = "-m32 -M2", args = "-a")), |
|
64351 | 225 |
List( |
66008 | 226 |
Remote_Build("Windows", "vmnipkow9", historic = true, history = 90, shared_home = false, |
65906 | 227 |
options = "-m32 -M4" + |
228 |
" -e ISABELLE_OCAML=ocaml -e ISABELLE_OCAMLC=ocamlc" + |
|
229 |
" -e ISABELLE_GHC=/usr/local/ghc-8.0.2/bin/ghc" + |
|
230 |
" -e ISABELLE_SMLNJ=/usr/local/smlnj-110.81/bin/sml", |
|
65902 | 231 |
args = "-a", |
65732 | 232 |
detect = Build_Log.Settings.ML_PLATFORM + " = " + SQL.string("x86-windows")), |
66008 | 233 |
Remote_Build("Windows", "vmnipkow9", historic = true, history = 90, shared_home = false, |
65906 | 234 |
options = "-m64 -M4" + |
235 |
" -e ISABELLE_OCAML=ocaml -e ISABELLE_OCAMLC=ocamlc" + |
|
236 |
" -e ISABELLE_GHC=/usr/local/ghc-8.0.2/bin/ghc" + |
|
237 |
" -e ISABELLE_SMLNJ=/usr/local/smlnj-110.81/bin/sml", |
|
65902 | 238 |
args = "-a", |
66868 | 239 |
detect = Build_Log.Settings.ML_PLATFORM + " = " + SQL.string("x86_64-windows"))), |
66864 | 240 |
) ::: |
241 |
{ |
|
242 |
for { (host, n) <- List("lxbroy6" -> 1, "lxbroy7" -> 2) } |
|
243 |
yield { |
|
244 |
List(Remote_Build("AFP " + n, host = host, |
|
245 |
options = "-m32 -M1x2 -t AFP -P" + n, |
|
246 |
args = "-N -X slow", |
|
247 |
afp = true, |
|
248 |
detect = Build_Log.Prop.build_tags + " = " + SQL.string("AFP"))) |
|
249 |
} |
|
250 |
} |
|
65732 | 251 |
} |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
252 |
|
66898
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
253 |
val remote_builds2: List[List[Remote_Build]] = |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
254 |
List( |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
255 |
List( |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
256 |
Remote_Build("AFP slow", "lrzcloud1", shared_home = false, |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
257 |
options = "-m64 -M6 -U30000 -s10 -t AFP", |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
258 |
args = "-g slow", |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
259 |
afp = true, |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
260 |
detect = Build_Log.Prop.build_tags + " = " + SQL.string("AFP")))) |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
261 |
|
66864 | 262 |
private def remote_build_history( |
263 |
rev: String, afp_rev: Option[String], i: Int, r: Remote_Build): Logger_Task = |
|
64294
303976a45afe
shared_home is default for classic isatest home setup;
wenzelm
parents:
64277
diff
changeset
|
264 |
{ |
303976a45afe
shared_home is default for classic isatest home setup;
wenzelm
parents:
64277
diff
changeset
|
265 |
val task_name = "build_history-" + r.host |
303976a45afe
shared_home is default for classic isatest home setup;
wenzelm
parents:
64277
diff
changeset
|
266 |
Logger_Task(task_name, logger => |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
267 |
{ |
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
268 |
using(logger.ssh_context.open_session(host = r.host, user = r.user, port = r.port))( |
64259 | 269 |
ssh => |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
270 |
{ |
64348
4c253e84ae62
clarified push/pull chain: current ISABELLE_HOME may server as source for changes that are not published on isabelle_repos_source yet (e.g. isabelle-release branch);
wenzelm
parents:
64346
diff
changeset
|
271 |
val self_update = !r.shared_home |
4c253e84ae62
clarified push/pull chain: current ISABELLE_HOME may server as source for changes that are not published on isabelle_repos_source yet (e.g. isabelle-release branch);
wenzelm
parents:
64346
diff
changeset
|
272 |
val push_isabelle_home = self_update && Mercurial.is_repository(Path.explode("~~")) |
4c253e84ae62
clarified push/pull chain: current ISABELLE_HOME may server as source for changes that are not published on isabelle_repos_source yet (e.g. isabelle-release branch);
wenzelm
parents:
64346
diff
changeset
|
273 |
|
65929
de3adcf6a276
prefer strict result (in contrast to 0f3b0a929c02);
wenzelm
parents:
65917
diff
changeset
|
274 |
val results = |
64468
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
275 |
Build_History.remote_build_history(ssh, |
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
276 |
isabelle_repos, |
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
277 |
isabelle_repos.ext(r.host), |
65917
2b7d4678bce6
proper ISABELLE_IDENTIFIER for remote Admin/build_history, e.g. relevant for settings;
wenzelm
parents:
65915
diff
changeset
|
278 |
isabelle_identifier = "cronjob_build_history", |
64468
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
279 |
self_update = self_update, |
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
280 |
push_isabelle_home = push_isabelle_home, |
66106
b5333fc056da
always start with fresh clone (with explicitly given rev): more robust on Windows;
wenzelm
parents:
66009
diff
changeset
|
281 |
rev = rev, |
66864 | 282 |
afp_rev = afp_rev, |
64468
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
283 |
options = |
65949 | 284 |
" -N " + Bash.string(task_name) + (if (i < 0) "" else "_" + (i + 1).toString) + |
65887
cc6fdf8d1dc2
more systematic separation of build_history directories (see also 80c1c1f53e72);
wenzelm
parents:
65871
diff
changeset
|
285 |
" -f " + r.options, |
64468
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
286 |
args = "-o timeout=10800 " + r.args) |
64346 | 287 |
|
64481 | 288 |
for ((log_name, bytes) <- results) { |
289 |
logger.log(Date.now(), log_name) |
|
64468
ed8940d6295c
back to more elementary result (see 5f49765a25ec): avoid concurrent use of ssh channel;
wenzelm
parents:
64452
diff
changeset
|
290 |
Bytes.write(logger.log_dir + Path.explode(log_name), bytes) |
64481 | 291 |
} |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
292 |
}) |
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
293 |
}) |
64294
303976a45afe
shared_home is default for classic isatest home setup;
wenzelm
parents:
64277
diff
changeset
|
294 |
} |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
295 |
|
65747 | 296 |
val build_status_profiles: List[Build_Status.Profile] = |
66898
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
297 |
(remote_builds_old :: remote_builds1 ::: remote_builds2).flatten.map(_.profile) |
65747 | 298 |
|
65746 | 299 |
|
64192 | 300 |
|
301 |
/** task logging **/ |
|
64171 | 302 |
|
64193 | 303 |
sealed case class Logger_Task(name: String = "", body: Logger => Unit) |
64154 | 304 |
|
64257 | 305 |
class Log_Service private[Isabelle_Cronjob](progress: Progress, val ssh_context: SSH.Context) |
64171 | 306 |
{ |
64219 | 307 |
current_log.file.delete |
308 |
||
64171 | 309 |
private val thread: Consumer_Thread[String] = |
310 |
Consumer_Thread.fork("cronjob: logger", daemon = true)( |
|
311 |
consume = (text: String) => |
|
64219 | 312 |
{ // critical |
313 |
File.append(current_log, text + "\n") |
|
314 |
File.append(cumulative_log, text + "\n") |
|
64171 | 315 |
progress.echo(text) |
316 |
true |
|
317 |
}) |
|
318 |
||
319 |
def shutdown() { thread.shutdown() } |
|
320 |
||
321 |
val hostname = Isabelle_System.hostname() |
|
322 |
||
323 |
def log(date: Date, task_name: String, msg: String): Unit = |
|
64193 | 324 |
if (task_name != "") |
325 |
thread.send( |
|
326 |
"[" + Build_Log.print_date(date) + ", " + hostname + ", " + task_name + "]: " + msg) |
|
64171 | 327 |
|
328 |
def start_logger(start_date: Date, task_name: String): Logger = |
|
329 |
new Logger(this, start_date, task_name) |
|
64154 | 330 |
|
64171 | 331 |
def run_task(start_date: Date, task: Logger_Task) |
332 |
{ |
|
333 |
val logger = start_logger(start_date, task.name) |
|
334 |
val res = Exn.capture { task.body(logger) } |
|
335 |
val end_date = Date.now() |
|
336 |
val err = |
|
337 |
res match { |
|
338 |
case Exn.Res(_) => None |
|
64295 | 339 |
case Exn.Exn(exn) => |
64410 | 340 |
System.err.println("Exception trace for " + quote(task.name) + ":") |
64397 | 341 |
exn.printStackTrace() |
64295 | 342 |
val first_line = Library.split_lines(Exn.message(exn)).headOption getOrElse "exception" |
343 |
Some(first_line) |
|
64171 | 344 |
} |
345 |
logger.log_end(end_date, err) |
|
346 |
} |
|
347 |
||
348 |
def fork_task(start_date: Date, task: Logger_Task): Task = |
|
349 |
new Task(task.name, run_task(start_date, task)) |
|
350 |
} |
|
351 |
||
352 |
class Logger private[Isabelle_Cronjob]( |
|
353 |
val log_service: Log_Service, val start_date: Date, val task_name: String) |
|
64162 | 354 |
{ |
64257 | 355 |
def ssh_context: SSH.Context = log_service.ssh_context |
65640 | 356 |
def options: Options = ssh_context.options |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
357 |
|
64171 | 358 |
def log(date: Date, msg: String): Unit = log_service.log(date, task_name, msg) |
359 |
||
360 |
def log_end(end_date: Date, err: Option[String]) |
|
361 |
{ |
|
362 |
val elapsed_time = end_date.time - start_date.time |
|
363 |
val msg = |
|
364 |
(if (err.isEmpty) "finished" else "ERROR " + err.get) + |
|
64197 | 365 |
(if (elapsed_time.seconds < 3.0) "" else " (" + elapsed_time.message_hms + " elapsed time)") |
64171 | 366 |
log(end_date, msg) |
367 |
} |
|
368 |
||
64195 | 369 |
val log_dir: Path = main_dir + Build_Log.log_subdir(start_date) |
370 |
||
371 |
Isabelle_System.mkdirs(log_dir) |
|
64171 | 372 |
log(start_date, "started") |
373 |
} |
|
374 |
||
375 |
class Task private[Isabelle_Cronjob](name: String, body: => Unit) |
|
376 |
{ |
|
377 |
private val future: Future[Unit] = Future.thread("cronjob: " + name) { body } |
|
378 |
def is_finished: Boolean = future.is_finished |
|
64162 | 379 |
} |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
380 |
|
64170 | 381 |
|
382 |
||
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
383 |
/** cronjob **/ |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
384 |
|
64187 | 385 |
def cronjob(progress: Progress, exclude_task: Set[String]) |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
386 |
{ |
64171 | 387 |
/* soft lock */ |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
388 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
389 |
val still_running = |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
390 |
try { Some(File.read(main_state_file)) } |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
391 |
catch { case ERROR(_) => None } |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
392 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
393 |
still_running match { |
64170 | 394 |
case None | Some("") => |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
395 |
case Some(running) => |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
396 |
error("Isabelle cronjob appears to be still running: " + running) |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
397 |
} |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
398 |
|
64199 | 399 |
|
400 |
/* log service */ |
|
401 |
||
64257 | 402 |
val log_service = new Log_Service(progress, SSH.init_context(Options.init())) |
64154 | 403 |
|
64199 | 404 |
def run(start_date: Date, task: Logger_Task) { log_service.run_task(start_date, task) } |
405 |
||
406 |
def run_now(task: Logger_Task) { run(Date.now(), task) } |
|
64154 | 407 |
|
408 |
||
64199 | 409 |
/* structured tasks */ |
64184 | 410 |
|
64350
3af8566788e7
remote_builds has PAR-SEQ semantics of old isatest-makedist;
wenzelm
parents:
64348
diff
changeset
|
411 |
def SEQ(tasks: List[Logger_Task]): Logger_Task = Logger_Task(body = _ => |
64193 | 412 |
for (task <- tasks.iterator if !exclude_task(task.name) || task.name == "") |
64199 | 413 |
run_now(task)) |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
414 |
|
64350
3af8566788e7
remote_builds has PAR-SEQ semantics of old isatest-makedist;
wenzelm
parents:
64348
diff
changeset
|
415 |
def PAR(tasks: List[Logger_Task]): Logger_Task = Logger_Task(body = _ => |
64170 | 416 |
{ |
64199 | 417 |
@tailrec def join(running: List[Task]) |
418 |
{ |
|
419 |
running.partition(_.is_finished) match { |
|
420 |
case (Nil, Nil) => |
|
421 |
case (Nil, _ :: _) => Thread.sleep(500); join(running) |
|
422 |
case (_ :: _, remaining) => join(remaining) |
|
423 |
} |
|
64170 | 424 |
} |
64199 | 425 |
val start_date = Date.now() |
426 |
val running = |
|
64350
3af8566788e7
remote_builds has PAR-SEQ semantics of old isatest-makedist;
wenzelm
parents:
64348
diff
changeset
|
427 |
for (task <- tasks if !exclude_task(task.name)) |
64199 | 428 |
yield log_service.fork_task(start_date, task) |
429 |
join(running) |
|
430 |
}) |
|
64193 | 431 |
|
64170 | 432 |
|
66897 | 433 |
/* repository structure */ |
64199 | 434 |
|
65820 | 435 |
val hg = Mercurial.repository(isabelle_repos) |
66895 | 436 |
val hg_graph = hg.graph() |
65820 | 437 |
val rev = hg.id() |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
438 |
|
66895 | 439 |
def history_base_filter(r: Remote_Build): Item => Boolean = |
440 |
{ |
|
441 |
val base_rev = hg.id(r.history_base) |
|
442 |
val nodes = hg_graph.all_succs(List(base_rev)).toSet |
|
443 |
(item: Item) => nodes(item.isabelle_version) |
|
444 |
} |
|
445 |
||
66897 | 446 |
|
447 |
/* main */ |
|
448 |
||
449 |
val main_start_date = Date.now() |
|
450 |
File.write(main_state_file, main_start_date + " " + log_service.hostname) |
|
451 |
||
64193 | 452 |
run(main_start_date, |
65783
d3d5cb2d6866
pick isabelle_version based on build_log database;
wenzelm
parents:
65771
diff
changeset
|
453 |
Logger_Task("isabelle_cronjob", logger => |
64231
dbc8294c75d3
added remote_build_history tasks: parallel on several remote hosts;
wenzelm
parents:
64220
diff
changeset
|
454 |
run_now( |
64405
81bac77929d9
just one task to identify Isabelle + AFP repository snapshots and build release;
wenzelm
parents:
64399
diff
changeset
|
455 |
SEQ(List(build_release, build_history_base, |
66898
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
456 |
PAR(List(remote_builds1, remote_builds2).map(remote_builds => |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
457 |
SEQ(List( |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
458 |
PAR(remote_builds.map(seq => |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
459 |
SEQ( |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
460 |
for { |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
461 |
(r, i) <- (if (seq.length <= 1) seq.map((_, -1)) else seq.zipWithIndex) |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
462 |
(isabelle_rev, afp_rev) <- r.pick(logger.options, rev, history_base_filter(r)) |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
463 |
} yield remote_build_history(isabelle_rev, afp_rev, i, r)))), |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
464 |
Logger_Task("jenkins_logs", _ => Jenkins.download_logs(jenkins_jobs, main_dir)), |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
465 |
Logger_Task("build_log_database", |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
466 |
logger => Isabelle_Devel.build_log_database(logger.options, build_log_dirs)), |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
467 |
Logger_Task("build_status", |
8b7c4679c05b
parallel remote_builds2 for the sake of "AFP slow" (with theoretical data race on build_log_dirs);
wenzelm
parents:
66897
diff
changeset
|
468 |
logger => Isabelle_Devel.build_status(logger.options))))))))))) |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
469 |
|
64171 | 470 |
log_service.shutdown() |
64170 | 471 |
|
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
472 |
main_state_file.file.delete |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
473 |
} |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
474 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
475 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
476 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
477 |
/** command line entry point **/ |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
478 |
|
64148 | 479 |
def main(args: Array[String]) |
480 |
{ |
|
481 |
Command_Line.tool0 { |
|
482 |
var force = false |
|
483 |
var verbose = false |
|
64187 | 484 |
var exclude_task = Set.empty[String] |
64148 | 485 |
|
486 |
val getopts = Getopts(""" |
|
487 |
Usage: Admin/cronjob/main [OPTIONS] |
|
488 |
||
489 |
Options are: |
|
490 |
-f apply force to do anything |
|
491 |
-v verbose |
|
64187 | 492 |
-x NAME exclude tasks with this name |
64148 | 493 |
""", |
494 |
"f" -> (_ => force = true), |
|
64187 | 495 |
"v" -> (_ => verbose = true), |
496 |
"x:" -> (arg => exclude_task += arg)) |
|
64148 | 497 |
|
498 |
val more_args = getopts(args) |
|
499 |
if (more_args.nonEmpty) getopts.usage() |
|
500 |
||
64909 | 501 |
val progress = if (verbose) new Console_Progress() else No_Progress |
64148 | 502 |
|
64187 | 503 |
if (force) cronjob(progress, exclude_task) |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
504 |
else error("Need to apply force to do anything") |
64148 | 505 |
} |
506 |
} |
|
507 |
} |