author | wenzelm |
Thu, 15 Feb 2024 11:33:36 +0100 | |
changeset 79616 | 12bb31d01510 |
parent 78830 | 348a5606baf3 |
child 79660 | 49475f8bb4cc |
permissions | -rw-r--r-- |
77570 | 1 |
/* Title: Pure/Admin/ci_build.scala |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
2 |
Author: Lars Hupel and Fabian Huch, TU Munich |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
3 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
4 |
Build profile for continuous integration services. |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
5 |
*/ |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
6 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
7 |
package isabelle |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
8 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
9 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
10 |
import java.time.ZoneId |
63472 | 11 |
import java.time.format.DateTimeFormatter |
68013 | 12 |
import java.util.{Properties => JProperties, Map => JMap} |
75631
809c37bfd823
clarified IO, following Java 11 and Isabelle/Scala;
wenzelm
parents:
75630
diff
changeset
|
13 |
import java.nio.file.Files |
63328
7a8515c58271
read Java system properties from ISABELLE_CI_PROPERTIES
Lars Hupel <lars.hupel@mytum.de>
parents:
63294
diff
changeset
|
14 |
|
7a8515c58271
read Java system properties from ISABELLE_CI_PROPERTIES
Lars Hupel <lars.hupel@mytum.de>
parents:
63294
diff
changeset
|
15 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
16 |
object CI_Build { |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
17 |
/* build result */ |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
18 |
|
73934
39e0c7fac69e
jenkins: pre/post-hook results
Fabian Huch <huch@in.tum.de>
parents:
73359
diff
changeset
|
19 |
case class Result(rc: Int) |
75393 | 20 |
case object Result { |
74306 | 21 |
def ok: Result = Result(Process_Result.RC.ok) |
22 |
def error: Result = Result(Process_Result.RC.error) |
|
73934
39e0c7fac69e
jenkins: pre/post-hook results
Fabian Huch <huch@in.tum.de>
parents:
73359
diff
changeset
|
23 |
} |
39e0c7fac69e
jenkins: pre/post-hook results
Fabian Huch <huch@in.tum.de>
parents:
73359
diff
changeset
|
24 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
25 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
26 |
/* executor profile */ |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
27 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
28 |
case class Profile(threads: Int, jobs: Int, numa: Boolean) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
29 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
30 |
object Profile { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
31 |
def from_host: Profile = { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
32 |
Isabelle_System.hostname() match { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
33 |
case "hpcisabelle" => Profile(8, 8, numa = true) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
34 |
case "lxcisa1" => Profile(4, 10, numa = false) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
35 |
case _ => Profile(2, 2, numa = false) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
36 |
} |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
37 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
38 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
39 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
40 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
41 |
/* build config */ |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
42 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
43 |
case class Build_Config( |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
44 |
documents: Boolean = true, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
45 |
clean: Boolean = true, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
46 |
include: List[Path] = Nil, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
47 |
select: List[Path] = Nil, |
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
48 |
pre_hook: Options => Result = _ => Result.ok, |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
49 |
post_hook: (Build.Results, Options, Time) => Result = (_, _, _) => Result.ok, |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
50 |
selection: Sessions.Selection = Sessions.Selection.empty) |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
51 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
52 |
|
78830
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
53 |
def mail_server(options: Options): Mail.Server = { |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
54 |
val sender = |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
55 |
proper_string(options.string("ci_mail_sender")).map(Mail.address) getOrElse |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
56 |
Mail.default_address |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
57 |
|
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
58 |
new Mail.Server(sender, |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
59 |
smtp_host = options.string("ci_mail_smtp_host"), |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
60 |
smtp_port = options.int("ci_mail_smtp_port"), |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
61 |
user = options.string("ci_mail_user"), |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
62 |
password = options.string("ci_mail_password")) |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
63 |
} |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
64 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
65 |
/* ci build jobs */ |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
66 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
67 |
sealed case class Job(name: String, description: String, profile: Profile, config: Build_Config) { |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
68 |
override def toString: String = name |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
69 |
} |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
70 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
71 |
private lazy val known_jobs: List[Job] = |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
72 |
Isabelle_System.make_services(classOf[Isabelle_CI_Builds]).flatMap(_.jobs) |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
73 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
74 |
def show_jobs: String = |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
75 |
cat_lines(known_jobs.sortBy(_.name).map(job => job.name + " - " + job.description)) |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
76 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
77 |
def the_job(name: String): Job = known_jobs.find(job => job.name == name) getOrElse |
76225 | 78 |
error("Unknown job " + quote(name)) |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
79 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
80 |
val timing = |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
81 |
Job( |
76225 | 82 |
"benchmark", "runs benchmark and timing sessions", |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
83 |
Profile(threads = 6, jobs = 1, numa = false), |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
84 |
Build_Config( |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
85 |
documents = false, |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
86 |
select = List( |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
87 |
Path.explode("$ISABELLE_HOME/src/Benchmarks")), |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
88 |
selection = Sessions.Selection(session_groups = List("timing")))) |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
89 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
90 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
91 |
/* session status */ |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
92 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
93 |
sealed abstract class Status(val str: String) { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
94 |
def merge(that: Status): Status = (this, that) match { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
95 |
case (Ok, s) => s |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
96 |
case (Failed, _) => Failed |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
97 |
case (Skipped, Failed) => Failed |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
98 |
case (Skipped, _) => Skipped |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
99 |
} |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
100 |
} |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
101 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
102 |
object Status { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
103 |
def merge(statuses: List[Status]): Status = |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
104 |
statuses.foldLeft(Ok: Status)(_ merge _) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
105 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
106 |
def from_results(results: Build.Results, session: String): Status = |
75629 | 107 |
if (results.cancelled(session)) Skipped |
108 |
else if (results(session).ok) Ok |
|
109 |
else Failed |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
110 |
} |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
111 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
112 |
case object Ok extends Status("ok") |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
113 |
case object Skipped extends Status("skipped") |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
114 |
case object Failed extends Status("failed") |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
115 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
116 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
117 |
/* ci build */ |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
118 |
|
75393 | 119 |
private def compute_timing(results: Build.Results, group: Option[String]): Timing = { |
75629 | 120 |
val timings = |
121 |
results.sessions.collect { |
|
122 |
case session if group.forall(results.info(session).groups.contains(_)) => |
|
123 |
results(session).timing |
|
124 |
} |
|
73359 | 125 |
timings.foldLeft(Timing.zero)(_ + _) |
63385 | 126 |
} |
127 |
||
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
128 |
private def with_documents(options: Options, config: Build_Config): Options = { |
75629 | 129 |
if (config.documents) { |
63894
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
130 |
options |
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
131 |
.bool.update("browser_info", true) |
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
132 |
.string.update("document", "pdf") |
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
133 |
.string.update("document_variants", "document:outline=/proof,/ML") |
75629 | 134 |
} |
135 |
else options |
|
63894
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
136 |
} |
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
137 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
138 |
def hg_id(path: Path): String = |
64232
367d83d6030e
clarified hg.id operation, with explicit tip as default;
wenzelm
parents:
64217
diff
changeset
|
139 |
Mercurial.repository(path).id() |
63385 | 140 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
141 |
def print_section(title: String): Unit = |
63349 | 142 |
println(s"\n=== $title ===\n") |
143 |
||
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
144 |
def ci_build(options: Options, job: Job): Unit = { |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
145 |
val profile = job.profile |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
146 |
val config = job.config |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
147 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
148 |
val isabelle_home = Path.explode(Isabelle_System.getenv_strict("ISABELLE_HOME")) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
149 |
val isabelle_id = hg_id(isabelle_home) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
150 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
151 |
val start_time = Time.now() |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
152 |
val formatted_time = start_time.instant.atZone(ZoneId.systemDefault).format( |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
153 |
DateTimeFormatter.RFC_1123_DATE_TIME) |
63385 | 154 |
|
63349 | 155 |
print_section("CONFIGURATION") |
64081 | 156 |
println(Build_Log.Settings.show()) |
63328
7a8515c58271
read Java system properties from ISABELLE_CI_PROPERTIES
Lars Hupel <lars.hupel@mytum.de>
parents:
63294
diff
changeset
|
157 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
158 |
val build_options = |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
159 |
with_documents(options, config) |
65056 | 160 |
.int.update("parallel_proofs", 1) |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
161 |
.int.update("threads", profile.threads) |
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
162 |
+ "system_heaps" |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
163 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
164 |
println(s"jobs = ${profile.jobs}, threads = ${profile.threads}, numa = ${profile.numa}") |
69120
9d3b41732fe0
Jenkins: detect machine; adjust job parameters accordingly
Lars Hupel <lars.hupel@mytum.de>
parents:
68498
diff
changeset
|
165 |
|
63349 | 166 |
print_section("BUILD") |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
167 |
println(s"Build started at $formatted_time") |
63472 | 168 |
println(s"Isabelle id $isabelle_id") |
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
169 |
val pre_result = config.pre_hook(options) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
170 |
|
63472 | 171 |
print_section("LOG") |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
172 |
val (results, elapsed_time) = { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
173 |
val progress = new Console_Progress(verbose = true) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
174 |
val start_time = Time.now() |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
175 |
val results = progress.interrupt_handler { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
176 |
Build.build( |
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
177 |
build_options, |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
178 |
selection = config.selection, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
179 |
progress = progress, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
180 |
clean_build = config.clean, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
181 |
numa_shuffling = profile.numa, |
79616 | 182 |
max_jobs = Some(profile.jobs), |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
183 |
dirs = config.include, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
184 |
select_dirs = config.select) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
185 |
} |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
186 |
val end_time = Time.now() |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
187 |
(results, end_time - start_time) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
188 |
} |
63349 | 189 |
|
190 |
print_section("TIMING") |
|
63385 | 191 |
|
192 |
val groups = results.sessions.map(results.info).flatMap(_.groups) |
|
193 |
for (group <- groups) |
|
194 |
println(s"Group $group: " + compute_timing(results, Some(group)).message_resources) |
|
63387
3395fe5e3893
more accurate total timing
Lars Hupel <lars.hupel@mytum.de>
parents:
63385
diff
changeset
|
195 |
|
3395fe5e3893
more accurate total timing
Lars Hupel <lars.hupel@mytum.de>
parents:
63385
diff
changeset
|
196 |
val total_timing = compute_timing(results, None).copy(elapsed = elapsed_time) |
3395fe5e3893
more accurate total timing
Lars Hupel <lars.hupel@mytum.de>
parents:
63385
diff
changeset
|
197 |
println("Overall: " + total_timing.message_resources) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
198 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
199 |
if (!results.ok) { |
63349 | 200 |
print_section("FAILED SESSIONS") |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
201 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
202 |
for (name <- results.sessions) { |
75629 | 203 |
if (results.cancelled(name)) println(s"Session $name: CANCELLED") |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
204 |
else { |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
205 |
val result = results(name) |
75629 | 206 |
if (!result.ok) println(s"Session $name: FAILED ${ result.rc }") |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
207 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
208 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
209 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
210 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
211 |
val post_result = config.post_hook(results, options, start_time) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
212 |
|
75630 | 213 |
sys.exit(List(pre_result.rc, results.rc, post_result.rc).max) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
214 |
} |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
215 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
216 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
217 |
/* Isabelle tool wrapper */ |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
218 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
219 |
val isabelle_tool = Isabelle_Tool("ci_build", "builds Isabelle jobs in ci environments", |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
220 |
Scala_Project.here, |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
221 |
{ args => |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
222 |
/* arguments */ |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
223 |
|
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
224 |
var options = Options.init() |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
225 |
|
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
226 |
val getopts = Getopts(""" |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
227 |
Usage: isabelle ci_build [OPTIONS] JOB |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
228 |
|
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
229 |
Options are: |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
230 |
-o OPTION override Isabelle system OPTION (via NAME=VAL or NAME) |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
231 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
232 |
Runs Isabelle builds in ci environment, with the following build jobs: |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
233 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
234 |
""" + Library.indent_lines(4, show_jobs) + "\n", |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
235 |
"o:" -> (arg => options = options + arg)) |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
236 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
237 |
val more_args = getopts(args) |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
238 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
239 |
val job = more_args match { |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
240 |
case job :: Nil => the_job(job) |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
241 |
case _ => getopts.usage() |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
242 |
} |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
243 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
244 |
ci_build(options, job) |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
245 |
}) |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
246 |
} |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
247 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
248 |
class Isabelle_CI_Builds(val jobs: CI_Build.Job*) extends Isabelle_System.Service |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
249 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
250 |
class CI_Builds extends Isabelle_CI_Builds(CI_Build.timing) |