author | Fabian Huch <huch@in.tum.de> |
Thu, 06 Jun 2024 14:51:28 +0200 | |
changeset 80261 | e3f472221f8f |
parent 80260 | ed9b1598d293 |
child 80281 | 17d2f775907a |
permissions | -rw-r--r-- |
77570 | 1 |
/* Title: Pure/Admin/ci_build.scala |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
2 |
Author: Lars Hupel and Fabian Huch, TU Munich |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
3 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
4 |
Build profile for continuous integration services. |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
5 |
*/ |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
6 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
7 |
package isabelle |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
8 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
9 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
10 |
import java.time.ZoneId |
63472 | 11 |
import java.time.format.DateTimeFormatter |
68013 | 12 |
import java.util.{Properties => JProperties, Map => JMap} |
75631
809c37bfd823
clarified IO, following Java 11 and Isabelle/Scala;
wenzelm
parents:
75630
diff
changeset
|
13 |
import java.nio.file.Files |
63328
7a8515c58271
read Java system properties from ISABELLE_CI_PROPERTIES
Lars Hupel <lars.hupel@mytum.de>
parents:
63294
diff
changeset
|
14 |
|
7a8515c58271
read Java system properties from ISABELLE_CI_PROPERTIES
Lars Hupel <lars.hupel@mytum.de>
parents:
63294
diff
changeset
|
15 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
16 |
object CI_Build { |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
17 |
/* build result */ |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
18 |
|
73934
39e0c7fac69e
jenkins: pre/post-hook results
Fabian Huch <huch@in.tum.de>
parents:
73359
diff
changeset
|
19 |
case class Result(rc: Int) |
75393 | 20 |
case object Result { |
74306 | 21 |
def ok: Result = Result(Process_Result.RC.ok) |
22 |
def error: Result = Result(Process_Result.RC.error) |
|
73934
39e0c7fac69e
jenkins: pre/post-hook results
Fabian Huch <huch@in.tum.de>
parents:
73359
diff
changeset
|
23 |
} |
39e0c7fac69e
jenkins: pre/post-hook results
Fabian Huch <huch@in.tum.de>
parents:
73359
diff
changeset
|
24 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
25 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
26 |
/* executor profile */ |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
27 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
28 |
case class Profile(threads: Int, jobs: Int, numa: Boolean) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
29 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
30 |
object Profile { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
31 |
def from_host: Profile = { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
32 |
Isabelle_System.hostname() match { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
33 |
case "hpcisabelle" => Profile(8, 8, numa = true) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
34 |
case "lxcisa1" => Profile(4, 10, numa = false) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
35 |
case _ => Profile(2, 2, numa = false) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
36 |
} |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
37 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
38 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
39 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
40 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
41 |
/* build config */ |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
42 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
43 |
case class Build_Config( |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
44 |
documents: Boolean = true, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
45 |
clean: Boolean = true, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
46 |
include: List[Path] = Nil, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
47 |
select: List[Path] = Nil, |
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
48 |
pre_hook: Options => Result = _ => Result.ok, |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
49 |
post_hook: (Build.Results, Options, Time) => Result = (_, _, _) => Result.ok, |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
50 |
selection: Sessions.Selection = Sessions.Selection.empty) |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
51 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
52 |
|
78830
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
53 |
def mail_server(options: Options): Mail.Server = { |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
54 |
val sender = |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
55 |
proper_string(options.string("ci_mail_sender")).map(Mail.address) getOrElse |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
56 |
Mail.default_address |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
57 |
|
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
58 |
new Mail.Server(sender, |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
59 |
smtp_host = options.string("ci_mail_smtp_host"), |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
60 |
smtp_port = options.int("ci_mail_smtp_port"), |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
61 |
user = options.string("ci_mail_user"), |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
62 |
password = options.string("ci_mail_password")) |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
63 |
} |
348a5606baf3
use mail module in CI build;
Fabian Huch <huch@in.tum.de>
parents:
78821
diff
changeset
|
64 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
65 |
/* ci build jobs */ |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
66 |
|
80261
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
67 |
sealed trait Trigger |
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
68 |
case object On_Commit extends Trigger |
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
69 |
|
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
70 |
object Timed { |
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
71 |
def nightly(start_time: Time = Time.zero): Timed = |
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
72 |
Timed { (before, now) => |
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
73 |
val start0 = before.midnight + start_time |
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
74 |
val start1 = now.midnight + start_time |
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
75 |
(before.time < start0.time && start0.time <= now.time) || |
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
76 |
(before.time < start1.time && start1.time <= now.time) |
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
77 |
} |
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
78 |
} |
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
79 |
|
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
80 |
case class Timed(in_interval: (Date, Date) => Boolean) extends Trigger |
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
81 |
|
80260
ed9b1598d293
manage components of ci builds;
Fabian Huch <huch@in.tum.de>
parents:
80245
diff
changeset
|
82 |
sealed case class Job( |
ed9b1598d293
manage components of ci builds;
Fabian Huch <huch@in.tum.de>
parents:
80245
diff
changeset
|
83 |
name: String, |
ed9b1598d293
manage components of ci builds;
Fabian Huch <huch@in.tum.de>
parents:
80245
diff
changeset
|
84 |
description: String, |
ed9b1598d293
manage components of ci builds;
Fabian Huch <huch@in.tum.de>
parents:
80245
diff
changeset
|
85 |
profile: Profile, |
ed9b1598d293
manage components of ci builds;
Fabian Huch <huch@in.tum.de>
parents:
80245
diff
changeset
|
86 |
config: Build_Config, |
80261
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
87 |
components: List[String] = Nil, |
e3f472221f8f
add triggers to ci jobs: on commit vs timed;
Fabian Huch <huch@in.tum.de>
parents:
80260
diff
changeset
|
88 |
trigger: Trigger = On_Commit |
80260
ed9b1598d293
manage components of ci builds;
Fabian Huch <huch@in.tum.de>
parents:
80245
diff
changeset
|
89 |
) { |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
90 |
override def toString: String = name |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
91 |
} |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
92 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
93 |
private lazy val known_jobs: List[Job] = |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
94 |
Isabelle_System.make_services(classOf[Isabelle_CI_Builds]).flatMap(_.jobs) |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
95 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
96 |
def show_jobs: String = |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
97 |
cat_lines(known_jobs.sortBy(_.name).map(job => job.name + " - " + job.description)) |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
98 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
99 |
def the_job(name: String): Job = known_jobs.find(job => job.name == name) getOrElse |
76225 | 100 |
error("Unknown job " + quote(name)) |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
101 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
102 |
val timing = |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
103 |
Job( |
76225 | 104 |
"benchmark", "runs benchmark and timing sessions", |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
105 |
Profile(threads = 6, jobs = 1, numa = false), |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
106 |
Build_Config( |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
107 |
documents = false, |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
108 |
select = List( |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
109 |
Path.explode("$ISABELLE_HOME/src/Benchmarks")), |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
110 |
selection = Sessions.Selection(session_groups = List("timing")))) |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
111 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
112 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
113 |
/* session status */ |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
114 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
115 |
sealed abstract class Status(val str: String) { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
116 |
def merge(that: Status): Status = (this, that) match { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
117 |
case (Ok, s) => s |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
118 |
case (Failed, _) => Failed |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
119 |
case (Skipped, Failed) => Failed |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
120 |
case (Skipped, _) => Skipped |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
121 |
} |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
122 |
} |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
123 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
124 |
object Status { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
125 |
def merge(statuses: List[Status]): Status = |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
126 |
statuses.foldLeft(Ok: Status)(_ merge _) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
127 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
128 |
def from_results(results: Build.Results, session: String): Status = |
75629 | 129 |
if (results.cancelled(session)) Skipped |
130 |
else if (results(session).ok) Ok |
|
131 |
else Failed |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
132 |
} |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
133 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
134 |
case object Ok extends Status("ok") |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
135 |
case object Skipped extends Status("skipped") |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
136 |
case object Failed extends Status("failed") |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
137 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
138 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
139 |
/* ci build */ |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
140 |
|
75393 | 141 |
private def compute_timing(results: Build.Results, group: Option[String]): Timing = { |
75629 | 142 |
val timings = |
143 |
results.sessions.collect { |
|
144 |
case session if group.forall(results.info(session).groups.contains(_)) => |
|
145 |
results(session).timing |
|
146 |
} |
|
73359 | 147 |
timings.foldLeft(Timing.zero)(_ + _) |
63385 | 148 |
} |
149 |
||
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
150 |
private def with_documents(options: Options, config: Build_Config): Options = { |
75629 | 151 |
if (config.documents) { |
79660 | 152 |
options + "browser_info" + "document=pdf" + "document_variants=document:outline=/proof,/ML" |
75629 | 153 |
} |
154 |
else options |
|
63894
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
155 |
} |
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
156 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
157 |
def hg_id(path: Path): String = |
80245
b6551e0c70c4
support ci job via hg_sync (cf. 7883f221d6d3);
Fabian Huch <huch@in.tum.de>
parents:
80094
diff
changeset
|
158 |
if (Mercurial.Hg_Sync.ok(path)) File.read(path + Mercurial.Hg_Sync.PATH_ID) |
b6551e0c70c4
support ci job via hg_sync (cf. 7883f221d6d3);
Fabian Huch <huch@in.tum.de>
parents:
80094
diff
changeset
|
159 |
else Mercurial.repository(path).id() |
63385 | 160 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
161 |
def print_section(title: String): Unit = |
80094 | 162 |
println("\n=== " + title + " ===\n") |
63349 | 163 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
164 |
def ci_build(options: Options, job: Job): Unit = { |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
165 |
val profile = job.profile |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
166 |
val config = job.config |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
167 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
168 |
val isabelle_home = Path.explode(Isabelle_System.getenv_strict("ISABELLE_HOME")) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
169 |
val isabelle_id = hg_id(isabelle_home) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
170 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
171 |
val start_time = Time.now() |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
172 |
val formatted_time = start_time.instant.atZone(ZoneId.systemDefault).format( |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
173 |
DateTimeFormatter.RFC_1123_DATE_TIME) |
63385 | 174 |
|
63349 | 175 |
print_section("CONFIGURATION") |
64081 | 176 |
println(Build_Log.Settings.show()) |
63328
7a8515c58271
read Java system properties from ISABELLE_CI_PROPERTIES
Lars Hupel <lars.hupel@mytum.de>
parents:
63294
diff
changeset
|
177 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
178 |
val build_options = |
79660 | 179 |
with_documents(options, config).int.update("threads", profile.threads) + |
180 |
"parallel_proofs=1" + "system_heaps" |
|
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
181 |
|
80094 | 182 |
println( |
183 |
"jobs = " + profile.jobs + ", threads = " + profile.threads + ", numa = " + profile.numa) |
|
69120
9d3b41732fe0
Jenkins: detect machine; adjust job parameters accordingly
Lars Hupel <lars.hupel@mytum.de>
parents:
68498
diff
changeset
|
184 |
|
63349 | 185 |
print_section("BUILD") |
80094 | 186 |
println("Build started at " + formatted_time) |
187 |
println("Isabelle id " + isabelle_id) |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
188 |
val pre_result = config.pre_hook(options) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
189 |
|
63472 | 190 |
print_section("LOG") |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
191 |
val (results, elapsed_time) = { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
192 |
val progress = new Console_Progress(verbose = true) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
193 |
val start_time = Time.now() |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
194 |
val results = progress.interrupt_handler { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
195 |
Build.build( |
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
196 |
build_options, |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
197 |
selection = config.selection, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
198 |
progress = progress, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
199 |
clean_build = config.clean, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
200 |
numa_shuffling = profile.numa, |
79616 | 201 |
max_jobs = Some(profile.jobs), |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
202 |
dirs = config.include, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
203 |
select_dirs = config.select) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
204 |
} |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
205 |
val end_time = Time.now() |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
206 |
(results, end_time - start_time) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
207 |
} |
63349 | 208 |
|
209 |
print_section("TIMING") |
|
63385 | 210 |
|
211 |
val groups = results.sessions.map(results.info).flatMap(_.groups) |
|
212 |
for (group <- groups) |
|
80094 | 213 |
println("Group " + group + ": " + compute_timing(results, Some(group)).message_resources) |
63387
3395fe5e3893
more accurate total timing
Lars Hupel <lars.hupel@mytum.de>
parents:
63385
diff
changeset
|
214 |
|
3395fe5e3893
more accurate total timing
Lars Hupel <lars.hupel@mytum.de>
parents:
63385
diff
changeset
|
215 |
val total_timing = compute_timing(results, None).copy(elapsed = elapsed_time) |
3395fe5e3893
more accurate total timing
Lars Hupel <lars.hupel@mytum.de>
parents:
63385
diff
changeset
|
216 |
println("Overall: " + total_timing.message_resources) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
217 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
218 |
if (!results.ok) { |
63349 | 219 |
print_section("FAILED SESSIONS") |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
220 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
221 |
for (name <- results.sessions) { |
80094 | 222 |
if (results.cancelled(name)) println("Session " + name + ": CANCELLED") |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
223 |
else { |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
224 |
val result = results(name) |
80094 | 225 |
if (!result.ok) println("Session " + name + ": FAILED " + result.rc) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
226 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
227 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
228 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
229 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
230 |
val post_result = config.post_hook(results, options, start_time) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
231 |
|
75630 | 232 |
sys.exit(List(pre_result.rc, results.rc, post_result.rc).max) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
233 |
} |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
234 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
235 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
236 |
/* Isabelle tool wrapper */ |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
237 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
238 |
val isabelle_tool = Isabelle_Tool("ci_build", "builds Isabelle jobs in ci environments", |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
239 |
Scala_Project.here, |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
240 |
{ args => |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
241 |
/* arguments */ |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
242 |
|
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
243 |
var options = Options.init() |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
244 |
|
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
245 |
val getopts = Getopts(""" |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
246 |
Usage: isabelle ci_build [OPTIONS] JOB |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
247 |
|
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
248 |
Options are: |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
249 |
-o OPTION override Isabelle system OPTION (via NAME=VAL or NAME) |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
250 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
251 |
Runs Isabelle builds in ci environment, with the following build jobs: |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
252 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
253 |
""" + Library.indent_lines(4, show_jobs) + "\n", |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
254 |
"o:" -> (arg => options = options + arg)) |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
255 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
256 |
val more_args = getopts(args) |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
257 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
258 |
val job = more_args match { |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
259 |
case job :: Nil => the_job(job) |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
260 |
case _ => getopts.usage() |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
261 |
} |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
262 |
|
78821
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
263 |
ci_build(options, job) |
4c5aadf1cb48
prefer Isabelle options for CI mail settings over ci.properties;
Fabian Huch <huch@in.tum.de>
parents:
77570
diff
changeset
|
264 |
}) |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
265 |
} |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
266 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
267 |
class Isabelle_CI_Builds(val jobs: CI_Build.Job*) extends Isabelle_System.Service |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
268 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
269 |
class CI_Builds extends Isabelle_CI_Builds(CI_Build.timing) |