author | Fabian Huch <huch@in.tum.de> |
Fri, 30 Sep 2022 09:27:25 +0200 | |
changeset 76225 | fb2be77a7819 |
parent 76222 | 3c4e373922ca |
child 77510 | f5d6cd98b16a |
permissions | -rw-r--r-- |
64160 | 1 |
/* Title: Pure/Admin/ci_profile.scala |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
2 |
Author: Lars Hupel and Fabian Huch, TU Munich |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
3 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
4 |
Build profile for continuous integration services. |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
5 |
*/ |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
6 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
7 |
package isabelle |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
8 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
9 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
10 |
import java.time.ZoneId |
63472 | 11 |
import java.time.format.DateTimeFormatter |
68013 | 12 |
import java.util.{Properties => JProperties, Map => JMap} |
75631
809c37bfd823
clarified IO, following Java 11 and Isabelle/Scala;
wenzelm
parents:
75630
diff
changeset
|
13 |
import java.nio.file.Files |
63328
7a8515c58271
read Java system properties from ISABELLE_CI_PROPERTIES
Lars Hupel <lars.hupel@mytum.de>
parents:
63294
diff
changeset
|
14 |
|
7a8515c58271
read Java system properties from ISABELLE_CI_PROPERTIES
Lars Hupel <lars.hupel@mytum.de>
parents:
63294
diff
changeset
|
15 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
16 |
object CI_Build { |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
17 |
/* build result */ |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
18 |
|
73934
39e0c7fac69e
jenkins: pre/post-hook results
Fabian Huch <huch@in.tum.de>
parents:
73359
diff
changeset
|
19 |
case class Result(rc: Int) |
75393 | 20 |
case object Result { |
74306 | 21 |
def ok: Result = Result(Process_Result.RC.ok) |
22 |
def error: Result = Result(Process_Result.RC.error) |
|
73934
39e0c7fac69e
jenkins: pre/post-hook results
Fabian Huch <huch@in.tum.de>
parents:
73359
diff
changeset
|
23 |
} |
39e0c7fac69e
jenkins: pre/post-hook results
Fabian Huch <huch@in.tum.de>
parents:
73359
diff
changeset
|
24 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
25 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
26 |
/* executor profile */ |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
27 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
28 |
case class Profile(threads: Int, jobs: Int, numa: Boolean) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
29 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
30 |
object Profile { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
31 |
def from_host: Profile = { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
32 |
Isabelle_System.hostname() match { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
33 |
case "hpcisabelle" => Profile(8, 8, numa = true) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
34 |
case "lxcisa1" => Profile(4, 10, numa = false) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
35 |
case _ => Profile(2, 2, numa = false) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
36 |
} |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
37 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
38 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
39 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
40 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
41 |
/* build config */ |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
42 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
43 |
case class Build_Config( |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
44 |
documents: Boolean = true, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
45 |
clean: Boolean = true, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
46 |
include: List[Path] = Nil, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
47 |
select: List[Path] = Nil, |
75629 | 48 |
pre_hook: () => Result = () => Result.ok, |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
49 |
post_hook: (Build.Results, Time) => Result = (_, _) => Result.ok, |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
50 |
selection: Sessions.Selection = Sessions.Selection.empty |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
51 |
) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
52 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
53 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
54 |
/* ci build jobs */ |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
55 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
56 |
sealed case class Job(name: String, description: String, profile: Profile, config: Build_Config) { |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
57 |
override def toString: String = name |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
58 |
} |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
59 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
60 |
private lazy val known_jobs: List[Job] = |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
61 |
Isabelle_System.make_services(classOf[Isabelle_CI_Builds]).flatMap(_.jobs) |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
62 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
63 |
def show_jobs: String = |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
64 |
cat_lines(known_jobs.sortBy(_.name).map(job => job.name + " - " + job.description)) |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
65 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
66 |
def the_job(name: String): Job = known_jobs.find(job => job.name == name) getOrElse |
76225 | 67 |
error("Unknown job " + quote(name)) |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
68 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
69 |
val timing = |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
70 |
Job( |
76225 | 71 |
"benchmark", "runs benchmark and timing sessions", |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
72 |
Profile(threads = 6, jobs = 1, numa = false), |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
73 |
Build_Config( |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
74 |
documents = false, |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
75 |
select = List( |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
76 |
Path.explode("$ISABELLE_HOME/src/Benchmarks")), |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
77 |
selection = Sessions.Selection(session_groups = List("timing")))) |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
78 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
79 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
80 |
/* session status */ |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
81 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
82 |
sealed abstract class Status(val str: String) { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
83 |
def merge(that: Status): Status = (this, that) match { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
84 |
case (Ok, s) => s |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
85 |
case (Failed, _) => Failed |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
86 |
case (Skipped, Failed) => Failed |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
87 |
case (Skipped, _) => Skipped |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
88 |
} |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
89 |
} |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
90 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
91 |
object Status { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
92 |
def merge(statuses: List[Status]): Status = |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
93 |
statuses.foldLeft(Ok: Status)(_ merge _) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
94 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
95 |
def from_results(results: Build.Results, session: String): Status = |
75629 | 96 |
if (results.cancelled(session)) Skipped |
97 |
else if (results(session).ok) Ok |
|
98 |
else Failed |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
99 |
} |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
100 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
101 |
case object Ok extends Status("ok") |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
102 |
case object Skipped extends Status("skipped") |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
103 |
case object Failed extends Status("failed") |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
104 |
|
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
105 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
106 |
/* ci build */ |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
107 |
|
75393 | 108 |
private def load_properties(): JProperties = { |
75631
809c37bfd823
clarified IO, following Java 11 and Isabelle/Scala;
wenzelm
parents:
75630
diff
changeset
|
109 |
val props = new JProperties |
63418 | 110 |
val file_name = Isabelle_System.getenv("ISABELLE_CI_PROPERTIES") |
75631
809c37bfd823
clarified IO, following Java 11 and Isabelle/Scala;
wenzelm
parents:
75630
diff
changeset
|
111 |
if (file_name.nonEmpty) { |
809c37bfd823
clarified IO, following Java 11 and Isabelle/Scala;
wenzelm
parents:
75630
diff
changeset
|
112 |
val path = Path.explode(file_name) |
809c37bfd823
clarified IO, following Java 11 and Isabelle/Scala;
wenzelm
parents:
75630
diff
changeset
|
113 |
if (path.is_file) props.load(Files.newBufferedReader(path.java_path)) |
63418 | 114 |
} |
75631
809c37bfd823
clarified IO, following Java 11 and Isabelle/Scala;
wenzelm
parents:
75630
diff
changeset
|
115 |
props |
63328
7a8515c58271
read Java system properties from ISABELLE_CI_PROPERTIES
Lars Hupel <lars.hupel@mytum.de>
parents:
63294
diff
changeset
|
116 |
} |
7a8515c58271
read Java system properties from ISABELLE_CI_PROPERTIES
Lars Hupel <lars.hupel@mytum.de>
parents:
63294
diff
changeset
|
117 |
|
75393 | 118 |
private def compute_timing(results: Build.Results, group: Option[String]): Timing = { |
75629 | 119 |
val timings = |
120 |
results.sessions.collect { |
|
121 |
case session if group.forall(results.info(session).groups.contains(_)) => |
|
122 |
results(session).timing |
|
123 |
} |
|
73359 | 124 |
timings.foldLeft(Timing.zero)(_ + _) |
63385 | 125 |
} |
126 |
||
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
127 |
private def with_documents(options: Options, config: Build_Config): Options = { |
75629 | 128 |
if (config.documents) { |
63894
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
129 |
options |
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
130 |
.bool.update("browser_info", true) |
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
131 |
.string.update("document", "pdf") |
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
132 |
.string.update("document_variants", "document:outline=/proof,/ML") |
75629 | 133 |
} |
134 |
else options |
|
63894
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
135 |
} |
7534eec7cfad
benchmark doesn't need to build documents
Lars Hupel <lars.hupel@mytum.de>
parents:
63685
diff
changeset
|
136 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
137 |
def hg_id(path: Path): String = |
64232
367d83d6030e
clarified hg.id operation, with explicit tip as default;
wenzelm
parents:
64217
diff
changeset
|
138 |
Mercurial.repository(path).id() |
63385 | 139 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
140 |
def print_section(title: String): Unit = |
63349 | 141 |
println(s"\n=== $title ===\n") |
142 |
||
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
143 |
def ci_build(job: Job): Unit = { |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
144 |
val profile = job.profile |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
145 |
val config = job.config |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
146 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
147 |
val isabelle_home = Path.explode(Isabelle_System.getenv_strict("ISABELLE_HOME")) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
148 |
val isabelle_id = hg_id(isabelle_home) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
149 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
150 |
val start_time = Time.now() |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
151 |
val formatted_time = start_time.instant.atZone(ZoneId.systemDefault).format( |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
152 |
DateTimeFormatter.RFC_1123_DATE_TIME) |
63385 | 153 |
|
63349 | 154 |
print_section("CONFIGURATION") |
64081 | 155 |
println(Build_Log.Settings.show()) |
63328
7a8515c58271
read Java system properties from ISABELLE_CI_PROPERTIES
Lars Hupel <lars.hupel@mytum.de>
parents:
63294
diff
changeset
|
156 |
val props = load_properties() |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
157 |
System.getProperties.asInstanceOf[JMap[AnyRef, AnyRef]].putAll(props) |
63328
7a8515c58271
read Java system properties from ISABELLE_CI_PROPERTIES
Lars Hupel <lars.hupel@mytum.de>
parents:
63294
diff
changeset
|
158 |
|
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
159 |
val options = |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
160 |
with_documents(Options.init(), config) |
65056 | 161 |
.int.update("parallel_proofs", 1) |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
162 |
.int.update("threads", profile.threads) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
163 |
|
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
164 |
println(s"jobs = ${profile.jobs}, threads = ${profile.threads}, numa = ${profile.numa}") |
69120
9d3b41732fe0
Jenkins: detect machine; adjust job parameters accordingly
Lars Hupel <lars.hupel@mytum.de>
parents:
68498
diff
changeset
|
165 |
|
63349 | 166 |
print_section("BUILD") |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
167 |
println(s"Build started at $formatted_time") |
63472 | 168 |
println(s"Isabelle id $isabelle_id") |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
169 |
val pre_result = config.pre_hook() |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
170 |
|
63472 | 171 |
print_section("LOG") |
75628
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
172 |
val (results, elapsed_time) = { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
173 |
val progress = new Console_Progress(verbose = true) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
174 |
val start_time = Time.now() |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
175 |
val results = progress.interrupt_handler { |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
176 |
Build.build( |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
177 |
options + "system_heaps", |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
178 |
selection = config.selection, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
179 |
progress = progress, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
180 |
clean_build = config.clean, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
181 |
verbose = true, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
182 |
numa_shuffling = profile.numa, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
183 |
max_jobs = profile.jobs, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
184 |
dirs = config.include, |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
185 |
select_dirs = config.select) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
186 |
} |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
187 |
val end_time = Time.now() |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
188 |
(results, end_time - start_time) |
6a5e4f17f285
switched to statically compiled ci profile;
Fabian Huch <huch@in.tum.de>
parents:
75393
diff
changeset
|
189 |
} |
63349 | 190 |
|
191 |
print_section("TIMING") |
|
63385 | 192 |
|
193 |
val groups = results.sessions.map(results.info).flatMap(_.groups) |
|
194 |
for (group <- groups) |
|
195 |
println(s"Group $group: " + compute_timing(results, Some(group)).message_resources) |
|
63387
3395fe5e3893
more accurate total timing
Lars Hupel <lars.hupel@mytum.de>
parents:
63385
diff
changeset
|
196 |
|
3395fe5e3893
more accurate total timing
Lars Hupel <lars.hupel@mytum.de>
parents:
63385
diff
changeset
|
197 |
val total_timing = compute_timing(results, None).copy(elapsed = elapsed_time) |
3395fe5e3893
more accurate total timing
Lars Hupel <lars.hupel@mytum.de>
parents:
63385
diff
changeset
|
198 |
println("Overall: " + total_timing.message_resources) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
199 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
200 |
if (!results.ok) { |
63349 | 201 |
print_section("FAILED SESSIONS") |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
202 |
|
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
203 |
for (name <- results.sessions) { |
75629 | 204 |
if (results.cancelled(name)) println(s"Session $name: CANCELLED") |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
205 |
else { |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
206 |
val result = results(name) |
75629 | 207 |
if (!result.ok) println(s"Session $name: FAILED ${ result.rc }") |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
208 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
209 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
210 |
} |
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
211 |
|
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
212 |
val post_result = config.post_hook(results, start_time) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
213 |
|
75630 | 214 |
sys.exit(List(pre_result.rc, results.rc, post_result.rc).max) |
63288
e0513d6e4916
start moving actual Jenkins build scripts into the repository
Lars Hupel <lars.hupel@mytum.de>
parents:
diff
changeset
|
215 |
} |
76222
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
216 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
217 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
218 |
/* Isabelle tool wrapper */ |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
219 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
220 |
val isabelle_tool = |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
221 |
Isabelle_Tool( |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
222 |
"ci_build", "builds Isabelle jobs in ci environments", Scala_Project.here, |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
223 |
{ args => |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
224 |
val getopts = Getopts(""" |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
225 |
Usage: isabelle ci_build [JOB] |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
226 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
227 |
Runs Isabelle builds in ci environment, with the following build jobs: |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
228 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
229 |
""" + Library.indent_lines(4, show_jobs) + "\n") |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
230 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
231 |
val more_args = getopts(args) |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
232 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
233 |
val job = more_args match { |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
234 |
case job :: Nil => the_job(job) |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
235 |
case _ => getopts.usage() |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
236 |
} |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
237 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
238 |
ci_build(job) |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
239 |
}) |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
240 |
} |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
241 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
242 |
class Isabelle_CI_Builds(val jobs: CI_Build.Job*) extends Isabelle_System.Service |
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
243 |
|
3c4e373922ca
restructured ci profile into modular ci build system;
Fabian Huch <huch@in.tum.de>
parents:
75633
diff
changeset
|
244 |
class CI_Builds extends Isabelle_CI_Builds(CI_Build.timing) |