author | wenzelm |
Sat, 01 Oct 2016 20:58:59 +0200 | |
changeset 63984 | 6ba87450894d |
parent 63926 | 70973a1b4ec0 |
child 64054 | 1fc9ab31720d |
permissions | -rw-r--r-- |
63686 | 1 |
/* Title: Pure/Tools/build_stats.scala |
2 |
Author: Makarius |
|
3 |
||
4 |
Statistics from session build output. |
|
5 |
*/ |
|
6 |
||
7 |
package isabelle |
|
8 |
||
9 |
||
10 |
import scala.collection.mutable |
|
11 |
import scala.util.matching.Regex |
|
12 |
||
13 |
||
14 |
object Build_Stats |
|
15 |
{ |
|
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
16 |
/* parse build output */ |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
17 |
|
63702
fed1d4dab990
cpu time is optional (see Timing.message_resources);
wenzelm
parents:
63701
diff
changeset
|
18 |
private val Session_Finished1 = |
63686 | 19 |
new Regex("""^Finished (\S+) \((\d+):(\d+):(\d+) elapsed time, (\d+):(\d+):(\d+) cpu time.*$""") |
63702
fed1d4dab990
cpu time is optional (see Timing.message_resources);
wenzelm
parents:
63701
diff
changeset
|
20 |
private val Session_Finished2 = |
fed1d4dab990
cpu time is optional (see Timing.message_resources);
wenzelm
parents:
63701
diff
changeset
|
21 |
new Regex("""^Finished (\S+) \((\d+):(\d+):(\d+) elapsed time.*$""") |
63686 | 22 |
private val Session_Timing = |
23 |
new Regex("""^Timing (\S+) \((\d) threads, (\d+\.\d+)s elapsed time, (\d+\.\d+)s cpu time, (\d+\.\d+)s GC time.*$""") |
|
24 |
||
25 |
private object ML_Option |
|
26 |
{ |
|
27 |
def unapply(s: String): Option[(String, String)] = |
|
28 |
s.indexOf('=') match { |
|
29 |
case -1 => None |
|
30 |
case i => |
|
31 |
val a = s.substring(0, i) |
|
32 |
Library.try_unquote(s.substring(i + 1)) match { |
|
33 |
case Some(b) if Build.ml_options.contains(a) => Some((a, b)) |
|
34 |
case _ => None |
|
35 |
} |
|
36 |
} |
|
37 |
} |
|
38 |
||
39 |
def parse(text: String): Build_Stats = |
|
40 |
{ |
|
41 |
val ml_options = new mutable.ListBuffer[(String, String)] |
|
42 |
var finished = Map.empty[String, Timing] |
|
43 |
var timing = Map.empty[String, Timing] |
|
44 |
var threads = Map.empty[String, Int] |
|
45 |
||
46 |
for (line <- split_lines(text)) { |
|
47 |
line match { |
|
63702
fed1d4dab990
cpu time is optional (see Timing.message_resources);
wenzelm
parents:
63701
diff
changeset
|
48 |
case Session_Finished1(name, |
63686 | 49 |
Value.Int(e1), Value.Int(e2), Value.Int(e3), |
50 |
Value.Int(c1), Value.Int(c2), Value.Int(c3)) => |
|
63700 | 51 |
val elapsed = Time.hms(e1, e2, e3) |
52 |
val cpu = Time.hms(c1, c2, c3) |
|
63686 | 53 |
finished += (name -> Timing(elapsed, cpu, Time.zero)) |
63702
fed1d4dab990
cpu time is optional (see Timing.message_resources);
wenzelm
parents:
63701
diff
changeset
|
54 |
case Session_Finished2(name, |
fed1d4dab990
cpu time is optional (see Timing.message_resources);
wenzelm
parents:
63701
diff
changeset
|
55 |
Value.Int(e1), Value.Int(e2), Value.Int(e3)) => |
fed1d4dab990
cpu time is optional (see Timing.message_resources);
wenzelm
parents:
63701
diff
changeset
|
56 |
val elapsed = Time.hms(e1, e2, e3) |
fed1d4dab990
cpu time is optional (see Timing.message_resources);
wenzelm
parents:
63701
diff
changeset
|
57 |
finished += (name -> Timing(elapsed, Time.zero, Time.zero)) |
63686 | 58 |
case Session_Timing(name, |
59 |
Value.Int(t), Value.Double(e), Value.Double(c), Value.Double(g)) => |
|
60 |
val elapsed = Time.seconds(e) |
|
61 |
val cpu = Time.seconds(c) |
|
62 |
val gc = Time.seconds(g) |
|
63 |
timing += (name -> Timing(elapsed, cpu, gc)) |
|
64 |
threads += (name -> t) |
|
63926 | 65 |
case ML_Option(a, b) => ml_options += (a -> b) |
63686 | 66 |
case _ => |
67 |
} |
|
68 |
} |
|
69 |
||
70 |
Build_Stats(ml_options.toList, finished, timing, threads) |
|
71 |
} |
|
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
72 |
|
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
73 |
|
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
74 |
/* presentation */ |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
75 |
|
63700 | 76 |
private val default_history_length = 100 |
77 |
private val default_size = (800, 600) |
|
78 |
private val default_only_sessions = Set.empty[String] |
|
79 |
private val default_elapsed_threshold = Time.zero |
|
63706 | 80 |
private val default_ml_timing: Option[Boolean] = None |
63700 | 81 |
|
63703 | 82 |
def present_job(job: String, dir: Path, |
63700 | 83 |
history_length: Int = default_history_length, |
84 |
size: (Int, Int) = default_size, |
|
85 |
only_sessions: Set[String] = default_only_sessions, |
|
63706 | 86 |
elapsed_threshold: Time = default_elapsed_threshold, |
87 |
ml_timing: Option[Boolean] = default_ml_timing): List[String] = |
|
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
88 |
{ |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
89 |
val build_infos = CI_API.build_job_builds(job).sortBy(_.timestamp).reverse.take(history_length) |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
90 |
if (build_infos.isEmpty) error("No build infos for job " + quote(job)) |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
91 |
|
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
92 |
val all_build_stats = |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
93 |
Par_List.map((info: CI_API.Build_Info) => |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
94 |
(info.timestamp / 1000, parse(Url.read(info.output))), build_infos) |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
95 |
val all_sessions = |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
96 |
(Set.empty[String] /: all_build_stats)( |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
97 |
{ case (s, (_, stats)) => s ++ stats.sessions }) |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
98 |
|
63700 | 99 |
def check_threshold(stats: Build_Stats, session: String): Boolean = |
100 |
stats.finished.get(session) match { |
|
101 |
case Some(t) => t.elapsed >= elapsed_threshold |
|
102 |
case None => false |
|
103 |
} |
|
104 |
||
63703 | 105 |
val sessions = |
106 |
for { |
|
107 |
session <- (if (only_sessions.isEmpty) all_sessions else all_sessions & only_sessions) |
|
108 |
if all_build_stats.filter({ case (_, stats) => check_threshold(stats, session) }).length >= 3 |
|
109 |
} yield session |
|
110 |
||
111 |
Isabelle_System.mkdirs(dir) |
|
112 |
for (session <- sessions) { |
|
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
113 |
Isabelle_System.with_tmp_file(session, "png") { data_file => |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
114 |
Isabelle_System.with_tmp_file(session, "gnuplot") { plot_file => |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
115 |
val data = |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
116 |
for { (t, stats) <- all_build_stats if stats.finished.isDefinedAt(session) } |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
117 |
yield { |
63708 | 118 |
val finished = stats.finished.getOrElse(session, Timing.zero) |
119 |
val timing = stats.timing.getOrElse(session, Timing.zero) |
|
63706 | 120 |
List(t.toString, finished.elapsed.minutes, finished.cpu.minutes, |
121 |
timing.elapsed.minutes, timing.cpu.minutes, timing.gc.minutes).mkString(" ") |
|
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
122 |
} |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
123 |
File.write(data_file, cat_lines(data)) |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
124 |
|
63706 | 125 |
val plots1 = |
126 |
List( |
|
63707
b7aab1a6cf0d
clarified presentation order, according to typical amounts;
wenzelm
parents:
63706
diff
changeset
|
127 |
""" using 1:3 smooth sbezier title "cpu time (smooth)" """, |
b7aab1a6cf0d
clarified presentation order, according to typical amounts;
wenzelm
parents:
63706
diff
changeset
|
128 |
""" using 1:3 smooth csplines title "cpu time" """, |
63706 | 129 |
""" using 1:2 smooth sbezier title "elapsed time (smooth)" """, |
63707
b7aab1a6cf0d
clarified presentation order, according to typical amounts;
wenzelm
parents:
63706
diff
changeset
|
130 |
""" using 1:2 smooth csplines title "elapsed time" """) |
63706 | 131 |
val plots2 = |
132 |
List( |
|
63707
b7aab1a6cf0d
clarified presentation order, according to typical amounts;
wenzelm
parents:
63706
diff
changeset
|
133 |
""" using 1:5 smooth sbezier title "ML cpu time (smooth)" """, |
b7aab1a6cf0d
clarified presentation order, according to typical amounts;
wenzelm
parents:
63706
diff
changeset
|
134 |
""" using 1:5 smooth csplines title "ML cpu time" """, |
63706 | 135 |
""" using 1:4 smooth sbezier title "ML elapsed time (smooth)" """, |
136 |
""" using 1:4 smooth csplines title "ML elapsed time" """, |
|
137 |
""" using 1:6 smooth sbezier title "ML gc time (smooth)" """, |
|
138 |
""" using 1:6 smooth csplines title "ML gc time" """) |
|
139 |
val plots = |
|
140 |
ml_timing match { |
|
141 |
case None => plots1 |
|
142 |
case Some(false) => plots1 ::: plots2 |
|
143 |
case Some(true) => plots2 |
|
144 |
} |
|
145 |
||
146 |
val data_file_name = File.standard_path(data_file.getAbsolutePath) |
|
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
147 |
File.write(plot_file, """ |
63700 | 148 |
set terminal png size """ + size._1 + "," + size._2 + """ |
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
149 |
set output """ + quote(File.standard_path(dir + Path.basic(session + ".png"))) + """ |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
150 |
set xdata time |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
151 |
set timefmt "%s" |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
152 |
set format x "%d-%b" |
63701 | 153 |
set xlabel """ + quote(session) + """ noenhanced |
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
154 |
set key left top |
63706 | 155 |
plot [] [0:] """ + plots.map(s => quote(data_file_name) + " " + s).mkString(", ") + "\n") |
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
156 |
val result = Isabelle_System.bash("\"$ISABELLE_GNUPLOT\" " + File.bash_path(plot_file)) |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
157 |
if (result.rc != 0) { |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
158 |
Output.error_message("Session " + session + ": gnuplot error") |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
159 |
result.print |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
160 |
} |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
161 |
} |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
162 |
} |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
163 |
} |
63703 | 164 |
|
165 |
sessions.toList.sorted |
|
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
166 |
} |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
167 |
|
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
168 |
|
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
169 |
/* Isabelle tool wrapper */ |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
170 |
|
63703 | 171 |
private val html_header = """<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN"> |
172 |
<html> |
|
173 |
<head><title>Performance statistics from session build output</title></head> |
|
174 |
<body> |
|
175 |
""" |
|
176 |
private val html_footer = """ |
|
177 |
</body> |
|
178 |
</html> |
|
179 |
""" |
|
180 |
||
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
181 |
val isabelle_tool = |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
182 |
Isabelle_Tool("build_stats", "present statistics from session build output", args => |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
183 |
{ |
63703 | 184 |
var target_dir = Path.explode("stats") |
63706 | 185 |
var ml_timing = default_ml_timing |
63700 | 186 |
var only_sessions = default_only_sessions |
187 |
var elapsed_threshold = default_elapsed_threshold |
|
188 |
var history_length = default_history_length |
|
189 |
var size = default_size |
|
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
190 |
|
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
191 |
val getopts = Getopts(""" |
63700 | 192 |
Usage: isabelle build_stats [OPTIONS] [JOBS ...] |
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
193 |
|
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
194 |
Options are: |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
195 |
-D DIR target directory (default "stats") |
63706 | 196 |
-M only ML timing |
63700 | 197 |
-S SESSIONS only given SESSIONS (comma separated) |
198 |
-T THRESHOLD only sessions with elapsed time >= THRESHOLD (minutes) |
|
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
199 |
-l LENGTH length of history (default 100) |
63706 | 200 |
-m include ML timing |
63700 | 201 |
-s WxH size of PNG image (default 800x600) |
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
202 |
|
63700 | 203 |
Present statistics from session build output of the given JOBS, from Jenkins |
204 |
continuous build service specified as URL via ISABELLE_JENKINS_ROOT. |
|
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
205 |
""", |
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
206 |
"D:" -> (arg => target_dir = Path.explode(arg)), |
63706 | 207 |
"M" -> (_ => ml_timing = Some(true)), |
63700 | 208 |
"S:" -> (arg => only_sessions = space_explode(',', arg).toSet), |
63805 | 209 |
"T:" -> (arg => elapsed_threshold = Time.minutes(Value.Double.parse(arg))), |
210 |
"l:" -> (arg => history_length = Value.Int.parse(arg)), |
|
63706 | 211 |
"m" -> (_ => ml_timing = Some(false)), |
63700 | 212 |
"s:" -> (arg => |
63805 | 213 |
space_explode('x', arg).map(Value.Int.parse(_)) match { |
63700 | 214 |
case List(w, h) if w > 0 && h > 0 => size = (w, h) |
215 |
case _ => error("Error bad PNG image size: " + quote(arg)) |
|
216 |
})) |
|
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
217 |
|
63700 | 218 |
val jobs = getopts(args) |
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
219 |
val all_jobs = CI_API.build_jobs() |
63700 | 220 |
val bad_jobs = jobs.filterNot(all_jobs.contains(_)).sorted |
221 |
||
222 |
if (jobs.isEmpty) |
|
63984 | 223 |
error("No build jobs given. Available jobs: " + all_jobs.sorted.mkString(" ")) |
63700 | 224 |
|
225 |
if (bad_jobs.nonEmpty) |
|
63984 | 226 |
error("Unknown build jobs: " + bad_jobs.mkString(" ") + |
227 |
"\nAvailable jobs: " + all_jobs.sorted.mkString(" ")) |
|
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
228 |
|
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
229 |
for (job <- jobs) { |
63703 | 230 |
val dir = target_dir + Path.basic(job) |
231 |
Output.writeln(dir.implode) |
|
63706 | 232 |
val sessions = |
233 |
present_job(job, dir, history_length, size, only_sessions, elapsed_threshold, ml_timing) |
|
63703 | 234 |
File.write(dir + Path.basic("index.html"), |
235 |
html_header + "\n<h1>" + HTML.output(job) + "</h1>\n" + |
|
236 |
cat_lines( |
|
237 |
sessions.map(session => |
|
238 |
"""<br/><img src=""" + quote(HTML.output(session + ".png")) + """><br/>""")) + |
|
239 |
"\n" + html_footer) |
|
63688
cc57255bf6ae
gnuplot presentation similar to former isatest-statistics;
wenzelm
parents:
63686
diff
changeset
|
240 |
} |
63703 | 241 |
|
242 |
File.write(target_dir + Path.basic("index.html"), |
|
243 |
html_header + "\n<ul>\n" + |
|
244 |
cat_lines( |
|
245 |
jobs.map(job => """<li> <a href=""" + quote(HTML.output(job + "/index.html")) + """>""" + |
|
246 |
HTML.output(job) + """</a> </li>""")) + |
|
247 |
"\n</ul>\n" + html_footer) |
|
248 |
}) |
|
63686 | 249 |
} |
250 |
||
251 |
sealed case class Build_Stats( |
|
252 |
ml_options: List[(String, String)], |
|
253 |
finished: Map[String, Timing], |
|
254 |
timing: Map[String, Timing], |
|
255 |
threads: Map[String, Int]) |
|
256 |
{ |
|
257 |
val sessions: Set[String] = finished.keySet ++ timing.keySet |
|
258 |
||
259 |
override def toString: String = |
|
260 |
sessions.toList.sorted.mkString("Build_Stats(", ", ", ")") |
|
261 |
} |