src/Pure/Tools/build_stats.scala
changeset 64054 1fc9ab31720d
parent 63984 6ba87450894d
child 64062 a7352cbde7d7
equal deleted inserted replaced
64053:7ece2e14fd6c 64054:1fc9ab31720d
     5 */
     5 */
     6 
     6 
     7 package isabelle
     7 package isabelle
     8 
     8 
     9 
     9 
    10 import scala.collection.mutable
       
    11 import scala.util.matching.Regex
       
    12 
       
    13 
       
    14 object Build_Stats
    10 object Build_Stats
    15 {
    11 {
    16   /* parse build output */
       
    17 
       
    18   private val Session_Finished1 =
       
    19     new Regex("""^Finished (\S+) \((\d+):(\d+):(\d+) elapsed time, (\d+):(\d+):(\d+) cpu time.*$""")
       
    20   private val Session_Finished2 =
       
    21     new Regex("""^Finished (\S+) \((\d+):(\d+):(\d+) elapsed time.*$""")
       
    22   private val Session_Timing =
       
    23     new Regex("""^Timing (\S+) \((\d) threads, (\d+\.\d+)s elapsed time, (\d+\.\d+)s cpu time, (\d+\.\d+)s GC time.*$""")
       
    24 
       
    25   private object ML_Option
       
    26   {
       
    27     def unapply(s: String): Option[(String, String)] =
       
    28       s.indexOf('=') match {
       
    29         case -1 => None
       
    30         case i =>
       
    31           val a = s.substring(0, i)
       
    32           Library.try_unquote(s.substring(i + 1)) match {
       
    33             case Some(b) if Build.ml_options.contains(a) => Some((a, b))
       
    34             case _ => None
       
    35           }
       
    36       }
       
    37   }
       
    38 
       
    39   def parse(text: String): Build_Stats =
       
    40   {
       
    41     val ml_options = new mutable.ListBuffer[(String, String)]
       
    42     var finished = Map.empty[String, Timing]
       
    43     var timing = Map.empty[String, Timing]
       
    44     var threads = Map.empty[String, Int]
       
    45 
       
    46     for (line <- split_lines(text)) {
       
    47       line match {
       
    48         case Session_Finished1(name,
       
    49             Value.Int(e1), Value.Int(e2), Value.Int(e3),
       
    50             Value.Int(c1), Value.Int(c2), Value.Int(c3)) =>
       
    51           val elapsed = Time.hms(e1, e2, e3)
       
    52           val cpu = Time.hms(c1, c2, c3)
       
    53           finished += (name -> Timing(elapsed, cpu, Time.zero))
       
    54         case Session_Finished2(name,
       
    55             Value.Int(e1), Value.Int(e2), Value.Int(e3)) =>
       
    56           val elapsed = Time.hms(e1, e2, e3)
       
    57           finished += (name -> Timing(elapsed, Time.zero, Time.zero))
       
    58         case Session_Timing(name,
       
    59             Value.Int(t), Value.Double(e), Value.Double(c), Value.Double(g)) =>
       
    60           val elapsed = Time.seconds(e)
       
    61           val cpu = Time.seconds(c)
       
    62           val gc = Time.seconds(g)
       
    63           timing += (name -> Timing(elapsed, cpu, gc))
       
    64           threads += (name -> t)
       
    65         case ML_Option(a, b) => ml_options += (a -> b)
       
    66         case _ =>
       
    67       }
       
    68     }
       
    69 
       
    70     Build_Stats(ml_options.toList, finished, timing, threads)
       
    71   }
       
    72 
       
    73 
       
    74   /* presentation */
    12   /* presentation */
    75 
    13 
    76   private val default_history_length = 100
    14   private val default_history_length = 100
    77   private val default_size = (800, 600)
    15   private val default_size = (800, 600)
    78   private val default_only_sessions = Set.empty[String]
    16   private val default_only_sessions = Set.empty[String]
    84     size: (Int, Int) = default_size,
    22     size: (Int, Int) = default_size,
    85     only_sessions: Set[String] = default_only_sessions,
    23     only_sessions: Set[String] = default_only_sessions,
    86     elapsed_threshold: Time = default_elapsed_threshold,
    24     elapsed_threshold: Time = default_elapsed_threshold,
    87     ml_timing: Option[Boolean] = default_ml_timing): List[String] =
    25     ml_timing: Option[Boolean] = default_ml_timing): List[String] =
    88   {
    26   {
    89     val build_infos = CI_API.build_job_builds(job).sortBy(_.timestamp).reverse.take(history_length)
    27     val job_infos = CI_API.build_job_builds(job).sortBy(_.timestamp).reverse.take(history_length)
    90     if (build_infos.isEmpty) error("No build infos for job " + quote(job))
    28     if (job_infos.isEmpty) error("No build infos for job " + quote(job))
    91 
    29 
    92     val all_build_stats =
    30     val all_infos =
    93       Par_List.map((info: CI_API.Build_Info) =>
    31       Par_List.map((job_info: CI_API.Job_Info) =>
    94         (info.timestamp / 1000, parse(Url.read(info.output))), build_infos)
    32         (job_info.timestamp / 1000, Build_Log.parse_info(Url.read(job_info.output))), job_infos)
    95     val all_sessions =
    33     val all_sessions =
    96       (Set.empty[String] /: all_build_stats)(
    34       (Set.empty[String] /: all_infos)(
    97         { case (s, (_, stats)) => s ++ stats.sessions })
    35         { case (s, (_, info)) => s ++ info.sessions })
    98 
    36 
    99     def check_threshold(stats: Build_Stats, session: String): Boolean =
    37     def check_threshold(info: Build_Log.Info, session: String): Boolean =
   100       stats.finished.get(session) match {
    38       info.finished.get(session) match {
   101         case Some(t) => t.elapsed >= elapsed_threshold
    39         case Some(t) => t.elapsed >= elapsed_threshold
   102         case None => false
    40         case None => false
   103       }
    41       }
   104 
    42 
   105     val sessions =
    43     val sessions =
   106       for {
    44       for {
   107         session <- (if (only_sessions.isEmpty) all_sessions else all_sessions & only_sessions)
    45         session <- (if (only_sessions.isEmpty) all_sessions else all_sessions & only_sessions)
   108         if all_build_stats.filter({ case (_, stats) => check_threshold(stats, session) }).length >= 3
    46         if all_infos.filter({ case (_, info) => check_threshold(info, session) }).length >= 3
   109       } yield session
    47       } yield session
   110 
    48 
   111     Isabelle_System.mkdirs(dir)
    49     Isabelle_System.mkdirs(dir)
   112     for (session <- sessions) {
    50     for (session <- sessions) {
   113       Isabelle_System.with_tmp_file(session, "png") { data_file =>
    51       Isabelle_System.with_tmp_file(session, "png") { data_file =>
   114         Isabelle_System.with_tmp_file(session, "gnuplot") { plot_file =>
    52         Isabelle_System.with_tmp_file(session, "gnuplot") { plot_file =>
   115           val data =
    53           val data =
   116             for { (t, stats) <- all_build_stats if stats.finished.isDefinedAt(session) }
    54             for { (t, info) <- all_infos if info.finished.isDefinedAt(session) }
   117             yield {
    55             yield {
   118               val finished = stats.finished.getOrElse(session, Timing.zero)
    56               val finished = info.finished.getOrElse(session, Timing.zero)
   119               val timing = stats.timing.getOrElse(session, Timing.zero)
    57               val timing = info.timing.getOrElse(session, Timing.zero)
   120               List(t.toString, finished.elapsed.minutes, finished.cpu.minutes,
    58               List(t.toString, finished.elapsed.minutes, finished.cpu.minutes,
   121                 timing.elapsed.minutes, timing.cpu.minutes, timing.gc.minutes).mkString(" ")
    59                 timing.elapsed.minutes, timing.cpu.minutes, timing.gc.minutes).mkString(" ")
   122             }
    60             }
   123           File.write(data_file, cat_lines(data))
    61           File.write(data_file, cat_lines(data))
   124 
    62 
   245           jobs.map(job => """<li> <a href=""" + quote(HTML.output(job + "/index.html")) + """>""" +
   183           jobs.map(job => """<li> <a href=""" + quote(HTML.output(job + "/index.html")) + """>""" +
   246             HTML.output(job) + """</a> </li>""")) +
   184             HTML.output(job) + """</a> </li>""")) +
   247         "\n</ul>\n" + html_footer)
   185         "\n</ul>\n" + html_footer)
   248   })
   186   })
   249 }
   187 }
   250 
       
   251 sealed case class Build_Stats(
       
   252   ml_options: List[(String, String)],
       
   253   finished: Map[String, Timing],
       
   254   timing: Map[String, Timing],
       
   255   threads: Map[String, Int])
       
   256 {
       
   257   val sessions: Set[String] = finished.keySet ++ timing.keySet
       
   258 
       
   259   override def toString: String =
       
   260     sessions.toList.sorted.mkString("Build_Stats(", ", ", ")")
       
   261 }