src/Pure/Admin/build_log.scala
author wenzelm
Tue, 02 May 2017 21:38:23 +0200
changeset 65688 2181b5615c64
parent 65685 47bbf7150aae
child 65689 c1eab527bfa7
permissions -rw-r--r--
tuned;
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
64160
1eea419fab65 clarified files;
wenzelm
parents: 64155
diff changeset
     1
/*  Title:      Pure/Admin/build_log.scala
64045
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
     2
    Author:     Makarius
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
     3
65608
d526ba7b0a2d tuned comment;
wenzelm
parents: 65607
diff changeset
     4
Management of build log files and database storage.
64045
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
     5
*/
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
     6
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
     7
package isabelle
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
     8
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
     9
64100
9b1573213ebe tuned error;
wenzelm
parents: 64099
diff changeset
    10
import java.io.{File => JFile}
9b1573213ebe tuned error;
wenzelm
parents: 64099
diff changeset
    11
import java.time.ZoneId
64110
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
    12
import java.time.format.{DateTimeFormatter, DateTimeParseException}
64096
5edeb60a7ec5 more flexible date formats;
wenzelm
parents: 64095
diff changeset
    13
import java.util.Locale
65599
08dfa79866ec database storage of Meta_Info and Build_Info;
wenzelm
parents: 65595
diff changeset
    14
import java.sql.PreparedStatement
64061
1bbea2b55d22 some support for header and data fields, notably from afp-test;
wenzelm
parents: 64054
diff changeset
    15
65600
138ffa41dc54 clarified filter_files: sorted and unique;
wenzelm
parents: 65599
diff changeset
    16
import scala.collection.immutable.SortedMap
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
    17
import scala.collection.mutable
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
    18
import scala.util.matching.Regex
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
    19
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
    20
64045
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
    21
object Build_Log
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
    22
{
64298
wenzelm
parents: 64297
diff changeset
    23
  /** content **/
64101
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
    24
64298
wenzelm
parents: 64297
diff changeset
    25
  /* properties */
64150
b10f2ddd7679 clarified modules;
wenzelm
parents: 64120
diff changeset
    26
64298
wenzelm
parents: 64297
diff changeset
    27
  object Prop
wenzelm
parents: 64297
diff changeset
    28
  {
65624
32fa61f694ef clarified multi-line properties;
wenzelm
parents: 65623
diff changeset
    29
    val build_tags = SQL.Column.string("build_tags")  // lines
32fa61f694ef clarified multi-line properties;
wenzelm
parents: 65623
diff changeset
    30
    val build_args = SQL.Column.string("build_args")  // lines
65591
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
    31
    val build_group_id = SQL.Column.string("build_group_id")
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
    32
    val build_id = SQL.Column.string("build_id")
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
    33
    val build_engine = SQL.Column.string("build_engine")
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
    34
    val build_host = SQL.Column.string("build_host")
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
    35
    val build_start = SQL.Column.date("build_start")
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
    36
    val build_end = SQL.Column.date("build_end")
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
    37
    val isabelle_version = SQL.Column.string("isabelle_version")
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
    38
    val afp_version = SQL.Column.string("afp_version")
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
    39
65611
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
    40
    val all_props: List[SQL.Column] =
65591
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
    41
      List(build_tags, build_args, build_group_id, build_id, build_engine,
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
    42
        build_host, build_start, build_end, isabelle_version, afp_version)
64298
wenzelm
parents: 64297
diff changeset
    43
  }
64150
b10f2ddd7679 clarified modules;
wenzelm
parents: 64120
diff changeset
    44
b10f2ddd7679 clarified modules;
wenzelm
parents: 64120
diff changeset
    45
64298
wenzelm
parents: 64297
diff changeset
    46
  /* settings */
64080
2e5c0bd708af clarified modules;
wenzelm
parents: 64079
diff changeset
    47
64081
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    48
  object Settings
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    49
  {
65611
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
    50
    val ISABELLE_BUILD_OPTIONS = SQL.Column.string("ISABELLE_BUILD_OPTIONS")
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
    51
    val ML_PLATFORM = SQL.Column.string("ML_PLATFORM")
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
    52
    val ML_HOME = SQL.Column.string("ML_HOME")
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
    53
    val ML_SYSTEM = SQL.Column.string("ML_SYSTEM")
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
    54
    val ML_OPTIONS = SQL.Column.string("ML_OPTIONS")
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
    55
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
    56
    val ml_settings = List(ML_PLATFORM, ML_HOME, ML_SYSTEM, ML_OPTIONS)
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
    57
    val all_settings = ISABELLE_BUILD_OPTIONS :: ml_settings
64081
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    58
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    59
    type Entry = (String, String)
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    60
    type T = List[Entry]
64080
2e5c0bd708af clarified modules;
wenzelm
parents: 64079
diff changeset
    61
64081
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    62
    object Entry
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    63
    {
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    64
      def unapply(s: String): Option[Entry] =
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    65
        s.indexOf('=') match {
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    66
          case -1 => None
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    67
          case i =>
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    68
            val a = s.substring(0, i)
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    69
            val b = Library.perhaps_unquote(s.substring(i + 1))
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    70
            Some((a, b))
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    71
        }
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    72
      def apply(a: String, b: String): String = a + "=" + quote(b)
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    73
      def getenv(a: String): String = apply(a, Isabelle_System.getenv(a))
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    74
    }
64080
2e5c0bd708af clarified modules;
wenzelm
parents: 64079
diff changeset
    75
64081
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    76
    def show(): String =
38bb09ed965b more uniform treatment of settings;
wenzelm
parents: 64080
diff changeset
    77
      cat_lines(
65611
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
    78
        List(Entry.getenv(ISABELLE_BUILD_OPTIONS.name), "") :::
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
    79
        ml_settings.map(c => Entry.getenv(c.name)))
64080
2e5c0bd708af clarified modules;
wenzelm
parents: 64079
diff changeset
    80
  }
2e5c0bd708af clarified modules;
wenzelm
parents: 64079
diff changeset
    81
2e5c0bd708af clarified modules;
wenzelm
parents: 64079
diff changeset
    82
64298
wenzelm
parents: 64297
diff changeset
    83
  /* file names */
wenzelm
parents: 64297
diff changeset
    84
wenzelm
parents: 64297
diff changeset
    85
  def log_date(date: Date): String =
wenzelm
parents: 64297
diff changeset
    86
    String.format(Locale.ROOT, "%s.%05d",
wenzelm
parents: 64297
diff changeset
    87
      DateTimeFormatter.ofPattern("yyyy-MM-dd").format(date.rep),
wenzelm
parents: 64297
diff changeset
    88
      new java.lang.Long((date.time - date.midnight.time).ms / 1000))
wenzelm
parents: 64297
diff changeset
    89
wenzelm
parents: 64297
diff changeset
    90
  def log_subdir(date: Date): Path =
wenzelm
parents: 64297
diff changeset
    91
    Path.explode("log") + Path.explode(date.rep.getYear.toString)
wenzelm
parents: 64297
diff changeset
    92
wenzelm
parents: 64297
diff changeset
    93
  def log_filename(engine: String, date: Date, more: List[String] = Nil): Path =
wenzelm
parents: 64297
diff changeset
    94
    Path.explode((engine :: log_date(date) :: more).mkString("", "_", ".log"))
wenzelm
parents: 64297
diff changeset
    95
wenzelm
parents: 64297
diff changeset
    96
64100
9b1573213ebe tuned error;
wenzelm
parents: 64099
diff changeset
    97
64062
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
    98
  /** log file **/
64045
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
    99
64155
646c4d6a6a02 tuned signature;
wenzelm
parents: 64150
diff changeset
   100
  def print_date(date: Date): String = Log_File.Date_Format(date)
646c4d6a6a02 tuned signature;
wenzelm
parents: 64150
diff changeset
   101
64062
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   102
  object Log_File
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   103
  {
65607
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   104
    /* log file */
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   105
65609
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   106
    def plain_name(name: String): String =
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   107
    {
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   108
      List(".log", ".log.gz", ".log.xz", ".gz", ".xz").find(name.endsWith(_)) match {
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   109
        case Some(s) => Library.try_unsuffix(s, name).get
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   110
        case None => name
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   111
      }
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   112
    }
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   113
64062
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   114
    def apply(name: String, lines: List[String]): Log_File =
65609
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   115
      new Log_File(plain_name(name), lines)
64062
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   116
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   117
    def apply(name: String, text: String): Log_File =
64063
2c5039363ea3 tuned signature;
wenzelm
parents: 64062
diff changeset
   118
      Log_File(name, Library.trim_split_lines(text))
64090
5a68280112b3 more operations;
wenzelm
parents: 64089
diff changeset
   119
5a68280112b3 more operations;
wenzelm
parents: 64089
diff changeset
   120
    def apply(file: JFile): Log_File =
5a68280112b3 more operations;
wenzelm
parents: 64089
diff changeset
   121
    {
5a68280112b3 more operations;
wenzelm
parents: 64089
diff changeset
   122
      val name = file.getName
65609
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   123
      val text =
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   124
        if (name.endsWith(".gz")) File.read_gzip(file)
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   125
        else if (name.endsWith(".xz")) File.read_xz(file)
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   126
        else File.read(file)
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   127
      apply(name, text)
64090
5a68280112b3 more operations;
wenzelm
parents: 64089
diff changeset
   128
    }
5a68280112b3 more operations;
wenzelm
parents: 64089
diff changeset
   129
5a68280112b3 more operations;
wenzelm
parents: 64089
diff changeset
   130
    def apply(path: Path): Log_File = apply(path.file)
64101
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   131
64110
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   132
65607
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   133
    /* log file collections */
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   134
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   135
    def is_log(file: JFile,
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   136
      prefixes: List[String] =
65664
c84db5e0dd6d detect Jenkins log files as well;
wenzelm
parents: 65663
diff changeset
   137
        List(Build_History.log_prefix, Identify.log_prefix, Isatest.log_prefix,
c84db5e0dd6d detect Jenkins log files as well;
wenzelm
parents: 65663
diff changeset
   138
          AFP_Test.log_prefix, Jenkins.log_prefix),
65609
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   139
      suffixes: List[String] = List(".log", ".log.gz", ".log.xz")): Boolean =
65607
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   140
    {
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   141
      val name = file.getName
65639
4c14da234221 exclude special log files;
wenzelm
parents: 65636
diff changeset
   142
65607
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   143
      prefixes.exists(name.startsWith(_)) &&
65639
4c14da234221 exclude special log files;
wenzelm
parents: 65636
diff changeset
   144
      suffixes.exists(name.endsWith(_)) &&
4c14da234221 exclude special log files;
wenzelm
parents: 65636
diff changeset
   145
      name != "isatest.log" &&
4c14da234221 exclude special log files;
wenzelm
parents: 65636
diff changeset
   146
      name != "afp-test.log" &&
4c14da234221 exclude special log files;
wenzelm
parents: 65636
diff changeset
   147
      name != "main.log"
65607
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   148
    }
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   149
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   150
    def find_files(dirs: Iterable[Path]): List[JFile] =
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   151
      dirs.iterator.flatMap(dir => File.find_files(dir.file, is_log(_))).toList
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   152
c937984c70e9 tuned signature;
wenzelm
parents: 65605
diff changeset
   153
64110
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   154
    /* date format */
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   155
64101
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   156
    val Date_Format =
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   157
    {
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   158
      val fmts =
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   159
        Date.Formatter.variants(
64116
6cfd429a4296 prefer explicit timezone offset for printing;
wenzelm
parents: 64111
diff changeset
   160
          List("EEE MMM d HH:mm:ss O yyyy", "EEE MMM d HH:mm:ss VV yyyy"),
64104
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   161
          List(Locale.ENGLISH, Locale.GERMAN)) :::
64110
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   162
        List(
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   163
          DateTimeFormatter.RFC_1123_DATE_TIME,
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   164
          Date.Formatter.pattern("EEE MMM d HH:mm:ss yyyy").withZone(ZoneId.of("Europe/Berlin")))
64101
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   165
64104
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   166
      def tune_timezone(s: String): String =
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   167
        s match {
64101
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   168
          case "CET" | "MET" => "GMT+1"
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   169
          case "CEST" | "MEST" => "GMT+2"
64104
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   170
          case "EST" => "Europe/Berlin"
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   171
          case _ => s
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   172
        }
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   173
      def tune_weekday(s: String): String =
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   174
        s match {
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   175
          case "Die" => "Di"
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   176
          case "Mit" => "Mi"
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   177
          case "Don" => "Do"
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   178
          case "Fre" => "Fr"
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   179
          case "Sam" => "Sa"
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   180
          case "Son" => "So"
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   181
          case _ => s
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   182
        }
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   183
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   184
      def tune(s: String): String =
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   185
        Word.implode(
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   186
          Word.explode(s) match {
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   187
            case a :: "M\uFFFDr" :: bs => tune_weekday(a) :: "Mär" :: bs.map(tune_timezone(_))
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   188
            case a :: bs => tune_weekday(a) :: bs.map(tune_timezone(_))
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   189
            case Nil => Nil
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   190
          }
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   191
        )
64101
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   192
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   193
      Date.Format.make(fmts, tune)
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   194
    }
64117
c2b41b073d8a build_history log files with formal meta info;
wenzelm
parents: 64116
diff changeset
   195
c2b41b073d8a build_history log files with formal meta info;
wenzelm
parents: 64116
diff changeset
   196
c2b41b073d8a build_history log files with formal meta info;
wenzelm
parents: 64116
diff changeset
   197
    /* inlined content */
c2b41b073d8a build_history log files with formal meta info;
wenzelm
parents: 64116
diff changeset
   198
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   199
    def print_props(marker: String, props: Properties.T): String =
65624
32fa61f694ef clarified multi-line properties;
wenzelm
parents: 65623
diff changeset
   200
      marker + YXML.string_of_body(XML.Encode.properties(Properties.encode_lines(props)))
64102
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   201
  }
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   202
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   203
  class Log_File private(val name: String, val lines: List[String])
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   204
  {
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   205
    log_file =>
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   206
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   207
    override def toString: String = name
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   208
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   209
    def text: String = cat_lines(lines)
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   210
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   211
    def err(msg: String): Nothing =
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   212
      error("Error in log file " + quote(name) + ": " + msg)
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   213
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   214
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   215
    /* date format */
64101
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   216
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   217
    object Strict_Date
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   218
    {
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   219
      def unapply(s: String): Some[Date] =
64102
1ec2adddf16b prefer static Date_Format;
wenzelm
parents: 64101
diff changeset
   220
        try { Some(Log_File.Date_Format.parse(s)) }
64101
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   221
        catch { case exn: DateTimeParseException => log_file.err(exn.getMessage) }
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   222
    }
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   223
976289c733e6 more formal directory content;
wenzelm
parents: 64100
diff changeset
   224
64062
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   225
    /* inlined content */
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   226
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   227
    def find[A](f: String => Option[A]): Option[A] =
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   228
      lines.iterator.map(f).find(_.isDefined).map(_.get)
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   229
64196
6688b9cd443b more robust wrt. old versions that use clear-text properties (e.g. Timing in build_history_base);
wenzelm
parents: 64193
diff changeset
   230
    def find_line(marker: String): Option[String] =
6688b9cd443b more robust wrt. old versions that use clear-text properties (e.g. Timing in build_history_base);
wenzelm
parents: 64193
diff changeset
   231
      find(Library.try_unprefix(marker, _))
6688b9cd443b more robust wrt. old versions that use clear-text properties (e.g. Timing in build_history_base);
wenzelm
parents: 64193
diff changeset
   232
65684
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   233
    def find_match(regexes: List[Regex]): Option[String] =
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   234
      regexes match {
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   235
        case Nil => None
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   236
        case regex :: rest =>
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   237
          lines.iterator.map(regex.unapplySeq(_)).find(res => res.isDefined && res.get.length == 1).
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   238
            map(res => res.get.head) orElse find_match(rest)
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   239
      }
64062
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   240
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   241
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   242
    /* settings */
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   243
64091
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   244
    def get_setting(a: String): Option[Settings.Entry] =
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   245
      lines.find(_.startsWith(a + "=")) match {
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   246
        case Some(line) => Settings.Entry.unapply(line)
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   247
        case None => None
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   248
      }
64045
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
   249
65611
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
   250
    def get_all_settings: Settings.T =
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
   251
      for { c <- Settings.all_settings; entry <- get_setting(c.name) }
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
   252
      yield entry
64062
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   253
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   254
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   255
    /* properties (YXML) */
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   256
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   257
    val xml_cache = new XML.Cache()
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   258
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   259
    def parse_props(text: String): Properties.T =
65624
32fa61f694ef clarified multi-line properties;
wenzelm
parents: 65623
diff changeset
   260
      xml_cache.props(Properties.decode_lines(XML.Decode.properties(YXML.parse_body(text))))
64062
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   261
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   262
    def filter_props(marker: String): List[Properties.T] =
64196
6688b9cd443b more robust wrt. old versions that use clear-text properties (e.g. Timing in build_history_base);
wenzelm
parents: 64193
diff changeset
   263
      for {
6688b9cd443b more robust wrt. old versions that use clear-text properties (e.g. Timing in build_history_base);
wenzelm
parents: 64193
diff changeset
   264
        line <- lines
6688b9cd443b more robust wrt. old versions that use clear-text properties (e.g. Timing in build_history_base);
wenzelm
parents: 64193
diff changeset
   265
        s <- Library.try_unprefix(marker, line)
6688b9cd443b more robust wrt. old versions that use clear-text properties (e.g. Timing in build_history_base);
wenzelm
parents: 64193
diff changeset
   266
        if YXML.detect(s)
6688b9cd443b more robust wrt. old versions that use clear-text properties (e.g. Timing in build_history_base);
wenzelm
parents: 64193
diff changeset
   267
      } yield parse_props(s)
64062
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   268
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   269
    def find_props(marker: String): Option[Properties.T] =
64196
6688b9cd443b more robust wrt. old versions that use clear-text properties (e.g. Timing in build_history_base);
wenzelm
parents: 64193
diff changeset
   270
      find_line(marker) match {
6688b9cd443b more robust wrt. old versions that use clear-text properties (e.g. Timing in build_history_base);
wenzelm
parents: 64193
diff changeset
   271
        case Some(text) if YXML.detect(text) => Some(parse_props(text))
6688b9cd443b more robust wrt. old versions that use clear-text properties (e.g. Timing in build_history_base);
wenzelm
parents: 64193
diff changeset
   272
        case _ => None
6688b9cd443b more robust wrt. old versions that use clear-text properties (e.g. Timing in build_history_base);
wenzelm
parents: 64193
diff changeset
   273
      }
64062
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   274
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   275
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   276
    /* parse various formats */
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   277
64105
d93bd6d253c6 tuned signature;
wenzelm
parents: 64104
diff changeset
   278
    def parse_meta_info(): Meta_Info = Build_Log.parse_meta_info(log_file)
d93bd6d253c6 tuned signature;
wenzelm
parents: 64104
diff changeset
   279
65646
014dbbe5331f parse ml_statistics only when required;
wenzelm
parents: 65645
diff changeset
   280
    def parse_build_info(ml_statistics: Boolean = false): Build_Info =
014dbbe5331f parse ml_statistics only when required;
wenzelm
parents: 65645
diff changeset
   281
      Build_Log.parse_build_info(log_file, ml_statistics)
64105
d93bd6d253c6 tuned signature;
wenzelm
parents: 64104
diff changeset
   282
64082
d57c7295f601 clarified signature;
wenzelm
parents: 64081
diff changeset
   283
    def parse_session_info(
d57c7295f601 clarified signature;
wenzelm
parents: 64081
diff changeset
   284
        command_timings: Boolean = false,
d57c7295f601 clarified signature;
wenzelm
parents: 64081
diff changeset
   285
        ml_statistics: Boolean = false,
d57c7295f601 clarified signature;
wenzelm
parents: 64081
diff changeset
   286
        task_statistics: Boolean = false): Session_Info =
65318
342efc382558 eliminated somewhat redundant inlined name (despite a7aa17a1f721);
wenzelm
parents: 65290
diff changeset
   287
      Build_Log.parse_session_info(log_file, command_timings, ml_statistics, task_statistics)
64045
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
   288
  }
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
   289
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
   290
64098
099518e8af2c misc tuning and clarification;
wenzelm
parents: 64096
diff changeset
   291
65276
fa1a5efee2ec tuned comments;
wenzelm
parents: 65052
diff changeset
   292
  /** digested meta info: produced by Admin/build_history in log.xz file **/
64045
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
   293
64108
623abb8fecdf clarified meta info;
wenzelm
parents: 64107
diff changeset
   294
  object Meta_Info
64099
wenzelm
parents: 64098
diff changeset
   295
  {
64108
623abb8fecdf clarified meta info;
wenzelm
parents: 64107
diff changeset
   296
    val empty: Meta_Info = Meta_Info(Nil, Nil)
65605
a6447eb6bc38 separate small meta_info vs. big build_info;
wenzelm
parents: 65603
diff changeset
   297
65609
9917b8e3b5c1 clarified plain_name / log_name;
wenzelm
parents: 65608
diff changeset
   298
    val log_name = SQL.Column.string("log_name", primary_key = true)
65605
a6447eb6bc38 separate small meta_info vs. big build_info;
wenzelm
parents: 65603
diff changeset
   299
    val table =
65611
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
   300
      SQL.Table("isabelle_build_log_meta_info", log_name :: Prop.all_props ::: Settings.all_settings)
64099
wenzelm
parents: 64098
diff changeset
   301
  }
64098
099518e8af2c misc tuning and clarification;
wenzelm
parents: 64096
diff changeset
   302
65611
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
   303
  sealed case class Meta_Info(props: Properties.T, settings: Settings.T)
64103
60d163f38056 accept spurious empty logs;
wenzelm
parents: 64102
diff changeset
   304
  {
60d163f38056 accept spurious empty logs;
wenzelm
parents: 64102
diff changeset
   305
    def is_empty: Boolean = props.isEmpty && settings.isEmpty
65599
08dfa79866ec database storage of Meta_Info and Build_Info;
wenzelm
parents: 65595
diff changeset
   306
65611
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
   307
    def get(c: SQL.Column): Option[String] =
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
   308
      Properties.get(props, c.name) orElse
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
   309
      Properties.get(settings, c.name)
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
   310
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
   311
    def get_date(c: SQL.Column): Option[Date] =
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
   312
      get(c).map(Log_File.Date_Format.parse(_))
64103
60d163f38056 accept spurious empty logs;
wenzelm
parents: 64102
diff changeset
   313
  }
64061
1bbea2b55d22 some support for header and data fields, notably from afp-test;
wenzelm
parents: 64054
diff changeset
   314
65625
13552d5c0005 more log files;
wenzelm
parents: 65624
diff changeset
   315
  object Identify
13552d5c0005 more log files;
wenzelm
parents: 65624
diff changeset
   316
  {
13552d5c0005 more log files;
wenzelm
parents: 65624
diff changeset
   317
    val log_prefix = "isabelle_identify_"
65674
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   318
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   319
    def engine(log_file: Log_File): String =
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   320
      if (log_file.name.startsWith(Jenkins.log_prefix)) "jenkins_identify"
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   321
      else "identify"
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   322
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   323
    def content(date: Date, isabelle_version: Option[String], afp_version: Option[String]): String =
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   324
      terminate_lines(
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   325
        List("isabelle_identify: " + Build_Log.print_date(date), "") :::
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   326
        isabelle_version.map("Isabelle version: " + _).toList :::
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   327
        afp_version.map("AFP version: " + _).toList)
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   328
65625
13552d5c0005 more log files;
wenzelm
parents: 65624
diff changeset
   329
    val Start = new Regex("""^isabelle_identify: (.+)$""")
13552d5c0005 more log files;
wenzelm
parents: 65624
diff changeset
   330
    val No_End = new Regex("""$.""")
65684
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   331
    val Isabelle_Version = List(new Regex("""^Isabelle version: (\S+)$"""))
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   332
    val AFP_Version = List(new Regex("""^AFP version: (\S+)$"""))
65625
13552d5c0005 more log files;
wenzelm
parents: 65624
diff changeset
   333
  }
13552d5c0005 more log files;
wenzelm
parents: 65624
diff changeset
   334
64095
1a6d37c31df9 support for isatest format;
wenzelm
parents: 64094
diff changeset
   335
  object Isatest
1a6d37c31df9 support for isatest format;
wenzelm
parents: 64094
diff changeset
   336
  {
65588
b0d8d97198b3 clarified signature;
wenzelm
parents: 65318
diff changeset
   337
    val log_prefix = "isatest-makeall-"
64108
623abb8fecdf clarified meta info;
wenzelm
parents: 64107
diff changeset
   338
    val engine = "isatest"
64109
wenzelm
parents: 64108
diff changeset
   339
    val Start = new Regex("""^------------------- starting test --- (.+) --- (.+)$""")
wenzelm
parents: 64108
diff changeset
   340
    val End = new Regex("""^------------------- test (?:successful|FAILED) --- (.+) --- .*$""")
65684
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   341
    val Isabelle_Version = List(new Regex("""^Isabelle version: (\S+)$"""))
64095
1a6d37c31df9 support for isatest format;
wenzelm
parents: 64094
diff changeset
   342
  }
1a6d37c31df9 support for isatest format;
wenzelm
parents: 64094
diff changeset
   343
64109
wenzelm
parents: 64108
diff changeset
   344
  object AFP_Test
64061
1bbea2b55d22 some support for header and data fields, notably from afp-test;
wenzelm
parents: 64054
diff changeset
   345
  {
65588
b0d8d97198b3 clarified signature;
wenzelm
parents: 65318
diff changeset
   346
    val log_prefix = "afp-test-devel-"
64108
623abb8fecdf clarified meta info;
wenzelm
parents: 64107
diff changeset
   347
    val engine = "afp-test"
64109
wenzelm
parents: 64108
diff changeset
   348
    val Start = new Regex("""^Start test(?: for .+)? at ([^,]+), (.*)$""")
wenzelm
parents: 64108
diff changeset
   349
    val Start_Old = new Regex("""^Start test(?: for .+)? at ([^,]+)$""")
wenzelm
parents: 64108
diff changeset
   350
    val End = new Regex("""^End test on (.+), .+, elapsed time:.*$""")
65684
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   351
    val Isabelle_Version = List(new Regex("""^Isabelle version: .* -- hg id (\S+)$"""))
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   352
    val AFP_Version = List(new Regex("""^AFP version: .* -- hg id (\S+)$"""))
64104
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   353
    val Bad_Init = new Regex("""^cp:.*: Disc quota exceeded$""")
64061
1bbea2b55d22 some support for header and data fields, notably from afp-test;
wenzelm
parents: 64054
diff changeset
   354
  }
1bbea2b55d22 some support for header and data fields, notably from afp-test;
wenzelm
parents: 64054
diff changeset
   355
64110
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   356
  object Jenkins
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   357
  {
65664
c84db5e0dd6d detect Jenkins log files as well;
wenzelm
parents: 65663
diff changeset
   358
    val log_prefix = "jenkins_"
64110
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   359
    val engine = "jenkins"
65663
61cd86bb9613 detect host name;
wenzelm
parents: 65649
diff changeset
   360
    val Host = new Regex("""^Building remotely on (\S+) \((\S+)\).*$""")
65665
9b7fb07b4a96 more robust detection of Jenkins log;
wenzelm
parents: 65664
diff changeset
   361
    val Start = new Regex("""^(?:Started by an SCM change|Started from command line by admin|).*$""")
64110
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   362
    val Start_Date = new Regex("""^Build started at (.+)$""")
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   363
    val No_End = new Regex("""$.""")
65674
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   364
    val Isabelle_Version =
65684
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   365
      List(new Regex("""^(?:Build for Isabelle id|Isabelle id) (\w+).*$"""),
65685
47bbf7150aae detect old log files (early 2016);
wenzelm
parents: 65684
diff changeset
   366
        new Regex("""^ISABELLE_CI_REPO_ID="(\w+)".*$"""),
47bbf7150aae detect old log files (early 2016);
wenzelm
parents: 65684
diff changeset
   367
        new Regex("""^(\w{12}) tip.*$"""))
65674
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   368
    val AFP_Version =
65684
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   369
      List(new Regex("""^(?:Build for AFP id|AFP id) (\w+).*$"""),
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   370
        new Regex("""^ISABELLE_CI_AFP_ID="(\w+)".*$"""))
64110
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   371
    val CONFIGURATION = "=== CONFIGURATION ==="
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   372
    val BUILD = "=== BUILD ==="
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   373
  }
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   374
64105
d93bd6d253c6 tuned signature;
wenzelm
parents: 64104
diff changeset
   375
  private def parse_meta_info(log_file: Log_File): Meta_Info =
64061
1bbea2b55d22 some support for header and data fields, notably from afp-test;
wenzelm
parents: 64054
diff changeset
   376
  {
64108
623abb8fecdf clarified meta info;
wenzelm
parents: 64107
diff changeset
   377
    def parse(engine: String, host: String, start: Date,
65684
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   378
      End: Regex, Isabelle_Version: List[Regex], AFP_Version: List[Regex]): Meta_Info =
64091
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   379
    {
64296
544481988e65 explicit identification of builds and correlated build groups;
wenzelm
parents: 64196
diff changeset
   380
      val build_id =
544481988e65 explicit identification of builds and correlated build groups;
wenzelm
parents: 64196
diff changeset
   381
      {
544481988e65 explicit identification of builds and correlated build groups;
wenzelm
parents: 64196
diff changeset
   382
        val prefix = if (host != "") host else if (engine != "") engine else ""
544481988e65 explicit identification of builds and correlated build groups;
wenzelm
parents: 64196
diff changeset
   383
        (if (prefix == "") "build" else prefix) + ":" + start.time.ms
544481988e65 explicit identification of builds and correlated build groups;
wenzelm
parents: 64196
diff changeset
   384
      }
65591
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
   385
      val build_engine = if (engine == "") Nil else List(Prop.build_engine.name -> engine)
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
   386
      val build_host = if (host == "") Nil else List(Prop.build_host.name -> host)
64108
623abb8fecdf clarified meta info;
wenzelm
parents: 64107
diff changeset
   387
65599
08dfa79866ec database storage of Meta_Info and Build_Info;
wenzelm
parents: 65595
diff changeset
   388
      val start_date = List(Prop.build_start.name -> print_date(start))
64091
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   389
      val end_date =
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   390
        log_file.lines.last match {
64109
wenzelm
parents: 64108
diff changeset
   391
          case End(log_file.Strict_Date(end_date)) =>
65599
08dfa79866ec database storage of Meta_Info and Build_Info;
wenzelm
parents: 65595
diff changeset
   392
            List(Prop.build_end.name -> print_date(end_date))
64091
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   393
          case _ => Nil
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   394
        }
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   395
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   396
      val isabelle_version =
65591
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
   397
        log_file.find_match(Isabelle_Version).map(Prop.isabelle_version.name -> _)
64091
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   398
      val afp_version =
65591
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
   399
        log_file.find_match(AFP_Version).map(Prop.afp_version.name -> _)
64062
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   400
65591
5953c7fbc2b8 more SQL operations;
wenzelm
parents: 65590
diff changeset
   401
      Meta_Info((Prop.build_id.name -> build_id) :: build_engine ::: build_host :::
64108
623abb8fecdf clarified meta info;
wenzelm
parents: 64107
diff changeset
   402
          start_date ::: end_date ::: isabelle_version.toList ::: afp_version.toList,
65611
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
   403
        log_file.get_all_settings)
64091
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   404
    }
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   405
f8dfba90e73f more liberal parsing for old AFP logs;
wenzelm
parents: 64090
diff changeset
   406
    log_file.lines match {
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   407
      case line :: _ if line.startsWith(Build_History.META_INFO_MARKER) =>
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   408
        Meta_Info(log_file.find_props(Build_History.META_INFO_MARKER).get,
65611
a4a7841ae84f more uniform storage of Meta_Info;
wenzelm
parents: 65609
diff changeset
   409
          log_file.get_all_settings)
64117
c2b41b073d8a build_history log files with formal meta info;
wenzelm
parents: 64116
diff changeset
   410
65625
13552d5c0005 more log files;
wenzelm
parents: 65624
diff changeset
   411
      case Identify.Start(log_file.Strict_Date(start)) :: _ =>
65674
23897f5d885d approximate repository identify job based on isabelle-nightly-slow;
wenzelm
parents: 65670
diff changeset
   412
        parse(Identify.engine(log_file), "", start, Identify.No_End,
65625
13552d5c0005 more log files;
wenzelm
parents: 65624
diff changeset
   413
          Identify.Isabelle_Version, Identify.AFP_Version)
13552d5c0005 more log files;
wenzelm
parents: 65624
diff changeset
   414
64109
wenzelm
parents: 64108
diff changeset
   415
      case Isatest.Start(log_file.Strict_Date(start), host) :: _ =>
wenzelm
parents: 64108
diff changeset
   416
        parse(Isatest.engine, host, start, Isatest.End,
65684
00d4663270d9 explicit support for multiple regexes;
wenzelm
parents: 65683
diff changeset
   417
          Isatest.Isabelle_Version, Nil)
64099
wenzelm
parents: 64098
diff changeset
   418
64109
wenzelm
parents: 64108
diff changeset
   419
      case AFP_Test.Start(log_file.Strict_Date(start), host) :: _ =>
wenzelm
parents: 64108
diff changeset
   420
        parse(AFP_Test.engine, host, start, AFP_Test.End,
wenzelm
parents: 64108
diff changeset
   421
          AFP_Test.Isabelle_Version, AFP_Test.AFP_Version)
64099
wenzelm
parents: 64098
diff changeset
   422
64109
wenzelm
parents: 64108
diff changeset
   423
      case AFP_Test.Start_Old(log_file.Strict_Date(start)) :: _ =>
wenzelm
parents: 64108
diff changeset
   424
        parse(AFP_Test.engine, "", start, AFP_Test.End,
wenzelm
parents: 64108
diff changeset
   425
          AFP_Test.Isabelle_Version, AFP_Test.AFP_Version)
64099
wenzelm
parents: 64098
diff changeset
   426
65665
9b7fb07b4a96 more robust detection of Jenkins log;
wenzelm
parents: 65664
diff changeset
   427
      case Jenkins.Start() :: _ =>
64110
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   428
        log_file.lines.dropWhile(_ != Jenkins.BUILD) match {
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   429
          case Jenkins.BUILD :: _ :: Jenkins.Start_Date(log_file.Strict_Date(start)) :: _ =>
65663
61cd86bb9613 detect host name;
wenzelm
parents: 65649
diff changeset
   430
            val host =
61cd86bb9613 detect host name;
wenzelm
parents: 65649
diff changeset
   431
              log_file.lines.takeWhile(_ != Jenkins.CONFIGURATION).collectFirst({
61cd86bb9613 detect host name;
wenzelm
parents: 65649
diff changeset
   432
                case Jenkins.Host(a, b) => a + "." + b
61cd86bb9613 detect host name;
wenzelm
parents: 65649
diff changeset
   433
              }).getOrElse("")
61cd86bb9613 detect host name;
wenzelm
parents: 65649
diff changeset
   434
            parse(Jenkins.engine, host, start.to(ZoneId.of("Europe/Berlin")), Jenkins.No_End,
64110
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   435
              Jenkins.Isabelle_Version, Jenkins.AFP_Version)
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   436
          case _ => Meta_Info.empty
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   437
        }
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   438
64341
45b6faeee56d avoid deprecated Scala;
wenzelm
parents: 64303
diff changeset
   439
      case line :: _ if line.startsWith("\u0000") => Meta_Info.empty
64109
wenzelm
parents: 64108
diff changeset
   440
      case List(Isatest.End(_)) => Meta_Info.empty
wenzelm
parents: 64108
diff changeset
   441
      case _ :: AFP_Test.Bad_Init() :: _ => Meta_Info.empty
64105
d93bd6d253c6 tuned signature;
wenzelm
parents: 64104
diff changeset
   442
      case Nil => Meta_Info.empty
64104
b70fa05d6746 more permissive: accept all historic isatest and afp-test logs;
wenzelm
parents: 64103
diff changeset
   443
64110
c0b96b34c7b9 support for Isabelle/Jenkins log file format;
wenzelm
parents: 64109
diff changeset
   444
      case _ => log_file.err("cannot detect log file format")
64061
1bbea2b55d22 some support for header and data fields, notably from afp-test;
wenzelm
parents: 64054
diff changeset
   445
    }
1bbea2b55d22 some support for header and data fields, notably from afp-test;
wenzelm
parents: 64054
diff changeset
   446
  }
1bbea2b55d22 some support for header and data fields, notably from afp-test;
wenzelm
parents: 64054
diff changeset
   447
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   448
64098
099518e8af2c misc tuning and clarification;
wenzelm
parents: 64096
diff changeset
   449
65276
fa1a5efee2ec tuned comments;
wenzelm
parents: 65052
diff changeset
   450
  /** build info: toplevel output of isabelle build or Admin/build_history **/
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   451
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   452
  val ML_STATISTICS_MARKER = "\fML_statistics = "
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   453
  val SESSION_NAME = "session_name"
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   454
64061
1bbea2b55d22 some support for header and data fields, notably from afp-test;
wenzelm
parents: 64054
diff changeset
   455
  object Session_Status extends Enumeration
1bbea2b55d22 some support for header and data fields, notably from afp-test;
wenzelm
parents: 64054
diff changeset
   456
  {
65633
wenzelm
parents: 65632
diff changeset
   457
    val existing, finished, failed, cancelled = Value
64061
1bbea2b55d22 some support for header and data fields, notably from afp-test;
wenzelm
parents: 64054
diff changeset
   458
  }
1bbea2b55d22 some support for header and data fields, notably from afp-test;
wenzelm
parents: 64054
diff changeset
   459
65643
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   460
  object Session_Entry
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   461
  {
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   462
    val empty: Session_Entry = Session_Entry()
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   463
  }
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   464
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   465
  sealed case class Session_Entry(
65643
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   466
    chapter: String = "",
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   467
    groups: List[String] = Nil,
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   468
    threads: Option[Int] = None,
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   469
    timing: Timing = Timing.zero,
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   470
    ml_timing: Timing = Timing.zero,
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   471
    heap_size: Option[Long] = None,
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   472
    status: Option[Session_Status.Value] = None,
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   473
    ml_statistics: List[Properties.T] = Nil)
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   474
  {
65631
ee917f172912 clarified database content;
wenzelm
parents: 65629
diff changeset
   475
    def proper_chapter: Option[String] = if (chapter == "") None else Some(chapter)
ee917f172912 clarified database content;
wenzelm
parents: 65629
diff changeset
   476
    def proper_groups: Option[String] = if (groups.isEmpty) None else Some(cat_lines(groups))
65643
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   477
    def finished: Boolean = status == Some(Session_Status.finished)
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   478
  }
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   479
65599
08dfa79866ec database storage of Meta_Info and Build_Info;
wenzelm
parents: 65595
diff changeset
   480
  object Build_Info
08dfa79866ec database storage of Meta_Info and Build_Info;
wenzelm
parents: 65595
diff changeset
   481
  {
65613
cfcafe9824d1 clarified database layout;
wenzelm
parents: 65611
diff changeset
   482
    val session_name = SQL.Column.string("session_name", primary_key = true)
cfcafe9824d1 clarified database layout;
wenzelm
parents: 65611
diff changeset
   483
    val chapter = SQL.Column.string("chapter")
cfcafe9824d1 clarified database layout;
wenzelm
parents: 65611
diff changeset
   484
    val groups = SQL.Column.string("groups")
cfcafe9824d1 clarified database layout;
wenzelm
parents: 65611
diff changeset
   485
    val threads = SQL.Column.int("threads")
65616
b8738569b8db clarified database layout;
wenzelm
parents: 65615
diff changeset
   486
    val timing_elapsed = SQL.Column.long("timing_elapsed")
b8738569b8db clarified database layout;
wenzelm
parents: 65615
diff changeset
   487
    val timing_cpu = SQL.Column.long("timing_cpu")
b8738569b8db clarified database layout;
wenzelm
parents: 65615
diff changeset
   488
    val timing_gc = SQL.Column.long("timing_gc")
65683
70b0ef74ef3a some derived data fields, to facilitate queries;
wenzelm
parents: 65682
diff changeset
   489
    val timing_factor = SQL.Column.double("timing_factor")
65616
b8738569b8db clarified database layout;
wenzelm
parents: 65615
diff changeset
   490
    val ml_timing_elapsed = SQL.Column.long("ml_timing_elapsed")
b8738569b8db clarified database layout;
wenzelm
parents: 65615
diff changeset
   491
    val ml_timing_cpu = SQL.Column.long("ml_timing_cpu")
b8738569b8db clarified database layout;
wenzelm
parents: 65615
diff changeset
   492
    val ml_timing_gc = SQL.Column.long("ml_timing_gc")
65683
70b0ef74ef3a some derived data fields, to facilitate queries;
wenzelm
parents: 65682
diff changeset
   493
    val ml_timing_factor = SQL.Column.double("ml_timing_factor")
65613
cfcafe9824d1 clarified database layout;
wenzelm
parents: 65611
diff changeset
   494
    val heap_size = SQL.Column.long("heap_size")
cfcafe9824d1 clarified database layout;
wenzelm
parents: 65611
diff changeset
   495
    val status = SQL.Column.string("status")
65627
bb185e442c95 put bulky data last;
wenzelm
parents: 65626
diff changeset
   496
    val ml_statistics = SQL.Column.bytes("ml_statistics")
65605
a6447eb6bc38 separate small meta_info vs. big build_info;
wenzelm
parents: 65603
diff changeset
   497
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   498
    val sessions_table =
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   499
      SQL.Table("isabelle_build_log_sessions",
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   500
        List(Meta_Info.log_name, session_name, chapter, groups, threads, timing_elapsed, timing_cpu,
65683
70b0ef74ef3a some derived data fields, to facilitate queries;
wenzelm
parents: 65682
diff changeset
   501
          timing_gc, timing_factor, ml_timing_elapsed, ml_timing_cpu, ml_timing_gc, ml_timing_factor,
70b0ef74ef3a some derived data fields, to facilitate queries;
wenzelm
parents: 65682
diff changeset
   502
          heap_size, status))
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   503
    val ml_statistics_table =
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   504
      SQL.Table("isabelle_build_log_ml_statistics",
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   505
        List(Meta_Info.log_name, session_name, ml_statistics))
65599
08dfa79866ec database storage of Meta_Info and Build_Info;
wenzelm
parents: 65595
diff changeset
   506
  }
08dfa79866ec database storage of Meta_Info and Build_Info;
wenzelm
parents: 65595
diff changeset
   507
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   508
  sealed case class Build_Info(sessions: Map[String, Session_Entry])
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   509
  {
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   510
    def session(name: String): Session_Entry = sessions(name)
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   511
    def get_session(name: String): Option[Session_Entry] = sessions.get(name)
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   512
64089
10d719dbb3ee more permissive timing data;
wenzelm
parents: 64088
diff changeset
   513
    def get_default[A](name: String, f: Session_Entry => A, x: A): A =
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   514
      get_session(name) match {
64089
10d719dbb3ee more permissive timing data;
wenzelm
parents: 64088
diff changeset
   515
        case Some(entry) => f(entry)
10d719dbb3ee more permissive timing data;
wenzelm
parents: 64088
diff changeset
   516
        case None => x
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   517
      }
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   518
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   519
    def finished_sessions: List[String] = sessions.keySet.iterator.filter(finished(_)).toList
64089
10d719dbb3ee more permissive timing data;
wenzelm
parents: 64088
diff changeset
   520
    def finished(name: String): Boolean = get_default(name, _.finished, false)
10d719dbb3ee more permissive timing data;
wenzelm
parents: 64088
diff changeset
   521
    def timing(name: String): Timing = get_default(name, _.timing, Timing.zero)
10d719dbb3ee more permissive timing data;
wenzelm
parents: 64088
diff changeset
   522
    def ml_timing(name: String): Timing = get_default(name, _.ml_timing, Timing.zero)
65052
7f825cc6debf more operations;
wenzelm
parents: 64341
diff changeset
   523
    def ml_statistics(name: String): ML_Statistics =
7f825cc6debf more operations;
wenzelm
parents: 64341
diff changeset
   524
      get_default(name, entry => ML_Statistics(name, entry.ml_statistics), ML_Statistics.empty)
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   525
  }
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   526
65646
014dbbe5331f parse ml_statistics only when required;
wenzelm
parents: 65645
diff changeset
   527
  private def parse_build_info(log_file: Log_File, parse_ml_statistics: Boolean): Build_Info =
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   528
  {
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   529
    object Chapter_Name
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   530
    {
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   531
      def unapply(s: String): Some[(String, String)] =
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   532
        space_explode('/', s) match {
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   533
          case List(chapter, name) => Some((chapter, name))
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   534
          case _ => Some(("", s))
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   535
        }
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   536
    }
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   537
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   538
    val Session_No_Groups = new Regex("""^Session (\S+)$""")
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   539
    val Session_Groups = new Regex("""^Session (\S+) \((.*)\)$""")
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   540
    val Session_Finished1 =
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   541
      new Regex("""^Finished (\S+) \((\d+):(\d+):(\d+) elapsed time, (\d+):(\d+):(\d+) cpu time.*$""")
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   542
    val Session_Finished2 =
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   543
      new Regex("""^Finished (\S+) \((\d+):(\d+):(\d+) elapsed time.*$""")
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   544
    val Session_Timing =
65679
45632d594bdb more general pattern;
wenzelm
parents: 65674
diff changeset
   545
      new Regex("""^Timing (\S+) \((\d+) threads, (\d+\.\d+)s elapsed time, (\d+\.\d+)s cpu time, (\d+\.\d+)s GC time.*$""")
64086
ac7ae5067783 clarified status: started sessions may bomb without explicit FAILED or CANCELLED (cf. in afp-test-devel-2016-01-03.log);
wenzelm
parents: 64085
diff changeset
   546
    val Session_Started = new Regex("""^(?:Running|Building) (\S+) \.\.\.$""")
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   547
    val Session_Failed = new Regex("""^(\S+) FAILED""")
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   548
    val Session_Cancelled = new Regex("""^(\S+) CANCELLED""")
64120
6c5039016321 record heap sizes;
wenzelm
parents: 64119
diff changeset
   549
    val Heap = new Regex("""^Heap (\S+) \((\d+) bytes\)$""")
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   550
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   551
    var chapter = Map.empty[String, String]
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   552
    var groups = Map.empty[String, List[String]]
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   553
    var threads = Map.empty[String, Int]
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   554
    var timing = Map.empty[String, Timing]
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   555
    var ml_timing = Map.empty[String, Timing]
64086
ac7ae5067783 clarified status: started sessions may bomb without explicit FAILED or CANCELLED (cf. in afp-test-devel-2016-01-03.log);
wenzelm
parents: 64085
diff changeset
   556
    var started = Set.empty[String]
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   557
    var failed = Set.empty[String]
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   558
    var cancelled = Set.empty[String]
65627
bb185e442c95 put bulky data last;
wenzelm
parents: 65626
diff changeset
   559
    var heap_sizes = Map.empty[String, Long]
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   560
    var ml_statistics = Map.empty[String, List[Properties.T]]
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   561
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   562
    def all_sessions: Set[String] =
64120
6c5039016321 record heap sizes;
wenzelm
parents: 64119
diff changeset
   563
      chapter.keySet ++ groups.keySet ++ threads.keySet ++ timing.keySet ++ ml_timing.keySet ++
65627
bb185e442c95 put bulky data last;
wenzelm
parents: 65626
diff changeset
   564
      failed ++ cancelled ++ started ++ heap_sizes.keySet ++ ml_statistics.keySet
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   565
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   566
64062
a7352cbde7d7 misc tuning and clarification;
wenzelm
parents: 64061
diff changeset
   567
    for (line <- log_file.lines) {
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   568
      line match {
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   569
        case Session_No_Groups(Chapter_Name(chapt, name)) =>
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   570
          chapter += (name -> chapt)
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   571
          groups += (name -> Nil)
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   572
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   573
        case Session_Groups(Chapter_Name(chapt, name), grps) =>
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   574
          chapter += (name -> chapt)
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   575
          groups += (name -> Word.explode(grps))
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   576
64086
ac7ae5067783 clarified status: started sessions may bomb without explicit FAILED or CANCELLED (cf. in afp-test-devel-2016-01-03.log);
wenzelm
parents: 64085
diff changeset
   577
        case Session_Started(name) =>
ac7ae5067783 clarified status: started sessions may bomb without explicit FAILED or CANCELLED (cf. in afp-test-devel-2016-01-03.log);
wenzelm
parents: 64085
diff changeset
   578
          started += name
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   579
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   580
        case Session_Finished1(name,
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   581
            Value.Int(e1), Value.Int(e2), Value.Int(e3),
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   582
            Value.Int(c1), Value.Int(c2), Value.Int(c3)) =>
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   583
          val elapsed = Time.hms(e1, e2, e3)
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   584
          val cpu = Time.hms(c1, c2, c3)
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   585
          timing += (name -> Timing(elapsed, cpu, Time.zero))
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   586
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   587
        case Session_Finished2(name,
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   588
            Value.Int(e1), Value.Int(e2), Value.Int(e3)) =>
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   589
          val elapsed = Time.hms(e1, e2, e3)
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   590
          timing += (name -> Timing(elapsed, Time.zero, Time.zero))
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   591
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   592
        case Session_Timing(name,
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   593
            Value.Int(t), Value.Double(e), Value.Double(c), Value.Double(g)) =>
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   594
          val elapsed = Time.seconds(e)
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   595
          val cpu = Time.seconds(c)
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   596
          val gc = Time.seconds(g)
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   597
          ml_timing += (name -> Timing(elapsed, cpu, gc))
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   598
          threads += (name -> t)
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   599
64120
6c5039016321 record heap sizes;
wenzelm
parents: 64119
diff changeset
   600
        case Heap(name, Value.Long(size)) =>
6c5039016321 record heap sizes;
wenzelm
parents: 64119
diff changeset
   601
          heap_sizes += (name -> size)
6c5039016321 record heap sizes;
wenzelm
parents: 64119
diff changeset
   602
65646
014dbbe5331f parse ml_statistics only when required;
wenzelm
parents: 65645
diff changeset
   603
        case _
014dbbe5331f parse ml_statistics only when required;
wenzelm
parents: 65645
diff changeset
   604
        if parse_ml_statistics && line.startsWith(ML_STATISTICS_MARKER) && YXML.detect(line) =>
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   605
          val (name, props) =
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   606
            Library.try_unprefix(ML_STATISTICS_MARKER, line).map(log_file.parse_props(_)) match {
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   607
              case Some((SESSION_NAME, session_name) :: props) => (session_name, props)
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   608
              case _ => log_file.err("malformed ML_statistics " + quote(line))
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   609
            }
64120
6c5039016321 record heap sizes;
wenzelm
parents: 64119
diff changeset
   610
          ml_statistics += (name -> (props :: ml_statistics.getOrElse(name, Nil)))
64119
8094eaa38d4b inline session ML statistics into main build log;
wenzelm
parents: 64117
diff changeset
   611
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   612
        case _ =>
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   613
      }
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   614
    }
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   615
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   616
    val sessions =
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   617
      Map(
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   618
        (for (name <- all_sessions.toList) yield {
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   619
          val status =
65633
wenzelm
parents: 65632
diff changeset
   620
            if (failed(name)) Session_Status.failed
wenzelm
parents: 65632
diff changeset
   621
            else if (cancelled(name)) Session_Status.cancelled
64086
ac7ae5067783 clarified status: started sessions may bomb without explicit FAILED or CANCELLED (cf. in afp-test-devel-2016-01-03.log);
wenzelm
parents: 64085
diff changeset
   622
            else if (timing.isDefinedAt(name) || ml_timing.isDefinedAt(name))
65633
wenzelm
parents: 65632
diff changeset
   623
              Session_Status.finished
wenzelm
parents: 65632
diff changeset
   624
            else if (started(name)) Session_Status.failed
wenzelm
parents: 65632
diff changeset
   625
            else Session_Status.existing
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   626
          val entry =
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   627
            Session_Entry(
65643
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   628
              chapter = chapter.getOrElse(name, ""),
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   629
              groups = groups.getOrElse(name, Nil),
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   630
              threads = threads.get(name),
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   631
              timing = timing.getOrElse(name, Timing.zero),
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   632
              ml_timing = ml_timing.getOrElse(name, Timing.zero),
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   633
              heap_size = heap_sizes.get(name),
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   634
              status = Some(status),
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   635
              ml_statistics = ml_statistics.getOrElse(name, Nil).reverse)
64085
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   636
          (name -> entry)
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   637
        }):_*)
1c451e5c145f clarified parse_build_info: isabelle build output;
wenzelm
parents: 64083
diff changeset
   638
    Build_Info(sessions)
64054
1fc9ab31720d clarified modules;
wenzelm
parents: 64053
diff changeset
   639
  }
64099
wenzelm
parents: 64098
diff changeset
   640
wenzelm
parents: 64098
diff changeset
   641
wenzelm
parents: 64098
diff changeset
   642
65276
fa1a5efee2ec tuned comments;
wenzelm
parents: 65052
diff changeset
   643
  /** session info: produced by isabelle build as session log.gz file **/
64099
wenzelm
parents: 64098
diff changeset
   644
wenzelm
parents: 64098
diff changeset
   645
  sealed case class Session_Info(
wenzelm
parents: 64098
diff changeset
   646
    session_timing: Properties.T,
wenzelm
parents: 64098
diff changeset
   647
    command_timings: List[Properties.T],
wenzelm
parents: 64098
diff changeset
   648
    ml_statistics: List[Properties.T],
wenzelm
parents: 64098
diff changeset
   649
    task_statistics: List[Properties.T])
wenzelm
parents: 64098
diff changeset
   650
wenzelm
parents: 64098
diff changeset
   651
  private def parse_session_info(
wenzelm
parents: 64098
diff changeset
   652
    log_file: Log_File,
wenzelm
parents: 64098
diff changeset
   653
    command_timings: Boolean,
wenzelm
parents: 64098
diff changeset
   654
    ml_statistics: Boolean,
wenzelm
parents: 64098
diff changeset
   655
    task_statistics: Boolean): Session_Info =
wenzelm
parents: 64098
diff changeset
   656
  {
65290
wenzelm
parents: 65276
diff changeset
   657
    Session_Info(
wenzelm
parents: 65276
diff changeset
   658
      session_timing = log_file.find_props("\fTiming = ") getOrElse Nil,
wenzelm
parents: 65276
diff changeset
   659
      command_timings = if (command_timings) log_file.filter_props("\fcommand_timing = ") else Nil,
wenzelm
parents: 65276
diff changeset
   660
      ml_statistics = if (ml_statistics) log_file.filter_props(ML_STATISTICS_MARKER) else Nil,
wenzelm
parents: 65276
diff changeset
   661
      task_statistics = if (task_statistics) log_file.filter_props("\ftask_statistics = ") else Nil)
64099
wenzelm
parents: 64098
diff changeset
   662
  }
65595
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   663
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   664
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   665
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   666
  /** persistent store **/
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   667
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   668
  def store(options: Options): Store = new Store(options)
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   669
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   670
  class Store private[Build_Log](options: Options) extends Properties.Store
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   671
  {
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   672
    def open_database(
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   673
      user: String = options.string("build_log_database_user"),
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   674
      password: String = options.string("build_log_database_password"),
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   675
      database: String = options.string("build_log_database_name"),
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   676
      host: String = options.string("build_log_database_host"),
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   677
      port: Int = options.int("build_log_database_port"),
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   678
      ssh_host: String = options.string("build_log_ssh_host"),
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   679
      ssh_user: String = options.string("build_log_ssh_user"),
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   680
      ssh_port: Int = options.int("build_log_ssh_port")): PostgreSQL.Database =
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   681
    {
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   682
      PostgreSQL.open_database(
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   683
        user = user, password = password, database = database, host = host, port = port,
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   684
        ssh =
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   685
          if (ssh_host == "") None
65636
df804cdba5f9 ssh_close for proper termination after use of database;
wenzelm
parents: 65633
diff changeset
   686
          else Some(SSH.init_context(options).open_session(ssh_host, ssh_user, port)),
df804cdba5f9 ssh_close for proper termination after use of database;
wenzelm
parents: 65633
diff changeset
   687
        ssh_close = true)
65595
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   688
    }
65599
08dfa79866ec database storage of Meta_Info and Build_Info;
wenzelm
parents: 65595
diff changeset
   689
65688
wenzelm
parents: 65685
diff changeset
   690
    def domain(db: SQL.Database, table: SQL.Table, column: SQL.Column): Set[String] =
wenzelm
parents: 65685
diff changeset
   691
      using(db.select(table, List(column), distinct = true))(stmt =>
wenzelm
parents: 65685
diff changeset
   692
        SQL.iterator(stmt.executeQuery)(db.string(_, column)).toSet)
wenzelm
parents: 65685
diff changeset
   693
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   694
    def update_meta_info(db: SQL.Database, log_file: Log_File)
65632
218dbe4fb484 tuned signature;
wenzelm
parents: 65631
diff changeset
   695
    {
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   696
      val meta_info = log_file.parse_meta_info()
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   697
      val table = Meta_Info.table
65614
325801edb37d clarified transaction boundaries: more robust incremental write operations;
wenzelm
parents: 65613
diff changeset
   698
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   699
      db.transaction {
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   700
        using(db.delete(table, Meta_Info.log_name.sql_where_equal(log_file.name)))(_.execute)
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   701
        using(db.insert(table))(stmt =>
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   702
        {
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   703
          db.set_string(stmt, 1, log_file.name)
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   704
          for ((c, i) <- table.columns.tail.zipWithIndex) {
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   705
            if (c.T == SQL.Type.Date)
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   706
              db.set_date(stmt, i + 2, meta_info.get_date(c))
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   707
            else
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   708
              db.set_string(stmt, i + 2, meta_info.get(c))
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   709
          }
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   710
          stmt.execute()
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   711
        })
65614
325801edb37d clarified transaction boundaries: more robust incremental write operations;
wenzelm
parents: 65613
diff changeset
   712
      }
65600
138ffa41dc54 clarified filter_files: sorted and unique;
wenzelm
parents: 65599
diff changeset
   713
    }
138ffa41dc54 clarified filter_files: sorted and unique;
wenzelm
parents: 65599
diff changeset
   714
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   715
    def update_sessions(db: SQL.Database, log_file: Log_File)
65599
08dfa79866ec database storage of Meta_Info and Build_Info;
wenzelm
parents: 65595
diff changeset
   716
    {
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   717
      val build_info = log_file.parse_build_info()
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   718
      val table = Build_Info.sessions_table
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   719
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   720
      db.transaction {
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   721
        using(db.delete(table, Meta_Info.log_name.sql_where_equal(log_file.name)))(_.execute)
65643
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   722
        using(db.insert(table))(stmt =>
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   723
        {
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   724
          val entries_iterator =
65643
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   725
            if (build_info.sessions.isEmpty) Iterator("" -> Session_Entry.empty)
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   726
            else build_info.sessions.iterator
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   727
          for ((session_name, session) <- entries_iterator) {
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   728
            db.set_string(stmt, 1, log_file.name)
65643
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   729
            db.set_string(stmt, 2, session_name)
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   730
            db.set_string(stmt, 3, session.proper_chapter)
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   731
            db.set_string(stmt, 4, session.proper_groups)
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   732
            db.set_int(stmt, 5, session.threads)
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   733
            db.set_long(stmt, 6, session.timing.elapsed.proper_ms)
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   734
            db.set_long(stmt, 7, session.timing.cpu.proper_ms)
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   735
            db.set_long(stmt, 8, session.timing.gc.proper_ms)
65683
70b0ef74ef3a some derived data fields, to facilitate queries;
wenzelm
parents: 65682
diff changeset
   736
            db.set_double(stmt, 9, session.timing.factor)
70b0ef74ef3a some derived data fields, to facilitate queries;
wenzelm
parents: 65682
diff changeset
   737
            db.set_long(stmt, 10, session.ml_timing.elapsed.proper_ms)
70b0ef74ef3a some derived data fields, to facilitate queries;
wenzelm
parents: 65682
diff changeset
   738
            db.set_long(stmt, 11, session.ml_timing.cpu.proper_ms)
70b0ef74ef3a some derived data fields, to facilitate queries;
wenzelm
parents: 65682
diff changeset
   739
            db.set_long(stmt, 12, session.ml_timing.gc.proper_ms)
70b0ef74ef3a some derived data fields, to facilitate queries;
wenzelm
parents: 65682
diff changeset
   740
            db.set_double(stmt, 13, session.ml_timing.factor)
70b0ef74ef3a some derived data fields, to facilitate queries;
wenzelm
parents: 65682
diff changeset
   741
            db.set_long(stmt, 14, session.heap_size)
70b0ef74ef3a some derived data fields, to facilitate queries;
wenzelm
parents: 65682
diff changeset
   742
            db.set_string(stmt, 15, session.status.map(_.toString))
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   743
            stmt.execute()
65643
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   744
          }
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   745
        })
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   746
      }
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   747
    }
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   748
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   749
    def update_ml_statistics(db: SQL.Database, log_file: Log_File)
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   750
    {
65646
014dbbe5331f parse ml_statistics only when required;
wenzelm
parents: 65645
diff changeset
   751
      val build_info = log_file.parse_build_info(ml_statistics = true)
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   752
      val table = Build_Info.ml_statistics_table
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   753
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   754
      db.transaction {
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   755
        using(db.delete(table, Meta_Info.log_name.sql_where_equal(log_file.name)))(_.execute)
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   756
        using(db.insert(table))(stmt =>
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   757
        {
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   758
          val ml_stats: List[(String, Option[Bytes])] =
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   759
            Par_List.map[(String, Session_Entry), (String, Option[Bytes])](
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   760
              { case (a, b) => (a, compress_properties(b.ml_statistics).proper) },
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   761
              build_info.sessions.iterator.filter(p => p._2.ml_statistics.nonEmpty).toList)
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   762
          val entries = if (ml_stats.nonEmpty) ml_stats else List("" -> None)
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   763
          for ((session_name, ml_statistics) <- entries) {
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   764
            db.set_string(stmt, 1, log_file.name)
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   765
            db.set_string(stmt, 2, session_name)
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   766
            db.set_bytes(stmt, 3, ml_statistics)
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   767
            stmt.execute()
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   768
          }
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   769
        })
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   770
      }
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   771
    }
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   772
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   773
    def write_info(db: SQL.Database, files: List[JFile], ml_statistics: Boolean = false)
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   774
    {
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   775
      class Table_Status(table: SQL.Table, update_db: (SQL.Database, Log_File) => Unit)
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   776
      {
65688
wenzelm
parents: 65685
diff changeset
   777
        db.create_table(table)
wenzelm
parents: 65685
diff changeset
   778
        private var known: Set[String] = domain(db, table, Meta_Info.log_name)
wenzelm
parents: 65685
diff changeset
   779
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   780
        def required(file: JFile): Boolean = !known(Log_File.plain_name(file.getName))
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   781
        def update(log_file: Log_File)
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   782
        {
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   783
          if (!known(log_file.name)) {
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   784
            update_db(db, log_file)
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   785
            known += log_file.name
65618
986fac3c60b4 tuned transaction granularity;
wenzelm
parents: 65616
diff changeset
   786
          }
65614
325801edb37d clarified transaction boundaries: more robust incremental write operations;
wenzelm
parents: 65613
diff changeset
   787
        }
65605
a6447eb6bc38 separate small meta_info vs. big build_info;
wenzelm
parents: 65603
diff changeset
   788
      }
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   789
      val status =
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   790
        List(
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   791
          new Table_Status(Meta_Info.table, update_meta_info _),
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   792
          new Table_Status(Build_Info.sessions_table, update_sessions _),
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   793
          new Table_Status(Build_Info.ml_statistics_table,
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   794
            if (ml_statistics) update_ml_statistics _
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   795
            else (_: SQL.Database, _: Log_File) => ()))
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   796
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   797
      for (file_group <- files.filter(file => status.exists(_.required(file))).grouped(100)) {
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   798
        val log_files = Par_List.map[JFile, Log_File](Log_File.apply _, file_group)
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   799
        db.transaction { log_files.foreach(log_file => status.foreach(_.update(log_file))) }
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   800
      }
65605
a6447eb6bc38 separate small meta_info vs. big build_info;
wenzelm
parents: 65603
diff changeset
   801
    }
a6447eb6bc38 separate small meta_info vs. big build_info;
wenzelm
parents: 65603
diff changeset
   802
65621
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   803
    def read_meta_info(db: SQL.Database, log_name: String): Option[Meta_Info] =
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   804
    {
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   805
      val table = Meta_Info.table
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   806
      val columns = table.columns.tail
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   807
      using(db.select(table, columns, Meta_Info.log_name.sql_where_equal(log_name)))(stmt =>
65621
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   808
      {
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   809
        val rs = stmt.executeQuery
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   810
        if (!rs.next) None
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   811
        else {
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   812
          val results =
65642
1423cbbc542d clarified database update operations;
wenzelm
parents: 65639
diff changeset
   813
            columns.map(c => c.name ->
65621
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   814
              (if (c.T == SQL.Type.Date)
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   815
                db.get(rs, c, db.date _).map(Log_File.Date_Format(_))
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   816
               else
65624
32fa61f694ef clarified multi-line properties;
wenzelm
parents: 65623
diff changeset
   817
                db.get(rs, c, db.string _)))
65621
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   818
          val n = Prop.all_props.length
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   819
          val props = for ((x, Some(y)) <- results.take(n)) yield (x, y)
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   820
          val settings = for ((x, Some(y)) <- results.drop(n)) yield (x, y)
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   821
          Some(Meta_Info(props, settings))
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   822
        }
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   823
      })
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   824
    }
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   825
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   826
    def read_build_info(
65629
e6c0afe672fa optional ml_statistics: much faster;
wenzelm
parents: 65628
diff changeset
   827
      db: SQL.Database,
e6c0afe672fa optional ml_statistics: much faster;
wenzelm
parents: 65628
diff changeset
   828
      log_name: String,
e6c0afe672fa optional ml_statistics: much faster;
wenzelm
parents: 65628
diff changeset
   829
      session_names: List[String] = Nil,
e6c0afe672fa optional ml_statistics: much faster;
wenzelm
parents: 65628
diff changeset
   830
      ml_statistics: Boolean = false): Build_Info =
65621
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   831
    {
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   832
      val table1 = Build_Info.sessions_table
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   833
      val table2 = Build_Info.ml_statistics_table
65629
e6c0afe672fa optional ml_statistics: much faster;
wenzelm
parents: 65628
diff changeset
   834
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   835
      val where_log_name =
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   836
        Meta_Info.log_name(table1).sql_where_equal(log_name) + " AND " +
65649
0818da4f67bb tuned signature;
wenzelm
parents: 65648
diff changeset
   837
          Build_Info.session_name(table1).sql + " <> ''"
65621
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   838
      val where =
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   839
        if (session_names.isEmpty) where_log_name
65621
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   840
        else
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   841
          where_log_name + " AND " +
65649
0818da4f67bb tuned signature;
wenzelm
parents: 65648
diff changeset
   842
          session_names.map(a => Build_Info.session_name(table1).sql + " = " + SQL.string(a)).
65621
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   843
            mkString("(", " OR ", ")")
65629
e6c0afe672fa optional ml_statistics: much faster;
wenzelm
parents: 65628
diff changeset
   844
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   845
      val columns1 = table1.columns.tail.map(_.apply(table1))
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   846
      val (columns, from) =
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   847
        if (ml_statistics) {
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   848
          val columns = columns1 ::: List(Build_Info.ml_statistics(table2))
65668
366bc4e6a238 more operations;
wenzelm
parents: 65665
diff changeset
   849
          val join =
366bc4e6a238 more operations;
wenzelm
parents: 65665
diff changeset
   850
            SQL.join_outer(table1, table2,
65649
0818da4f67bb tuned signature;
wenzelm
parents: 65648
diff changeset
   851
              Meta_Info.log_name(table1).sql + " = " +
0818da4f67bb tuned signature;
wenzelm
parents: 65648
diff changeset
   852
              Meta_Info.log_name(table2).sql + " AND " +
0818da4f67bb tuned signature;
wenzelm
parents: 65648
diff changeset
   853
              Build_Info.session_name(table1).sql + " = " +
65668
366bc4e6a238 more operations;
wenzelm
parents: 65665
diff changeset
   854
              Build_Info.session_name(table2).sql)
366bc4e6a238 more operations;
wenzelm
parents: 65665
diff changeset
   855
          (columns, SQL.enclose(join))
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   856
        }
65649
0818da4f67bb tuned signature;
wenzelm
parents: 65648
diff changeset
   857
        else (columns1, table1.sql)
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   858
65621
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   859
      val sessions =
65645
2c704ae04db1 clarified database layout: bulky ml_statistics are stored/retrieved separately;
wenzelm
parents: 65643
diff changeset
   860
        using(db.statement(SQL.select(columns) + from + " " + where))(stmt =>
65621
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   861
        {
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   862
          SQL.iterator(stmt.executeQuery)(rs =>
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   863
          {
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   864
            val session_name = db.string(rs, Build_Info.session_name)
65626
wenzelm
parents: 65625
diff changeset
   865
            val session_entry =
wenzelm
parents: 65625
diff changeset
   866
              Session_Entry(
wenzelm
parents: 65625
diff changeset
   867
                chapter = db.string(rs, Build_Info.chapter),
wenzelm
parents: 65625
diff changeset
   868
                groups = split_lines(db.string(rs, Build_Info.groups)),
wenzelm
parents: 65625
diff changeset
   869
                threads = db.get(rs, Build_Info.threads, db.int _),
wenzelm
parents: 65625
diff changeset
   870
                timing =
wenzelm
parents: 65625
diff changeset
   871
                  Timing(Time.ms(db.long(rs, Build_Info.timing_elapsed)),
wenzelm
parents: 65625
diff changeset
   872
                    Time.ms(db.long(rs, Build_Info.timing_cpu)),
wenzelm
parents: 65625
diff changeset
   873
                    Time.ms(db.long(rs, Build_Info.timing_gc))),
wenzelm
parents: 65625
diff changeset
   874
                ml_timing =
wenzelm
parents: 65625
diff changeset
   875
                  Timing(Time.ms(db.long(rs, Build_Info.ml_timing_elapsed)),
wenzelm
parents: 65625
diff changeset
   876
                    Time.ms(db.long(rs, Build_Info.ml_timing_cpu)),
wenzelm
parents: 65625
diff changeset
   877
                    Time.ms(db.long(rs, Build_Info.ml_timing_gc))),
wenzelm
parents: 65625
diff changeset
   878
                heap_size = db.get(rs, Build_Info.heap_size, db.long _),
65643
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   879
                status =
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   880
                  db.get(rs, Build_Info.status, db.string _).
a54371226182 clarified dummy Session_Entry;
wenzelm
parents: 65642
diff changeset
   881
                    map(Session_Status.withName(_)),
65629
e6c0afe672fa optional ml_statistics: much faster;
wenzelm
parents: 65628
diff changeset
   882
                ml_statistics =
e6c0afe672fa optional ml_statistics: much faster;
wenzelm
parents: 65628
diff changeset
   883
                  if (ml_statistics) uncompress_properties(db.bytes(rs, Build_Info.ml_statistics))
e6c0afe672fa optional ml_statistics: much faster;
wenzelm
parents: 65628
diff changeset
   884
                  else Nil)
65626
wenzelm
parents: 65625
diff changeset
   885
            session_name -> session_entry
65621
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   886
          }).toMap
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   887
        })
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   888
      Build_Info(sessions)
551950dccec6 operations to read database;
wenzelm
parents: 65619
diff changeset
   889
    }
65595
ffd8283b7be0 support for database connection;
wenzelm
parents: 65591
diff changeset
   890
  }
65670
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   891
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   892
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   893
  /* full view on build_log data */
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   894
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   895
  // WARNING: This may cause performance problems, e.g. with sqlitebrowser
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   896
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   897
  val full_view: SQL.View =
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   898
    SQL.View("isabelle_build_log",
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   899
      Meta_Info.table.columns :::
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   900
        Build_Info.sessions_table.columns.tail.map(_.copy(primary_key = false)))
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   901
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   902
  def create_full_view(db: SQL.Database)
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   903
  {
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   904
    if (!db.tables.contains(full_view.name)) {
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   905
      val table1 = Meta_Info.table
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   906
      val table2 = Build_Info.sessions_table
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   907
      db.create_view(full_view,
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   908
        SQL.select(Meta_Info.log_name(table1) :: full_view.columns.tail) +
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   909
        SQL.join(table1, table2,
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   910
          Meta_Info.log_name(table1).sql + " = " + Meta_Info.log_name(table2).sql))
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   911
    }
490649872acc create full view on build_log data, but avoid bombing sqlitebrowser;
wenzelm
parents: 65668
diff changeset
   912
  }
65682
3722be87305c tuned signature;
wenzelm
parents: 65679
diff changeset
   913
3722be87305c tuned signature;
wenzelm
parents: 65679
diff changeset
   914
3722be87305c tuned signature;
wenzelm
parents: 65679
diff changeset
   915
  /* main operations */
3722be87305c tuned signature;
wenzelm
parents: 65679
diff changeset
   916
3722be87305c tuned signature;
wenzelm
parents: 65679
diff changeset
   917
  def database_update(store: Store, db: SQL.Database, dirs: List[Path],
3722be87305c tuned signature;
wenzelm
parents: 65679
diff changeset
   918
    ml_statistics: Boolean = false, full_view: Boolean = false)
3722be87305c tuned signature;
wenzelm
parents: 65679
diff changeset
   919
  {
3722be87305c tuned signature;
wenzelm
parents: 65679
diff changeset
   920
    val files = Log_File.find_files(dirs)
3722be87305c tuned signature;
wenzelm
parents: 65679
diff changeset
   921
    store.write_info(db, files, ml_statistics = ml_statistics)
3722be87305c tuned signature;
wenzelm
parents: 65679
diff changeset
   922
    if (full_view) Build_Log.create_full_view(db)
3722be87305c tuned signature;
wenzelm
parents: 65679
diff changeset
   923
  }
64045
c6160d0b0337 clarified modules;
wenzelm
parents:
diff changeset
   924
}