src/Pure/Admin/build_log.scala
changeset 65937 fde7b5d209d5
parent 65934 5f202ba9f590
child 66046 37226f74f33a
equal deleted inserted replaced
65936:aece72468de5 65937:fde7b5d209d5
   442 
   442 
   443 
   443 
   444   /** build info: toplevel output of isabelle build or Admin/build_history **/
   444   /** build info: toplevel output of isabelle build or Admin/build_history **/
   445 
   445 
   446   val ML_STATISTICS_MARKER = "\fML_statistics = "
   446   val ML_STATISTICS_MARKER = "\fML_statistics = "
       
   447   val ERROR_MESSAGE_MARKER = "\ferror_message = "
   447   val SESSION_NAME = "session_name"
   448   val SESSION_NAME = "session_name"
   448 
   449 
   449   object Session_Status extends Enumeration
   450   object Session_Status extends Enumeration
   450   {
   451   {
   451     val existing, finished, failed, cancelled = Value
   452     val existing, finished, failed, cancelled = Value
   462     threads: Option[Int] = None,
   463     threads: Option[Int] = None,
   463     timing: Timing = Timing.zero,
   464     timing: Timing = Timing.zero,
   464     ml_timing: Timing = Timing.zero,
   465     ml_timing: Timing = Timing.zero,
   465     heap_size: Option[Long] = None,
   466     heap_size: Option[Long] = None,
   466     status: Option[Session_Status.Value] = None,
   467     status: Option[Session_Status.Value] = None,
       
   468     errors: List[String] = Nil,
   467     ml_statistics: List[Properties.T] = Nil)
   469     ml_statistics: List[Properties.T] = Nil)
   468   {
   470   {
   469     def proper_groups: Option[String] = if (groups.isEmpty) None else Some(cat_lines(groups))
   471     def proper_groups: Option[String] = if (groups.isEmpty) None else Some(cat_lines(groups))
   470     def finished: Boolean = status == Some(Session_Status.finished)
   472     def finished: Boolean = status == Some(Session_Status.finished)
       
   473     def failed: Boolean = status == Some(Session_Status.failed)
   471   }
   474   }
   472 
   475 
   473   sealed case class Build_Info(sessions: Map[String, Session_Entry])
   476   sealed case class Build_Info(sessions: Map[String, Session_Entry])
   474   {
   477   {
   475     def finished_sessions: List[String] =
   478     def finished_sessions: List[String] = for ((a, b) <- sessions.toList if b.finished) yield a
   476       for ((name, entry) <- sessions.toList if entry.finished) yield name
   479     def failed_sessions: List[String] = for ((a, b) <- sessions.toList if b.failed) yield a
   477   }
   480   }
   478 
   481 
   479   private def parse_build_info(log_file: Log_File, parse_ml_statistics: Boolean): Build_Info =
   482   private def parse_build_info(log_file: Log_File, parse_ml_statistics: Boolean): Build_Info =
   480   {
   483   {
   481     object Chapter_Name
   484     object Chapter_Name
   508     var started = Set.empty[String]
   511     var started = Set.empty[String]
   509     var failed = Set.empty[String]
   512     var failed = Set.empty[String]
   510     var cancelled = Set.empty[String]
   513     var cancelled = Set.empty[String]
   511     var heap_sizes = Map.empty[String, Long]
   514     var heap_sizes = Map.empty[String, Long]
   512     var ml_statistics = Map.empty[String, List[Properties.T]]
   515     var ml_statistics = Map.empty[String, List[Properties.T]]
       
   516     var errors = Map.empty[String, List[String]]
   513 
   517 
   514     def all_sessions: Set[String] =
   518     def all_sessions: Set[String] =
   515       chapter.keySet ++ groups.keySet ++ threads.keySet ++ timing.keySet ++ ml_timing.keySet ++
   519       chapter.keySet ++ groups.keySet ++ threads.keySet ++ timing.keySet ++ ml_timing.keySet ++
   516       failed ++ cancelled ++ started ++ heap_sizes.keySet ++ ml_statistics.keySet
   520       failed ++ cancelled ++ started ++ heap_sizes.keySet ++ ml_statistics.keySet
   517 
   521 
   550           threads += (name -> t)
   554           threads += (name -> t)
   551 
   555 
   552         case Heap(name, Value.Long(size)) =>
   556         case Heap(name, Value.Long(size)) =>
   553           heap_sizes += (name -> size)
   557           heap_sizes += (name -> size)
   554 
   558 
   555         case _
   559         case _ if parse_ml_statistics && line.startsWith(ML_STATISTICS_MARKER) && YXML.detect(line) =>
   556         if parse_ml_statistics && line.startsWith(ML_STATISTICS_MARKER) && YXML.detect(line) =>
       
   557           val (name, props) =
   560           val (name, props) =
   558             Library.try_unprefix(ML_STATISTICS_MARKER, line).map(log_file.parse_props(_)) match {
   561             Library.try_unprefix(ML_STATISTICS_MARKER, line).map(log_file.parse_props(_)) match {
   559               case Some((SESSION_NAME, session_name) :: props) => (session_name, props)
   562               case Some((SESSION_NAME, name) :: props) => (name, props)
   560               case _ => log_file.err("malformed ML_statistics " + quote(line))
   563               case _ => log_file.err("malformed ML_statistics " + quote(line))
   561             }
   564             }
   562           ml_statistics += (name -> (props :: ml_statistics.getOrElse(name, Nil)))
   565           ml_statistics += (name -> (props :: ml_statistics.getOrElse(name, Nil)))
       
   566 
       
   567         case _ if line.startsWith(ERROR_MESSAGE_MARKER) && YXML.detect(line) =>
       
   568           val (name, msg) =
       
   569             Library.try_unprefix(ERROR_MESSAGE_MARKER, line).map(log_file.parse_props(_)) match {
       
   570               case Some(List((SESSION_NAME, name), (Markup.CONTENT, msg))) => (name, msg)
       
   571               case _ => log_file.err("malformed error message " + quote(line))
       
   572             }
       
   573           errors += (name -> (Library.decode_lines(msg) :: errors.getOrElse(name, Nil)))
   563 
   574 
   564         case _ =>
   575         case _ =>
   565       }
   576       }
   566     }
   577     }
   567 
   578 
   582               threads = threads.get(name),
   593               threads = threads.get(name),
   583               timing = timing.getOrElse(name, Timing.zero),
   594               timing = timing.getOrElse(name, Timing.zero),
   584               ml_timing = ml_timing.getOrElse(name, Timing.zero),
   595               ml_timing = ml_timing.getOrElse(name, Timing.zero),
   585               heap_size = heap_sizes.get(name),
   596               heap_size = heap_sizes.get(name),
   586               status = Some(status),
   597               status = Some(status),
       
   598               errors = errors.getOrElse(name, Nil).reverse,
   587               ml_statistics = ml_statistics.getOrElse(name, Nil).reverse)
   599               ml_statistics = ml_statistics.getOrElse(name, Nil).reverse)
   588           (name -> entry)
   600           (name -> entry)
   589         }):_*)
   601         }):_*)
   590     Build_Info(sessions)
   602     Build_Info(sessions)
   591   }
   603   }
   610     Session_Info(
   622     Session_Info(
   611       session_timing = log_file.find_props("\fTiming = ") getOrElse Nil,
   623       session_timing = log_file.find_props("\fTiming = ") getOrElse Nil,
   612       command_timings = if (command_timings) log_file.filter_props("\fcommand_timing = ") else Nil,
   624       command_timings = if (command_timings) log_file.filter_props("\fcommand_timing = ") else Nil,
   613       ml_statistics = if (ml_statistics) log_file.filter_props(ML_STATISTICS_MARKER) else Nil,
   625       ml_statistics = if (ml_statistics) log_file.filter_props(ML_STATISTICS_MARKER) else Nil,
   614       task_statistics = if (task_statistics) log_file.filter_props("\ftask_statistics = ") else Nil,
   626       task_statistics = if (task_statistics) log_file.filter_props("\ftask_statistics = ") else Nil,
   615       errors = log_file.filter_lines("\ferror_message = ").map(Library.decode_lines(_)))
   627       errors = log_file.filter_lines(ERROR_MESSAGE_MARKER).map(Library.decode_lines(_)))
   616   }
   628   }
       
   629 
       
   630   def compress_errors(errors: List[String]): Option[Bytes] =
       
   631     if (errors.isEmpty) None
       
   632     else Some(Bytes(YXML.string_of_body(XML.Encode.list(XML.Encode.string)(errors))).compress())
       
   633 
       
   634   def uncompress_errors(bytes: Bytes): List[String] =
       
   635     if (bytes.isEmpty) Nil
       
   636     else XML.Decode.list(YXML.string_of_body(_))(YXML.parse_body(bytes.uncompress().text))
   617 
   637 
   618 
   638 
   619 
   639 
   620   /** persistent store **/
   640   /** persistent store **/
   621 
   641 
   642     val ml_timing_cpu = SQL.Column.long("ml_timing_cpu")
   662     val ml_timing_cpu = SQL.Column.long("ml_timing_cpu")
   643     val ml_timing_gc = SQL.Column.long("ml_timing_gc")
   663     val ml_timing_gc = SQL.Column.long("ml_timing_gc")
   644     val ml_timing_factor = SQL.Column.double("ml_timing_factor")
   664     val ml_timing_factor = SQL.Column.double("ml_timing_factor")
   645     val heap_size = SQL.Column.long("heap_size")
   665     val heap_size = SQL.Column.long("heap_size")
   646     val status = SQL.Column.string("status")
   666     val status = SQL.Column.string("status")
       
   667     val errors = SQL.Column.bytes("errors")
   647     val ml_statistics = SQL.Column.bytes("ml_statistics")
   668     val ml_statistics = SQL.Column.bytes("ml_statistics")
   648     val known = SQL.Column.bool("known")
   669     val known = SQL.Column.bool("known")
   649 
   670 
   650     val meta_info_table =
   671     val meta_info_table =
   651       build_log_table("meta_info", log_name :: Prop.all_props ::: Settings.all_settings)
   672       build_log_table("meta_info", log_name :: Prop.all_props ::: Settings.all_settings)
   652 
   673 
   653     val sessions_table =
   674     val sessions_table =
   654       build_log_table("sessions",
   675       build_log_table("sessions",
   655         List(log_name, session_name, chapter, groups, threads, timing_elapsed, timing_cpu,
   676         List(log_name, session_name, chapter, groups, threads, timing_elapsed, timing_cpu,
   656           timing_gc, timing_factor, ml_timing_elapsed, ml_timing_cpu, ml_timing_gc, ml_timing_factor,
   677           timing_gc, timing_factor, ml_timing_elapsed, ml_timing_cpu, ml_timing_gc, ml_timing_factor,
   657           heap_size, status))
   678           heap_size, status, errors))
   658 
   679 
   659     val ml_statistics_table =
   680     val ml_statistics_table =
   660       build_log_table("ml_statistics", List(log_name, session_name, ml_statistics))
   681       build_log_table("ml_statistics", List(log_name, session_name, ml_statistics))
   661 
   682 
   662 
   683 
   878           stmt.long(11) = session.ml_timing.cpu.proper_ms
   899           stmt.long(11) = session.ml_timing.cpu.proper_ms
   879           stmt.long(12) = session.ml_timing.gc.proper_ms
   900           stmt.long(12) = session.ml_timing.gc.proper_ms
   880           stmt.double(13) = session.ml_timing.factor
   901           stmt.double(13) = session.ml_timing.factor
   881           stmt.long(14) = session.heap_size
   902           stmt.long(14) = session.heap_size
   882           stmt.string(15) = session.status.map(_.toString)
   903           stmt.string(15) = session.status.map(_.toString)
       
   904           stmt.bytes(16) = compress_errors(session.errors)
   883           stmt.execute()
   905           stmt.execute()
   884         }
   906         }
   885       })
   907       })
   886     }
   908     }
   887 
   909 
  1011                 timing = res.timing(Data.timing_elapsed, Data.timing_cpu, Data.timing_gc),
  1033                 timing = res.timing(Data.timing_elapsed, Data.timing_cpu, Data.timing_gc),
  1012                 ml_timing =
  1034                 ml_timing =
  1013                   res.timing(Data.ml_timing_elapsed, Data.ml_timing_cpu, Data.ml_timing_gc),
  1035                   res.timing(Data.ml_timing_elapsed, Data.ml_timing_cpu, Data.ml_timing_gc),
  1014                 heap_size = res.get_long(Data.heap_size),
  1036                 heap_size = res.get_long(Data.heap_size),
  1015                 status = res.get_string(Data.status).map(Session_Status.withName(_)),
  1037                 status = res.get_string(Data.status).map(Session_Status.withName(_)),
       
  1038                 errors = uncompress_errors(res.bytes(Data.errors)),
  1016                 ml_statistics =
  1039                 ml_statistics =
  1017                   if (ml_statistics)
  1040                   if (ml_statistics)
  1018                     Properties.uncompress(res.bytes(Data.ml_statistics), Some(xml_cache))
  1041                     Properties.uncompress(res.bytes(Data.ml_statistics), Some(xml_cache))
  1019                   else Nil)
  1042                   else Nil)
  1020             session_name -> session_entry
  1043             session_name -> session_entry