442 |
442 |
443 |
443 |
444 /** build info: toplevel output of isabelle build or Admin/build_history **/ |
444 /** build info: toplevel output of isabelle build or Admin/build_history **/ |
445 |
445 |
446 val ML_STATISTICS_MARKER = "\fML_statistics = " |
446 val ML_STATISTICS_MARKER = "\fML_statistics = " |
|
447 val ERROR_MESSAGE_MARKER = "\ferror_message = " |
447 val SESSION_NAME = "session_name" |
448 val SESSION_NAME = "session_name" |
448 |
449 |
449 object Session_Status extends Enumeration |
450 object Session_Status extends Enumeration |
450 { |
451 { |
451 val existing, finished, failed, cancelled = Value |
452 val existing, finished, failed, cancelled = Value |
462 threads: Option[Int] = None, |
463 threads: Option[Int] = None, |
463 timing: Timing = Timing.zero, |
464 timing: Timing = Timing.zero, |
464 ml_timing: Timing = Timing.zero, |
465 ml_timing: Timing = Timing.zero, |
465 heap_size: Option[Long] = None, |
466 heap_size: Option[Long] = None, |
466 status: Option[Session_Status.Value] = None, |
467 status: Option[Session_Status.Value] = None, |
|
468 errors: List[String] = Nil, |
467 ml_statistics: List[Properties.T] = Nil) |
469 ml_statistics: List[Properties.T] = Nil) |
468 { |
470 { |
469 def proper_groups: Option[String] = if (groups.isEmpty) None else Some(cat_lines(groups)) |
471 def proper_groups: Option[String] = if (groups.isEmpty) None else Some(cat_lines(groups)) |
470 def finished: Boolean = status == Some(Session_Status.finished) |
472 def finished: Boolean = status == Some(Session_Status.finished) |
|
473 def failed: Boolean = status == Some(Session_Status.failed) |
471 } |
474 } |
472 |
475 |
473 sealed case class Build_Info(sessions: Map[String, Session_Entry]) |
476 sealed case class Build_Info(sessions: Map[String, Session_Entry]) |
474 { |
477 { |
475 def finished_sessions: List[String] = |
478 def finished_sessions: List[String] = for ((a, b) <- sessions.toList if b.finished) yield a |
476 for ((name, entry) <- sessions.toList if entry.finished) yield name |
479 def failed_sessions: List[String] = for ((a, b) <- sessions.toList if b.failed) yield a |
477 } |
480 } |
478 |
481 |
479 private def parse_build_info(log_file: Log_File, parse_ml_statistics: Boolean): Build_Info = |
482 private def parse_build_info(log_file: Log_File, parse_ml_statistics: Boolean): Build_Info = |
480 { |
483 { |
481 object Chapter_Name |
484 object Chapter_Name |
508 var started = Set.empty[String] |
511 var started = Set.empty[String] |
509 var failed = Set.empty[String] |
512 var failed = Set.empty[String] |
510 var cancelled = Set.empty[String] |
513 var cancelled = Set.empty[String] |
511 var heap_sizes = Map.empty[String, Long] |
514 var heap_sizes = Map.empty[String, Long] |
512 var ml_statistics = Map.empty[String, List[Properties.T]] |
515 var ml_statistics = Map.empty[String, List[Properties.T]] |
|
516 var errors = Map.empty[String, List[String]] |
513 |
517 |
514 def all_sessions: Set[String] = |
518 def all_sessions: Set[String] = |
515 chapter.keySet ++ groups.keySet ++ threads.keySet ++ timing.keySet ++ ml_timing.keySet ++ |
519 chapter.keySet ++ groups.keySet ++ threads.keySet ++ timing.keySet ++ ml_timing.keySet ++ |
516 failed ++ cancelled ++ started ++ heap_sizes.keySet ++ ml_statistics.keySet |
520 failed ++ cancelled ++ started ++ heap_sizes.keySet ++ ml_statistics.keySet |
517 |
521 |
550 threads += (name -> t) |
554 threads += (name -> t) |
551 |
555 |
552 case Heap(name, Value.Long(size)) => |
556 case Heap(name, Value.Long(size)) => |
553 heap_sizes += (name -> size) |
557 heap_sizes += (name -> size) |
554 |
558 |
555 case _ |
559 case _ if parse_ml_statistics && line.startsWith(ML_STATISTICS_MARKER) && YXML.detect(line) => |
556 if parse_ml_statistics && line.startsWith(ML_STATISTICS_MARKER) && YXML.detect(line) => |
|
557 val (name, props) = |
560 val (name, props) = |
558 Library.try_unprefix(ML_STATISTICS_MARKER, line).map(log_file.parse_props(_)) match { |
561 Library.try_unprefix(ML_STATISTICS_MARKER, line).map(log_file.parse_props(_)) match { |
559 case Some((SESSION_NAME, session_name) :: props) => (session_name, props) |
562 case Some((SESSION_NAME, name) :: props) => (name, props) |
560 case _ => log_file.err("malformed ML_statistics " + quote(line)) |
563 case _ => log_file.err("malformed ML_statistics " + quote(line)) |
561 } |
564 } |
562 ml_statistics += (name -> (props :: ml_statistics.getOrElse(name, Nil))) |
565 ml_statistics += (name -> (props :: ml_statistics.getOrElse(name, Nil))) |
|
566 |
|
567 case _ if line.startsWith(ERROR_MESSAGE_MARKER) && YXML.detect(line) => |
|
568 val (name, msg) = |
|
569 Library.try_unprefix(ERROR_MESSAGE_MARKER, line).map(log_file.parse_props(_)) match { |
|
570 case Some(List((SESSION_NAME, name), (Markup.CONTENT, msg))) => (name, msg) |
|
571 case _ => log_file.err("malformed error message " + quote(line)) |
|
572 } |
|
573 errors += (name -> (Library.decode_lines(msg) :: errors.getOrElse(name, Nil))) |
563 |
574 |
564 case _ => |
575 case _ => |
565 } |
576 } |
566 } |
577 } |
567 |
578 |
582 threads = threads.get(name), |
593 threads = threads.get(name), |
583 timing = timing.getOrElse(name, Timing.zero), |
594 timing = timing.getOrElse(name, Timing.zero), |
584 ml_timing = ml_timing.getOrElse(name, Timing.zero), |
595 ml_timing = ml_timing.getOrElse(name, Timing.zero), |
585 heap_size = heap_sizes.get(name), |
596 heap_size = heap_sizes.get(name), |
586 status = Some(status), |
597 status = Some(status), |
|
598 errors = errors.getOrElse(name, Nil).reverse, |
587 ml_statistics = ml_statistics.getOrElse(name, Nil).reverse) |
599 ml_statistics = ml_statistics.getOrElse(name, Nil).reverse) |
588 (name -> entry) |
600 (name -> entry) |
589 }):_*) |
601 }):_*) |
590 Build_Info(sessions) |
602 Build_Info(sessions) |
591 } |
603 } |
610 Session_Info( |
622 Session_Info( |
611 session_timing = log_file.find_props("\fTiming = ") getOrElse Nil, |
623 session_timing = log_file.find_props("\fTiming = ") getOrElse Nil, |
612 command_timings = if (command_timings) log_file.filter_props("\fcommand_timing = ") else Nil, |
624 command_timings = if (command_timings) log_file.filter_props("\fcommand_timing = ") else Nil, |
613 ml_statistics = if (ml_statistics) log_file.filter_props(ML_STATISTICS_MARKER) else Nil, |
625 ml_statistics = if (ml_statistics) log_file.filter_props(ML_STATISTICS_MARKER) else Nil, |
614 task_statistics = if (task_statistics) log_file.filter_props("\ftask_statistics = ") else Nil, |
626 task_statistics = if (task_statistics) log_file.filter_props("\ftask_statistics = ") else Nil, |
615 errors = log_file.filter_lines("\ferror_message = ").map(Library.decode_lines(_))) |
627 errors = log_file.filter_lines(ERROR_MESSAGE_MARKER).map(Library.decode_lines(_))) |
616 } |
628 } |
|
629 |
|
630 def compress_errors(errors: List[String]): Option[Bytes] = |
|
631 if (errors.isEmpty) None |
|
632 else Some(Bytes(YXML.string_of_body(XML.Encode.list(XML.Encode.string)(errors))).compress()) |
|
633 |
|
634 def uncompress_errors(bytes: Bytes): List[String] = |
|
635 if (bytes.isEmpty) Nil |
|
636 else XML.Decode.list(YXML.string_of_body(_))(YXML.parse_body(bytes.uncompress().text)) |
617 |
637 |
618 |
638 |
619 |
639 |
620 /** persistent store **/ |
640 /** persistent store **/ |
621 |
641 |
642 val ml_timing_cpu = SQL.Column.long("ml_timing_cpu") |
662 val ml_timing_cpu = SQL.Column.long("ml_timing_cpu") |
643 val ml_timing_gc = SQL.Column.long("ml_timing_gc") |
663 val ml_timing_gc = SQL.Column.long("ml_timing_gc") |
644 val ml_timing_factor = SQL.Column.double("ml_timing_factor") |
664 val ml_timing_factor = SQL.Column.double("ml_timing_factor") |
645 val heap_size = SQL.Column.long("heap_size") |
665 val heap_size = SQL.Column.long("heap_size") |
646 val status = SQL.Column.string("status") |
666 val status = SQL.Column.string("status") |
|
667 val errors = SQL.Column.bytes("errors") |
647 val ml_statistics = SQL.Column.bytes("ml_statistics") |
668 val ml_statistics = SQL.Column.bytes("ml_statistics") |
648 val known = SQL.Column.bool("known") |
669 val known = SQL.Column.bool("known") |
649 |
670 |
650 val meta_info_table = |
671 val meta_info_table = |
651 build_log_table("meta_info", log_name :: Prop.all_props ::: Settings.all_settings) |
672 build_log_table("meta_info", log_name :: Prop.all_props ::: Settings.all_settings) |
652 |
673 |
653 val sessions_table = |
674 val sessions_table = |
654 build_log_table("sessions", |
675 build_log_table("sessions", |
655 List(log_name, session_name, chapter, groups, threads, timing_elapsed, timing_cpu, |
676 List(log_name, session_name, chapter, groups, threads, timing_elapsed, timing_cpu, |
656 timing_gc, timing_factor, ml_timing_elapsed, ml_timing_cpu, ml_timing_gc, ml_timing_factor, |
677 timing_gc, timing_factor, ml_timing_elapsed, ml_timing_cpu, ml_timing_gc, ml_timing_factor, |
657 heap_size, status)) |
678 heap_size, status, errors)) |
658 |
679 |
659 val ml_statistics_table = |
680 val ml_statistics_table = |
660 build_log_table("ml_statistics", List(log_name, session_name, ml_statistics)) |
681 build_log_table("ml_statistics", List(log_name, session_name, ml_statistics)) |
661 |
682 |
662 |
683 |
878 stmt.long(11) = session.ml_timing.cpu.proper_ms |
899 stmt.long(11) = session.ml_timing.cpu.proper_ms |
879 stmt.long(12) = session.ml_timing.gc.proper_ms |
900 stmt.long(12) = session.ml_timing.gc.proper_ms |
880 stmt.double(13) = session.ml_timing.factor |
901 stmt.double(13) = session.ml_timing.factor |
881 stmt.long(14) = session.heap_size |
902 stmt.long(14) = session.heap_size |
882 stmt.string(15) = session.status.map(_.toString) |
903 stmt.string(15) = session.status.map(_.toString) |
|
904 stmt.bytes(16) = compress_errors(session.errors) |
883 stmt.execute() |
905 stmt.execute() |
884 } |
906 } |
885 }) |
907 }) |
886 } |
908 } |
887 |
909 |
1011 timing = res.timing(Data.timing_elapsed, Data.timing_cpu, Data.timing_gc), |
1033 timing = res.timing(Data.timing_elapsed, Data.timing_cpu, Data.timing_gc), |
1012 ml_timing = |
1034 ml_timing = |
1013 res.timing(Data.ml_timing_elapsed, Data.ml_timing_cpu, Data.ml_timing_gc), |
1035 res.timing(Data.ml_timing_elapsed, Data.ml_timing_cpu, Data.ml_timing_gc), |
1014 heap_size = res.get_long(Data.heap_size), |
1036 heap_size = res.get_long(Data.heap_size), |
1015 status = res.get_string(Data.status).map(Session_Status.withName(_)), |
1037 status = res.get_string(Data.status).map(Session_Status.withName(_)), |
|
1038 errors = uncompress_errors(res.bytes(Data.errors)), |
1016 ml_statistics = |
1039 ml_statistics = |
1017 if (ml_statistics) |
1040 if (ml_statistics) |
1018 Properties.uncompress(res.bytes(Data.ml_statistics), Some(xml_cache)) |
1041 Properties.uncompress(res.bytes(Data.ml_statistics), Some(xml_cache)) |
1019 else Nil) |
1042 else Nil) |
1020 session_name -> session_entry |
1043 session_name -> session_entry |