734     }  | 
   734     }  | 
   735   | 
   735   | 
   736   | 
   736   | 
   737     /* earliest pull date for repository version (PostgreSQL queries) */  | 
   737     /* earliest pull date for repository version (PostgreSQL queries) */  | 
   738   | 
   738   | 
   739     val pull_date = SQL.Column.date("pull_date") | 
   739     def pull_date(afp: Boolean = false) =  | 
         | 
   740       if (afp) SQL.Column.date("afp_pull_date") | 
         | 
   741       else SQL.Column.date("pull_date") | 
   740   | 
   742   | 
   741     def pull_date_table(afp: Boolean = false): SQL.Table =  | 
   743     def pull_date_table(afp: Boolean = false): SQL.Table =  | 
   742     { | 
   744     { | 
   743       val (name, versions) =  | 
   745       val (name, versions) =  | 
   744         if (afp) ("afp_pull_date", List(Prop.isabelle_version, Prop.afp_version)) | 
   746         if (afp) ("afp_pull_date", List(Prop.isabelle_version, Prop.afp_version)) | 
   745         else ("pull_date", List(Prop.isabelle_version)) | 
   747         else ("pull_date", List(Prop.isabelle_version)) | 
   746   | 
   748   | 
   747       build_log_table(name, versions.map(_.make_primary_key) ::: List(pull_date),  | 
   749       build_log_table(name, versions.map(_.make_primary_key) ::: List(pull_date(afp)),  | 
   748         "SELECT " + versions.mkString(", ") + ", min(" + Prop.build_start + ") AS " + pull_date + | 
   750         "SELECT " + versions.mkString(", ") + | 
         | 
   751           ", min(" + Prop.build_start + ") AS " + pull_date(afp) + | 
   749         " FROM " + meta_info_table +  | 
   752         " FROM " + meta_info_table +  | 
   750         " WHERE " + (versions ::: List(Prop.build_start)).map(_.defined).mkString(" AND ") + | 
   753         " WHERE " + (versions ::: List(Prop.build_start)).map(_.defined).mkString(" AND ") + | 
   751         " GROUP BY " + versions.mkString(", ")) | 
   754         " GROUP BY " + versions.mkString(", ")) | 
   752     }  | 
   755     }  | 
   753   | 
   756   | 
   769       val eq1 = version1(table) + " = " + SQL.string(rev)  | 
   772       val eq1 = version1(table) + " = " + SQL.string(rev)  | 
   770       val eq2 = version2(table) + " = " + SQL.string(rev2)  | 
   773       val eq2 = version2(table) + " = " + SQL.string(rev2)  | 
   771   | 
   774   | 
   772       SQL.Table("recent_pull_date", table.columns, | 
   775       SQL.Table("recent_pull_date", table.columns, | 
   773         table.select(table.columns,  | 
   776         table.select(table.columns,  | 
   774           "WHERE " + pull_date(table) + " > " + recent_time(days) +  | 
   777           "WHERE " + pull_date(afp)(table) + " > " + recent_time(days) +  | 
   775           (if (rev != "" && rev2 == "") " OR " + eq1  | 
   778           (if (rev != "" && rev2 == "") " OR " + eq1  | 
   776            else if (rev == "" && rev2 != "") " OR " + eq2  | 
   779            else if (rev == "" && rev2 != "") " OR " + eq2  | 
   777            else if (rev != "" && rev2 != "") " OR (" + eq1 + " AND " + eq2 + ")" | 
   780            else if (rev != "" && rev2 != "") " OR (" + eq1 + " AND " + eq2 + ")" | 
   778            else "")))  | 
   781            else "")))  | 
   779     }  | 
   782     }  | 
   787     }  | 
   790     }  | 
   788   | 
   791   | 
   789     def select_recent_versions(days: Int,  | 
   792     def select_recent_versions(days: Int,  | 
   790       rev: String = "", afp_rev: Option[String] = None, sql: SQL.Source = ""): SQL.Source =  | 
   793       rev: String = "", afp_rev: Option[String] = None, sql: SQL.Source = ""): SQL.Source =  | 
   791     { | 
   794     { | 
         | 
   795       val afp = afp_rev.isDefined  | 
   792       val version = Prop.isabelle_version  | 
   796       val version = Prop.isabelle_version  | 
   793       val table1 = recent_pull_date_table(days, rev = rev, afp_rev = afp_rev)  | 
   797       val table1 = recent_pull_date_table(days, rev = rev, afp_rev = afp_rev)  | 
   794       val table2 = meta_info_table  | 
   798       val table2 = meta_info_table  | 
   795       val aux_table = SQL.Table("aux", table2.columns, table2.select(sql = sql)) | 
   799       val aux_table = SQL.Table("aux", table2.columns, table2.select(sql = sql)) | 
   796   | 
   800   | 
   798         table1.columns.map(c => c(table1)) :::  | 
   802         table1.columns.map(c => c(table1)) :::  | 
   799           List(known.copy(expr = log_name(aux_table).defined))  | 
   803           List(known.copy(expr = log_name(aux_table).defined))  | 
   800       SQL.select(columns, distinct = true) +  | 
   804       SQL.select(columns, distinct = true) +  | 
   801         table1.query_named + SQL.join_outer + aux_table.query_named +  | 
   805         table1.query_named + SQL.join_outer + aux_table.query_named +  | 
   802         " ON " + version(table1) + " = " + version(aux_table) +  | 
   806         " ON " + version(table1) + " = " + version(aux_table) +  | 
   803         " ORDER BY " + pull_date(table1) + " DESC"  | 
   807         " ORDER BY " + pull_date(afp)(table1) + " DESC"  | 
   804     }  | 
   808     }  | 
   805   | 
   809   | 
   806   | 
   810   | 
   807     /* universal view on main data */  | 
   811     /* universal view on main data */  | 
   808   | 
   812   | 
   809     val universal_table: SQL.Table =  | 
   813     val universal_table: SQL.Table =  | 
   810     { | 
   814     { | 
         | 
   815       val afp_pull_date = pull_date(afp = true)  | 
         | 
   816       val version1 = Prop.isabelle_version  | 
         | 
   817       val version2 = Prop.afp_version  | 
   811       val table1 = meta_info_table  | 
   818       val table1 = meta_info_table  | 
   812       val table2 = pull_date_table()  | 
   819       val table2 = pull_date_table(afp = true)  | 
   813       val table3 = sessions_table  | 
   820       val table3 = pull_date_table()  | 
   814       val table4 = ml_statistics_table  | 
   821   | 
   815   | 
   822       val a_columns = log_name :: afp_pull_date :: table1.columns.tail  | 
   816       val a_columns = log_name :: pull_date :: meta_info_table.columns.tail  | 
         | 
   817       val a_table =  | 
   823       val a_table =  | 
   818         SQL.Table("a", a_columns, | 
   824         SQL.Table("a", a_columns, | 
   819           SQL.select(a_columns.take(2) ::: a_columns.drop(2).map(_.apply(table1))) +  | 
   825           SQL.select(List(log_name, afp_pull_date) ::: table1.columns.tail.map(_.apply(table1))) +  | 
   820             table1 + SQL.join_outer + table2 + " ON " +  | 
   826           table1 + SQL.join_outer + table2 +  | 
   821             Prop.isabelle_version(table1) + " = " + Prop.isabelle_version(table2))  | 
   827           " ON " + version1(table1) + " = " + version1(table2) +  | 
   822   | 
   828           " AND " + version2(table1) + " = " + version2(table2))  | 
   823       val b_columns = a_columns ::: sessions_table.columns.tail  | 
   829   | 
         | 
   830       val b_columns = log_name :: pull_date() :: a_columns.tail  | 
   824       val b_table =  | 
   831       val b_table =  | 
   825         SQL.Table("b", b_columns, | 
   832         SQL.Table("b", b_columns, | 
   826           SQL.select(log_name(a_table) :: b_columns.tail) + a_table.query_named +  | 
   833           SQL.select(  | 
   827           SQL.join_inner + table3 + " ON " + log_name(a_table) + " = " + log_name(table3))  | 
   834             List(log_name(a_table), pull_date()(table3)) ::: a_columns.tail.map(_.apply(a_table))) +  | 
   828   | 
   835           a_table.query_named + SQL.join_outer + table3 +  | 
   829       SQL.Table("isabelle_build_log", b_columns ::: List(ml_statistics), | 
   836           " ON " + version1(a_table) + " = " + version1(table3))  | 
         | 
   837   | 
         | 
   838       val c_columns = b_columns ::: sessions_table.columns.tail  | 
         | 
   839       val c_table =  | 
         | 
   840         SQL.Table("c", c_columns, | 
         | 
   841           SQL.select(log_name(b_table) :: c_columns.tail) +  | 
         | 
   842           b_table.query_named + SQL.join_inner + sessions_table +  | 
         | 
   843           " ON " + log_name(b_table) + " = " + log_name(sessions_table))  | 
         | 
   844   | 
         | 
   845       SQL.Table("isabelle_build_log", c_columns ::: List(ml_statistics), | 
   830         { | 
   846         { | 
   831           SQL.select(b_columns.map(_.apply(b_table)) ::: List(ml_statistics)) +  | 
   847           SQL.select(c_columns.map(_.apply(c_table)) ::: List(ml_statistics)) +  | 
   832             b_table.query_named + SQL.join_outer + table4 + " ON " +  | 
   848           c_table.query_named + SQL.join_outer + ml_statistics_table +  | 
   833             log_name(b_table) + " = " + log_name(table4) + " AND " +  | 
   849           " ON " + log_name(c_table) + " = " + log_name(ml_statistics_table) +  | 
   834             session_name(b_table) + " = " + session_name(table4)  | 
   850           " AND " + session_name(c_table) + " = " + session_name(ml_statistics_table)  | 
   835         })  | 
   851         })  | 
   836     }  | 
   852     }  | 
   837   }  | 
   853   }  | 
   838   | 
   854   | 
   839   | 
   855   | 
   903                   read_build_info(db, log_name, ml_statistics = true))  | 
   919                   read_build_info(db, log_name, ml_statistics = true))  | 
   904               }  | 
   920               }  | 
   905             }  | 
   921             }  | 
   906   | 
   922   | 
   907             // pull_date  | 
   923             // pull_date  | 
         | 
   924             for (afp <- List(false, true))  | 
   908             { | 
   925             { | 
   909               val table = Data.pull_date_table()  | 
   926               val afp_rev = if (afp) Some("") else None | 
         | 
   927               val table = Data.pull_date_table(afp)  | 
   910               db2.create_table(table)  | 
   928               db2.create_table(table)  | 
   911               db2.using_statement(table.insert())(stmt2 =>  | 
   929               db2.using_statement(table.insert())(stmt2 =>  | 
   912               { | 
   930               { | 
   913                 db.using_statement(Data.recent_pull_date_table(days).query)(stmt =>  | 
   931                 db.using_statement(  | 
         | 
   932                   Data.recent_pull_date_table(days, afp_rev = afp_rev).query)(stmt =>  | 
   914                 { | 
   933                 { | 
   915                   val res = stmt.execute_query()  | 
   934                   val res = stmt.execute_query()  | 
   916                   while (res.next()) { | 
   935                   while (res.next()) { | 
   917                     for ((c, i) <- table.columns.zipWithIndex) { | 
   936                     for ((c, i) <- table.columns.zipWithIndex) { | 
   918                       stmt2.string(i + 1) = res.get_string(c)  | 
   937                       stmt2.string(i + 1) = res.get_string(c)  |