829 user = user, password = password, database = database, host = host, port = port, |
829 user = user, password = password, database = database, host = host, port = port, |
830 ssh = |
830 ssh = |
831 if (ssh_host == "") None |
831 if (ssh_host == "") None |
832 else Some(SSH.open_session(options, host = ssh_host, user = ssh_user, port = ssh_port)), |
832 else Some(SSH.open_session(options, host = ssh_host, user = ssh_user, port = ssh_port)), |
833 ssh_close = true) |
833 ssh_close = true) |
834 } |
|
835 |
|
836 def update_database( |
|
837 db: PostgreSQL.Database, dirs: List[Path], ml_statistics: Boolean = false): Unit = { |
|
838 val log_files = |
|
839 dirs.flatMap(dir => |
|
840 File.find_files(dir.file, pred = Log_File.is_log(_), follow_links = true)) |
|
841 write_info(db, log_files, ml_statistics = ml_statistics) |
|
842 |
|
843 db.create_view(Data.pull_date_table()) |
|
844 db.create_view(Data.pull_date_table(afp = true)) |
|
845 db.create_view(Data.universal_table) |
|
846 } |
834 } |
847 |
835 |
848 def snapshot_database( |
836 def snapshot_database( |
849 db: PostgreSQL.Database, |
837 db: PostgreSQL.Database, |
850 sqlite_database: Path, |
838 sqlite_database: Path, |
1025 files.filter(file => status.exists(_.required(file))). |
1013 files.filter(file => status.exists(_.required(file))). |
1026 grouped(options.int("build_log_transaction_size") max 1)) { |
1014 grouped(options.int("build_log_transaction_size") max 1)) { |
1027 val log_files = Par_List.map[JFile, Log_File](Log_File.apply, file_group) |
1015 val log_files = Par_List.map[JFile, Log_File](Log_File.apply, file_group) |
1028 db.transaction { log_files.foreach(log_file => status.foreach(_.update(log_file))) } |
1016 db.transaction { log_files.foreach(log_file => status.foreach(_.update(log_file))) } |
1029 } |
1017 } |
|
1018 |
|
1019 db.create_view(Data.pull_date_table()) |
|
1020 db.create_view(Data.pull_date_table(afp = true)) |
|
1021 db.create_view(Data.universal_table) |
1030 } |
1022 } |
1031 |
1023 |
1032 def read_meta_info(db: SQL.Database, log_name: String): Option[Meta_Info] = { |
1024 def read_meta_info(db: SQL.Database, log_name: String): Option[Meta_Info] = { |
1033 val table = Data.meta_info_table |
1025 val table = Data.meta_info_table |
1034 val columns = table.columns.tail |
1026 val columns = table.columns.tail |
1120 |
1112 |
1121 def build_log_database(options: Options, log_dirs: List[Path], |
1113 def build_log_database(options: Options, log_dirs: List[Path], |
1122 snapshot: Option[Path] = None |
1114 snapshot: Option[Path] = None |
1123 ): Unit = { |
1115 ): Unit = { |
1124 val store = Build_Log.store(options) |
1116 val store = Build_Log.store(options) |
|
1117 |
|
1118 val log_files = |
|
1119 log_dirs.flatMap(dir => |
|
1120 File.find_files(dir.file, pred = Log_File.is_log(_), follow_links = true)) |
|
1121 |
1125 using(store.open_database()) { db => |
1122 using(store.open_database()) { db => |
1126 db.vacuum() |
1123 db.vacuum() |
1127 store.update_database(db, log_dirs) |
1124 store.write_info(db, log_files) |
1128 store.update_database(db, log_dirs, ml_statistics = true) |
1125 store.write_info(db, log_files, ml_statistics = true) |
1129 snapshot.foreach(store.snapshot_database(db, _)) |
1126 snapshot.foreach(store.snapshot_database(db, _)) |
1130 } |
1127 } |
1131 } |
1128 } |
1132 } |
1129 } |