author | wenzelm |
Wed, 12 Oct 2016 21:53:30 +0200 | |
changeset 64173 | 85ff21510ba9 |
parent 64172 | e7863057df41 |
child 64184 | 68e95e5b2b7d |
permissions | -rw-r--r-- |
64148 | 1 |
/* Title: Pure/Admin/isabelle_cronjob.scala |
2 |
Author: Makarius |
|
3 |
||
4 |
Main entry point for administrative cronjob at TUM. |
|
5 |
*/ |
|
6 |
||
7 |
package isabelle |
|
8 |
||
9 |
||
64170 | 10 |
import scala.annotation.tailrec |
11 |
import scala.collection.mutable |
|
12 |
||
13 |
||
64148 | 14 |
object Isabelle_Cronjob |
15 |
{ |
|
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
16 |
/** file-system state: owned by main cronjob **/ |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
17 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
18 |
val main_dir = Path.explode("~/cronjob") |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
19 |
val run_dir = main_dir + Path.explode("run") |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
20 |
val log_dir = main_dir + Path.explode("log") |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
21 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
22 |
val main_state_file = run_dir + Path.explode("main.state") |
64171 | 23 |
val main_log = log_dir + Path.explode("main.log") // owned by log service |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
24 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
25 |
|
64171 | 26 |
/* task logging */ |
27 |
||
28 |
sealed case class Logger_Task(name: String, body: Logger => Unit) |
|
64154 | 29 |
|
64171 | 30 |
class Log_Service private[Isabelle_Cronjob](progress: Progress) |
31 |
{ |
|
32 |
private val thread: Consumer_Thread[String] = |
|
33 |
Consumer_Thread.fork("cronjob: logger", daemon = true)( |
|
34 |
consume = (text: String) => |
|
35 |
{ |
|
36 |
File.append(main_log, text + "\n") // critical |
|
37 |
progress.echo(text) |
|
38 |
true |
|
39 |
}) |
|
40 |
||
41 |
def shutdown() { thread.shutdown() } |
|
42 |
||
43 |
val hostname = Isabelle_System.hostname() |
|
44 |
||
45 |
def log(date: Date, task_name: String, msg: String): Unit = |
|
46 |
thread.send( |
|
47 |
"[" + Build_Log.print_date(date) + ", " + hostname + ", " + task_name + "]: " + msg) |
|
48 |
||
49 |
def start_logger(start_date: Date, task_name: String): Logger = |
|
50 |
new Logger(this, start_date, task_name) |
|
64154 | 51 |
|
64171 | 52 |
def run_task(start_date: Date, task: Logger_Task) |
53 |
{ |
|
54 |
val logger = start_logger(start_date, task.name) |
|
55 |
val res = Exn.capture { task.body(logger) } |
|
56 |
val end_date = Date.now() |
|
57 |
val err = |
|
58 |
res match { |
|
59 |
case Exn.Res(_) => None |
|
60 |
case Exn.Exn(exn) => Some(Exn.message(exn)) |
|
61 |
} |
|
62 |
logger.log_end(end_date, err) |
|
63 |
} |
|
64 |
||
65 |
def fork_task(start_date: Date, task: Logger_Task): Task = |
|
66 |
new Task(task.name, run_task(start_date, task)) |
|
67 |
} |
|
68 |
||
69 |
class Logger private[Isabelle_Cronjob]( |
|
70 |
val log_service: Log_Service, val start_date: Date, val task_name: String) |
|
64162 | 71 |
{ |
64171 | 72 |
def log(date: Date, msg: String): Unit = log_service.log(date, task_name, msg) |
73 |
||
74 |
def log_end(end_date: Date, err: Option[String]) |
|
75 |
{ |
|
76 |
val elapsed_time = end_date.time - start_date.time |
|
77 |
val msg = |
|
78 |
(if (err.isEmpty) "finished" else "ERROR " + err.get) + |
|
79 |
(if (elapsed_time.seconds < 3.0) "" else ", elapsed time " + elapsed_time.message_hms) |
|
80 |
log(end_date, msg) |
|
81 |
} |
|
82 |
||
83 |
log(start_date, "started") |
|
84 |
} |
|
85 |
||
86 |
class Task private[Isabelle_Cronjob](name: String, body: => Unit) |
|
87 |
{ |
|
88 |
private val future: Future[Unit] = Future.thread("cronjob: " + name) { body } |
|
89 |
def is_finished: Boolean = future.is_finished |
|
64162 | 90 |
} |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
91 |
|
64170 | 92 |
|
93 |
||
94 |
/** particular tasks **/ |
|
95 |
||
96 |
/* identify repository snapshots */ |
|
97 |
||
64171 | 98 |
val isabelle_repos = main_dir + Path.explode("isabelle-build_history") |
99 |
val afp_repos = main_dir + Path.explode("AFP-build_history") |
|
100 |
||
101 |
val isabelle_identify = |
|
102 |
Logger_Task("isabelle_identify", logger => |
|
103 |
{ |
|
104 |
def pull_repos(root: Path): String = |
|
105 |
{ |
|
106 |
val hg = Mercurial.repository(root) |
|
107 |
hg.pull(options = "-q") |
|
108 |
hg.identify("tip", options = "-i") |
|
109 |
} |
|
64170 | 110 |
|
64171 | 111 |
val isabelle_id = pull_repos(isabelle_repos) |
112 |
val afp_id = pull_repos(afp_repos) |
|
113 |
||
114 |
val log_path = log_dir + Build_Log.log_path("isabelle_identify", logger.start_date) |
|
115 |
Isabelle_System.mkdirs(log_path.dir) |
|
116 |
File.write(log_path, |
|
64173 | 117 |
terminate_lines( |
64171 | 118 |
List("isabelle_identify: " + Build_Log.print_date(logger.start_date), |
119 |
"", |
|
120 |
"Isabelle version: " + isabelle_id, |
|
121 |
"AFP version: " + afp_id))) |
|
122 |
}) |
|
64170 | 123 |
|
124 |
||
125 |
||
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
126 |
/** cronjob **/ |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
127 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
128 |
def cronjob(progress: Progress) |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
129 |
{ |
64171 | 130 |
/* soft lock */ |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
131 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
132 |
val still_running = |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
133 |
try { Some(File.read(main_state_file)) } |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
134 |
catch { case ERROR(_) => None } |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
135 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
136 |
still_running match { |
64170 | 137 |
case None | Some("") => |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
138 |
case Some(running) => |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
139 |
error("Isabelle cronjob appears to be still running: " + running) |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
140 |
} |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
141 |
|
64171 | 142 |
val main_start_date = Date.now() |
143 |
val log_service = new Log_Service(progress) |
|
64154 | 144 |
|
64171 | 145 |
File.write(main_state_file, main_start_date + " " + log_service.hostname) |
64154 | 146 |
|
147 |
||
64171 | 148 |
/* parallel tasks */ |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
149 |
|
64171 | 150 |
def parallel_tasks(tasks: List[Logger_Task]) |
64170 | 151 |
{ |
152 |
@tailrec def await(running: List[Task]) |
|
153 |
{ |
|
154 |
running.partition(_.is_finished) match { |
|
155 |
case (Nil, Nil) => |
|
156 |
case (Nil, _ :: _) => Thread.sleep(500); await(running) |
|
64171 | 157 |
case (_ :: _, remaining) => await(remaining) |
64170 | 158 |
} |
159 |
} |
|
64171 | 160 |
val start_date = Date.now() |
161 |
await(tasks.map(task => log_service.fork_task(start_date, task))) |
|
64170 | 162 |
} |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
163 |
|
64170 | 164 |
|
165 |
/* main */ |
|
166 |
||
64171 | 167 |
log_service.run_task(main_start_date, |
64172 | 168 |
Logger_Task("isabelle_cronjob", _ => parallel_tasks(List(isabelle_identify)))) |
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
169 |
|
64171 | 170 |
log_service.shutdown() |
64170 | 171 |
|
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
172 |
main_state_file.file.delete |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
173 |
} |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
174 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
175 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
176 |
|
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
177 |
/** command line entry point **/ |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
178 |
|
64148 | 179 |
def main(args: Array[String]) |
180 |
{ |
|
181 |
Command_Line.tool0 { |
|
182 |
var force = false |
|
183 |
var verbose = false |
|
184 |
||
185 |
val getopts = Getopts(""" |
|
186 |
Usage: Admin/cronjob/main [OPTIONS] |
|
187 |
||
188 |
Options are: |
|
189 |
-f apply force to do anything |
|
190 |
-v verbose |
|
191 |
""", |
|
192 |
"f" -> (_ => force = true), |
|
193 |
"v" -> (_ => verbose = true)) |
|
194 |
||
195 |
val more_args = getopts(args) |
|
196 |
if (more_args.nonEmpty) getopts.usage() |
|
197 |
||
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
198 |
val progress = if (verbose) new Console_Progress() else Ignore_Progress |
64148 | 199 |
|
64153
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
200 |
if (force) cronjob(progress) |
769791954872
some timing and logging, similar to old isatest.log;
wenzelm
parents:
64148
diff
changeset
|
201 |
else error("Need to apply force to do anything") |
64148 | 202 |
} |
203 |
} |
|
204 |
} |