14 val iterations : int Config.T |
14 val iterations : int Config.T |
15 val no_assms : bool Config.T |
15 val no_assms : bool Config.T |
16 val report : bool Config.T |
16 val report : bool Config.T |
17 val quiet : bool Config.T |
17 val quiet : bool Config.T |
18 val timeout : real Config.T |
18 val timeout : real Config.T |
|
19 val finite_types : bool Config.T |
|
20 val finite_type_size : int Config.T |
19 datatype report = Report of |
21 datatype report = Report of |
20 { iterations : int, raised_match_errors : int, |
22 { iterations : int, raised_match_errors : int, |
21 satisfied_assms : int list, positive_concl_tests : int } |
23 satisfied_assms : int list, positive_concl_tests : int } |
22 datatype expectation = No_Expectation | No_Counterexample | Counterexample; |
24 datatype expectation = No_Expectation | No_Counterexample | Counterexample; |
23 datatype test_params = Test_Params of {default_type: typ list, expect : expectation}; |
25 datatype test_params = Test_Params of {default_type: typ list, expect : expectation}; |
30 (* testing terms and proof states *) |
32 (* testing terms and proof states *) |
31 val test_term_small: Proof.context -> term -> |
33 val test_term_small: Proof.context -> term -> |
32 (string * term) list option * ((string * int) list * ((int * report list) list) option) |
34 (string * term) list option * ((string * int) list * ((int * report list) list) option) |
33 val test_term: Proof.context -> string option -> term -> |
35 val test_term: Proof.context -> string option -> term -> |
34 (string * term) list option * ((string * int) list * ((int * report list) list) option) |
36 (string * term) list option * ((string * int) list * ((int * report list) list) option) |
|
37 val test_goal_terms: |
|
38 Proof.context -> string option * (string * typ) list -> term list |
|
39 -> (string * term) list option * ((string * int) list * ((int * report list) list) option) list |
35 val quickcheck: (string * string list) list -> int -> Proof.state -> (string * term) list option |
40 val quickcheck: (string * string list) list -> int -> Proof.state -> (string * term) list option |
36 end; |
41 end; |
37 |
42 |
38 structure Quickcheck : QUICKCHECK = |
43 structure Quickcheck : QUICKCHECK = |
39 struct |
44 struct |
197 val result = with_size 1 [comp_time] |
202 val result = with_size 1 [comp_time] |
198 in |
203 in |
199 apsnd (rpair NONE) (apfst (Option.map (curry (op ~~) names)) result) |
204 apsnd (rpair NONE) (apfst (Option.map (curry (op ~~) names)) result) |
200 end |
205 end |
201 |
206 |
|
207 (* we actually assume we know the generators and its behaviour *) |
|
208 fun is_iteratable "small" = false |
|
209 | is_iteratable "random" = true |
|
210 |
202 fun test_term ctxt generator_name t = |
211 fun test_term ctxt generator_name t = |
203 let |
212 let |
204 val (names, t') = prep_test_term t; |
213 val (names, t') = prep_test_term t; |
205 val current_size = Unsynchronized.ref 0 |
214 val current_size = Unsynchronized.ref 0 |
206 fun excipit s = |
215 fun excipit s = |
221 end |
230 end |
222 val empty_report = Report { iterations = 0, raised_match_errors = 0, |
231 val empty_report = Report { iterations = 0, raised_match_errors = 0, |
223 satisfied_assms = [], positive_concl_tests = 0 } |
232 satisfied_assms = [], positive_concl_tests = 0 } |
224 fun with_testers k [] = (NONE, []) |
233 fun with_testers k [] = (NONE, []) |
225 | with_testers k (tester :: testers) = |
234 | with_testers k (tester :: testers) = |
226 case iterate (fn () => tester (k - 1)) (Config.get ctxt iterations) empty_report |
235 let |
|
236 val niterations = case generator_name of |
|
237 SOME generator_name => |
|
238 if is_iteratable generator_name then Config.get ctxt iterations else 1 |
|
239 | NONE => Config.get ctxt iterations |
|
240 val (result, timing) = cpu_time ("size " ^ string_of_int k) |
|
241 (fn () => iterate (fn () => tester (k - 1)) niterations empty_report) |
|
242 in |
|
243 case result |
227 of (NONE, report) => apsnd (cons report) (with_testers k testers) |
244 of (NONE, report) => apsnd (cons report) (with_testers k testers) |
228 | (SOME q, report) => (SOME q, [report]); |
245 | (SOME q, report) => (SOME q, [report]) |
|
246 end |
229 fun with_size k reports = |
247 fun with_size k reports = |
230 if k > Config.get ctxt size then (NONE, reports) |
248 if k > Config.get ctxt size then (NONE, reports) |
231 else |
249 else |
232 (if Config.get ctxt quiet then () else Output.urgent_message ("Test data size: " ^ string_of_int k); |
250 (if Config.get ctxt quiet then () else Output.urgent_message ("Test data size: " ^ string_of_int k); |
233 let |
251 let |
234 val _ = current_size := k |
252 val _ = current_size := k |
235 val (result, new_report) = with_testers k testers |
253 val (result, new_report) = with_testers k testers |
236 val reports = ((k, new_report) :: reports) |
254 val reports = ((k, new_report) :: reports) |
237 in case result of NONE => with_size (k + 1) reports | SOME q => (SOME q, reports) end); |
255 in case result of NONE => with_size (k + 1) reports | SOME q => (SOME q, reports) end); |
238 val ((result, reports), exec_time) = |
256 val ((result, reports), exec_time) = |
239 TimeLimit.timeLimit (seconds (Config.get ctxt timeout)) (fn () => cpu_time "quickcheck execution" |
257 TimeLimit.timeLimit (seconds (Config.get ctxt timeout)) (fn () => cpu_time "quickcheck execution" |
270 | subst T = T; |
288 | subst T = T; |
271 in (map_types o map_atyps) subst end; |
289 in (map_types o map_atyps) subst end; |
272 |
290 |
273 datatype wellsorted_error = Wellsorted_Error of string | Term of term |
291 datatype wellsorted_error = Wellsorted_Error of string | Term of term |
274 |
292 |
275 fun test_goal (generator_name, insts) i state = |
293 fun test_goal_terms lthy (generator_name, insts) check_goals = |
276 let |
294 let |
277 val lthy = Proof.context_of state; |
295 val thy = ProofContext.theory_of lthy |
278 val thy = Proof.theory_of state; |
|
279 fun strip (Const ("all", _) $ Abs (_, _, t)) = strip t |
|
280 | strip t = t; |
|
281 val {goal = st, ...} = Proof.raw_goal state; |
|
282 val (gi, frees) = Logic.goal_params (prop_of st) i; |
|
283 val some_locale = case (Option.map #target o Named_Target.peek) lthy |
|
284 of NONE => NONE |
|
285 | SOME "" => NONE |
|
286 | SOME locale => SOME locale; |
|
287 val assms = if Config.get lthy no_assms then [] else case some_locale |
|
288 of NONE => Assumption.all_assms_of lthy |
|
289 | SOME locale => Assumption.local_assms_of lthy (Locale.init locale thy); |
|
290 val proto_goal = Logic.list_implies (map Thm.term_of assms, subst_bounds (frees, strip gi)); |
|
291 val check_goals = case some_locale |
|
292 of NONE => [proto_goal] |
|
293 | SOME locale => map (fn (_, phi) => Morphism.term phi proto_goal) |
|
294 (Locale.registrations_of (Context.Theory thy) (*FIXME*) locale); |
|
295 val inst_goals = |
296 val inst_goals = |
296 if Config.get lthy finite_types then |
297 if Config.get lthy finite_types then |
297 maps (fn check_goal => map (fn T => |
298 maps (fn check_goal => map (fn T => |
298 Term ((Object_Logic.atomize_term thy o monomorphic_term thy insts T) check_goal) |
299 Term ((Object_Logic.atomize_term thy o monomorphic_term thy insts T) check_goal) |
299 handle WELLSORTED s => Wellsorted_Error s) (get_finite_types lthy)) check_goals |
300 handle WELLSORTED s => Wellsorted_Error s) (get_finite_types lthy)) check_goals |
312 case f t of |
313 case f t of |
313 (SOME res, report) => (SOME res, rev (report :: reports)) |
314 (SOME res, report) => (SOME res, rev (report :: reports)) |
314 | (NONE, report) => collect_results f (report :: reports) ts |
315 | (NONE, report) => collect_results f (report :: reports) ts |
315 in collect_results (test_term lthy generator_name) [] correct_inst_goals end; |
316 in collect_results (test_term lthy generator_name) [] correct_inst_goals end; |
316 |
317 |
|
318 fun test_goal (generator_name, insts) i state = |
|
319 let |
|
320 val lthy = Proof.context_of state; |
|
321 val thy = Proof.theory_of state; |
|
322 fun strip (Const ("all", _) $ Abs (_, _, t)) = strip t |
|
323 | strip t = t; |
|
324 val {goal = st, ...} = Proof.raw_goal state; |
|
325 val (gi, frees) = Logic.goal_params (prop_of st) i; |
|
326 val some_locale = case (Option.map #target o Named_Target.peek) lthy |
|
327 of NONE => NONE |
|
328 | SOME "" => NONE |
|
329 | SOME locale => SOME locale; |
|
330 val assms = if Config.get lthy no_assms then [] else case some_locale |
|
331 of NONE => Assumption.all_assms_of lthy |
|
332 | SOME locale => Assumption.local_assms_of lthy (Locale.init locale thy); |
|
333 val proto_goal = Logic.list_implies (map Thm.term_of assms, subst_bounds (frees, strip gi)); |
|
334 val check_goals = case some_locale |
|
335 of NONE => [proto_goal] |
|
336 | SOME locale => map (fn (_, phi) => Morphism.term phi proto_goal) |
|
337 (Locale.registrations_of (Context.Theory thy) (*FIXME*) locale); |
|
338 in |
|
339 test_goal_terms lthy (generator_name, insts) check_goals |
|
340 end |
|
341 |
317 (* pretty printing *) |
342 (* pretty printing *) |
318 |
343 |
319 fun tool_name auto = (if auto then "Auto " else "") ^ "Quickcheck" |
344 fun tool_name auto = (if auto then "Auto " else "") ^ "Quickcheck" |
320 |
345 |
321 fun pretty_counterex ctxt auto NONE = Pretty.str (tool_name auto ^ " found no counterexample.") |
346 fun pretty_counterex ctxt auto NONE = Pretty.str (tool_name auto ^ " found no counterexample.") |