103 |
103 |
104 (* possible invocations *) |
104 (* possible invocations *) |
105 |
105 |
106 (** quickcheck **) |
106 (** quickcheck **) |
107 |
107 |
108 fun invoke_quickcheck change_options quickcheck_generator thy t = |
108 fun invoke_quickcheck change_options thy t = |
109 TimeLimit.timeLimit (seconds 30.0) |
109 TimeLimit.timeLimit (seconds 30.0) |
110 (fn _ => |
110 (fn _ => |
111 let |
111 let |
112 val ctxt' = change_options (Proof_Context.init_global thy) |
112 val ctxt' = change_options (Proof_Context.init_global thy) |
113 val (result :: _) = case Quickcheck.active_testers ctxt' of |
113 val (result :: _) = case Quickcheck.active_testers ctxt' of |
121 end) () |
121 end) () |
122 handle TimeLimit.TimeOut => |
122 handle TimeLimit.TimeOut => |
123 (Timeout, [("timelimit", Real.floor (!Try.auto_time_limit))]) |
123 (Timeout, [("timelimit", Real.floor (!Try.auto_time_limit))]) |
124 |
124 |
125 fun quickcheck_mtd change_options quickcheck_generator = |
125 fun quickcheck_mtd change_options quickcheck_generator = |
126 ("quickcheck_" ^ quickcheck_generator, invoke_quickcheck change_options quickcheck_generator) |
126 ("quickcheck_" ^ quickcheck_generator, invoke_quickcheck change_options) |
127 |
127 |
128 (** solve direct **) |
128 (** solve direct **) |
129 |
129 |
130 fun invoke_solve_direct thy t = |
130 fun invoke_solve_direct thy t = |
131 let |
131 let |
333 filter |
333 filter |
334 (fn th => (all orelse Context.theory_name (theory_of_thm th) = Context.theory_name thy) |
334 (fn th => (all orelse Context.theory_name (theory_of_thm th) = Context.theory_name thy) |
335 (* andalso is_executable_thm thy th *)) |
335 (* andalso is_executable_thm thy th *)) |
336 (map snd (filter_out is_forbidden_theorem (Mutabelle.all_unconcealed_thms_of thy))) |
336 (map snd (filter_out is_forbidden_theorem (Mutabelle.all_unconcealed_thms_of thy))) |
337 |
337 |
338 fun count x = (length oo filter o equal) x |
|
339 |
|
340 fun elapsed_time description e = |
338 fun elapsed_time description e = |
341 let val ({elapsed, ...}, result) = Timing.timing e () |
339 let val ({elapsed, ...}, result) = Timing.timing e () |
342 in (result, (description, Time.toMilliseconds elapsed)) end |
340 in (result, (description, Time.toMilliseconds elapsed)) end |
343 (* |
341 (* |
344 fun unsafe_invoke_mtd thy (mtd_name, invoke_mtd) t = |
342 fun unsafe_invoke_mtd thy (mtd_name, invoke_mtd) t = |
352 fun safe_invoke_mtd thy (mtd_name, invoke_mtd) t = |
350 fun safe_invoke_mtd thy (mtd_name, invoke_mtd) t = |
353 let |
351 let |
354 val _ = Output.urgent_message ("Invoking " ^ mtd_name) |
352 val _ = Output.urgent_message ("Invoking " ^ mtd_name) |
355 val (res, timing) = elapsed_time "total time" |
353 val (res, timing) = elapsed_time "total time" |
356 (fn () => case try (invoke_mtd thy) t of |
354 (fn () => case try (invoke_mtd thy) t of |
357 SOME (res, timing) => res |
355 SOME (res, _) => res |
358 | NONE => (Output.urgent_message ("**** PROBLEMS WITH " ^ Syntax.string_of_term_global thy t); |
356 | NONE => (Output.urgent_message ("**** PROBLEMS WITH " ^ Syntax.string_of_term_global thy t); |
359 Error)) |
357 Error)) |
360 val _ = Output.urgent_message (" Done") |
358 val _ = Output.urgent_message (" Done") |
361 in (res, [timing]) end |
359 in (res, [timing]) end |
362 |
360 |
363 (* theory -> term list -> mtd -> subentry *) |
|
364 |
|
365 fun test_mutants_using_one_method thy mutants (mtd_name, invoke_mtd) = |
|
366 let |
|
367 val res = map (fst o safe_invoke_mtd thy (mtd_name, invoke_mtd)) mutants |
|
368 in |
|
369 (mtd_name, count GenuineCex res, count PotentialCex res, count NoCex res, |
|
370 count Donno res, count Timeout res, count Error res) |
|
371 end |
|
372 |
|
373 (* creating entries *) |
361 (* creating entries *) |
374 |
|
375 fun create_entry thy thm exec mutants mtds = |
|
376 (Thm.get_name_hint thm, exec, map (test_mutants_using_one_method thy mutants) mtds) |
|
377 |
362 |
378 fun create_detailed_entry thy thm exec mutants mtds = |
363 fun create_detailed_entry thy thm exec mutants mtds = |
379 let |
364 let |
380 fun create_mutant_subentry mutant = (mutant, |
365 fun create_mutant_subentry mutant = (mutant, |
381 map (fn (mtd_name, invoke_mtd) => |
366 map (fn (mtd_name, invoke_mtd) => |
420 val _ = map (fn t => Output.urgent_message ("MUTANT: " ^ Syntax.string_of_term_global thy t)) mutants |
405 val _ = map (fn t => Output.urgent_message ("MUTANT: " ^ Syntax.string_of_term_global thy t)) mutants |
421 in |
406 in |
422 create_entry thy thm exec mutants mtds |
407 create_entry thy thm exec mutants mtds |
423 end |
408 end |
424 |
409 |
425 (* theory -> mtd list -> thm list -> report *) |
|
426 val mutate_theorems = map oooo mutate_theorem |
|
427 |
|
428 fun string_of_mutant_subentry thy thm_name (t, results) = |
|
429 "mutant: " ^ Syntax.string_of_term_global thy t ^ "\n" ^ |
|
430 space_implode "; " |
|
431 (map (fn (mtd_name, (outcome, timing)) => mtd_name ^ ": " ^ string_of_outcome outcome) results) ^ |
|
432 "\n" |
|
433 |
|
434 (* string -> string *) |
410 (* string -> string *) |
435 val unyxml = XML.content_of o YXML.parse_body |
411 val unyxml = XML.content_of o YXML.parse_body |
436 |
412 |
437 fun string_of_mutant_subentry' thy thm_name (t, results) = |
413 fun string_of_mutant_subentry' thy thm_name (t, results) = |
438 let |
414 let |
455 fun string_of_detailed_entry thy (thm_name, exec, t, mutant_subentries) = |
431 fun string_of_detailed_entry thy (thm_name, exec, t, mutant_subentries) = |
456 thm_name ^ " " ^ (if exec then "[exe]" else "[noexe]") ^ ": " ^ |
432 thm_name ^ " " ^ (if exec then "[exe]" else "[noexe]") ^ ": " ^ |
457 Syntax.string_of_term_global thy t ^ "\n" ^ |
433 Syntax.string_of_term_global thy t ^ "\n" ^ |
458 cat_lines (map (string_of_mutant_subentry' thy thm_name) mutant_subentries) ^ "\n" |
434 cat_lines (map (string_of_mutant_subentry' thy thm_name) mutant_subentries) ^ "\n" |
459 |
435 |
460 fun theoryfile_string_of_mutant_subentry thy thm_name (i, (t, results)) = |
|
461 "lemma " ^ thm_name ^ "_" ^ string_of_int (i + 1) ^ ":\n" ^ |
|
462 "\"" ^ unyxml (Syntax.string_of_term_global thy t) ^ |
|
463 "\" \nquickcheck\noops\n" |
|
464 |
|
465 fun theoryfile_string_of_detailed_entry thy (thm_name, exec, t, mutant_subentries) = |
|
466 "subsubsection {* mutants of " ^ thm_name ^ " *}\n\n" ^ |
|
467 cat_lines (map_index |
|
468 (theoryfile_string_of_mutant_subentry thy thm_name) mutant_subentries) ^ "\n" |
|
469 |
|
470 (* subentry -> string *) |
436 (* subentry -> string *) |
471 fun string_for_subentry (mtd_name, genuine_cex, potential_cex, no_cex, donno, |
437 fun string_for_subentry (mtd_name, genuine_cex, potential_cex, no_cex, donno, |
472 timeout, error) = |
438 timeout, error) = |
473 " " ^ mtd_name ^ ": " ^ string_of_int genuine_cex ^ "+ " ^ |
439 " " ^ mtd_name ^ ": " ^ string_of_int genuine_cex ^ "+ " ^ |
474 string_of_int potential_cex ^ "= " ^ string_of_int no_cex ^ "- " ^ |
440 string_of_int potential_cex ^ "= " ^ string_of_int no_cex ^ "- " ^ |