src/Tools/quickcheck.ML
changeset 41517 7267fb5b724b
parent 41472 f6ab14e61604
child 41735 bd7ee90267f2
equal deleted inserted replaced
41516:3a70387b5e01 41517:7267fb5b724b
     8 sig
     8 sig
     9   val setup: theory -> theory
     9   val setup: theory -> theory
    10   (* configuration *)
    10   (* configuration *)
    11   val auto: bool Unsynchronized.ref
    11   val auto: bool Unsynchronized.ref
    12   val timing : bool Unsynchronized.ref
    12   val timing : bool Unsynchronized.ref
    13   val tester : string Config.T 
    13   val tester : string Config.T
    14   val size : int Config.T
    14   val size : int Config.T
    15   val iterations : int Config.T
    15   val iterations : int Config.T
    16   val no_assms : bool Config.T
    16   val no_assms : bool Config.T
    17   val report : bool Config.T
    17   val report : bool Config.T
    18   val quiet : bool Config.T
    18   val quiet : bool Config.T
    20   val finite_types : bool Config.T
    20   val finite_types : bool Config.T
    21   val finite_type_size : int Config.T
    21   val finite_type_size : int Config.T
    22   datatype report = Report of
    22   datatype report = Report of
    23     { iterations : int, raised_match_errors : int,
    23     { iterations : int, raised_match_errors : int,
    24       satisfied_assms : int list, positive_concl_tests : int }
    24       satisfied_assms : int list, positive_concl_tests : int }
    25   datatype expectation = No_Expectation | No_Counterexample | Counterexample;  
    25   datatype expectation = No_Expectation | No_Counterexample | Counterexample;
    26   datatype test_params = Test_Params of {default_type: typ list, expect : expectation};
    26   datatype test_params = Test_Params of {default_type: typ list, expect : expectation};
    27   val test_params_of : Proof.context -> test_params
    27   val test_params_of : Proof.context -> test_params
    28   val map_test_params : (typ list * expectation -> typ list * expectation)
    28   val map_test_params : (typ list * expectation -> typ list * expectation)
    29     -> Context.generic -> Context.generic
    29     -> Context.generic -> Context.generic
    30   val add_generator:
    30   val add_generator:
    61   { iterations : int, raised_match_errors : int,
    61   { iterations : int, raised_match_errors : int,
    62     satisfied_assms : int list, positive_concl_tests : int }
    62     satisfied_assms : int list, positive_concl_tests : int }
    63 
    63 
    64 (* expectation *)
    64 (* expectation *)
    65 
    65 
    66 datatype expectation = No_Expectation | No_Counterexample | Counterexample; 
    66 datatype expectation = No_Expectation | No_Counterexample | Counterexample;
    67 
    67 
    68 fun merge_expectation (expect1, expect2) =
    68 fun merge_expectation (expect1, expect2) =
    69   if expect1 = expect2 then expect1 else No_Expectation
    69   if expect1 = expect2 then expect1 else No_Expectation
    70 
    70 
    71 (* quickcheck configuration -- default parameters, test generators *)
    71 (* quickcheck configuration -- default parameters, test generators *)
    75 val (no_assms, setup_no_assms) = Attrib.config_bool "quickcheck_no_assms" (K false)
    75 val (no_assms, setup_no_assms) = Attrib.config_bool "quickcheck_no_assms" (K false)
    76 val (report, setup_report) = Attrib.config_bool "quickcheck_report" (K true)
    76 val (report, setup_report) = Attrib.config_bool "quickcheck_report" (K true)
    77 val (quiet, setup_quiet) = Attrib.config_bool "quickcheck_quiet" (K false)
    77 val (quiet, setup_quiet) = Attrib.config_bool "quickcheck_quiet" (K false)
    78 val (timeout, setup_timeout) = Attrib.config_real "quickcheck_timeout" (K 30.0)
    78 val (timeout, setup_timeout) = Attrib.config_real "quickcheck_timeout" (K 30.0)
    79 val (finite_types, setup_finite_types) = Attrib.config_bool "quickcheck_finite_types" (K true)
    79 val (finite_types, setup_finite_types) = Attrib.config_bool "quickcheck_finite_types" (K true)
    80 val (finite_type_size, setup_finite_type_size) = Attrib.config_int "quickcheck_finite_type_size" (K 3)
    80 val (finite_type_size, setup_finite_type_size) =
       
    81   Attrib.config_int "quickcheck_finite_type_size" (K 3)
    81 
    82 
    82 val setup_config =
    83 val setup_config =
    83   setup_tester #> setup_size #> setup_iterations #> setup_no_assms #> setup_report #> setup_quiet
    84   setup_tester #> setup_size #> setup_iterations #> setup_no_assms #> setup_report #> setup_quiet
    84     #> setup_timeout #> setup_finite_types #> setup_finite_type_size
    85     #> setup_timeout #> setup_finite_types #> setup_finite_type_size
    85 
    86 
    86 datatype test_params = Test_Params of
    87 datatype test_params = Test_Params of
    87   {default_type: typ list, expect : expectation};
    88   {default_type: typ list, expect : expectation};
    88 
    89 
    89 fun dest_test_params (Test_Params {default_type, expect}) = (default_type, expect);
    90 fun dest_test_params (Test_Params {default_type, expect}) = (default_type, expect);
    90 
    91 
    91 fun make_test_params (default_type, expect) = Test_Params {default_type = default_type, expect = expect};
    92 fun make_test_params (default_type, expect) =
    92 
    93   Test_Params {default_type = default_type, expect = expect};
    93 fun map_test_params' f (Test_Params {default_type, expect}) = make_test_params (f (default_type, expect));
    94 
       
    95 fun map_test_params' f (Test_Params {default_type, expect}) =
       
    96   make_test_params (f (default_type, expect));
    94 
    97 
    95 fun merge_test_params
    98 fun merge_test_params
    96   (Test_Params {default_type = default_type1, expect = expect1},
    99   (Test_Params {default_type = default_type1, expect = expect1},
    97     Test_Params {default_type = default_type2, expect = expect2}) =
   100     Test_Params {default_type = default_type2, expect = expect2}) =
    98   make_test_params
   101   make_test_params
   155   let
   158   let
   156     val start = start_timing ()
   159     val start = start_timing ()
   157     val result = Exn.capture f ()
   160     val result = Exn.capture f ()
   158     val time = Time.toMilliseconds (#cpu (end_timing start))
   161     val time = Time.toMilliseconds (#cpu (end_timing start))
   159   in (Exn.release result, (description, time)) end
   162   in (Exn.release result, (description, time)) end
   160   
   163 
   161 fun test_term ctxt is_interactive t =
   164 fun test_term ctxt is_interactive t =
   162   let
   165   let
   163     val (names, t') = apfst (map fst) (prep_test_term t);
   166     val (names, t') = apfst (map fst) (prep_test_term t);
   164     val current_size = Unsynchronized.ref 0
   167     val current_size = Unsynchronized.ref 0
   165     fun excipit s =
   168     fun excipit s =
   174             ("Test data size: " ^ string_of_int k)
   177             ("Test data size: " ^ string_of_int k)
   175           val _ = current_size := k
   178           val _ = current_size := k
   176           val ((result, new_report), timing) =
   179           val ((result, new_report), timing) =
   177             cpu_time ("size " ^ string_of_int k) (fn () => test_fun (k - 1))
   180             cpu_time ("size " ^ string_of_int k) (fn () => test_fun (k - 1))
   178           val reports = case new_report of NONE => reports | SOME rep => (k, rep) :: reports
   181           val reports = case new_report of NONE => reports | SOME rep => (k, rep) :: reports
   179         in case result of NONE => with_size test_fun (k + 1) reports | SOME q => (SOME q, reports) end;
   182         in
   180   in
   183           case result of NONE => with_size test_fun (k + 1) reports | SOME q => (SOME q, reports)
   181     case test_fun of NONE => (NONE, ([comp_time], NONE)) 
   184         end;
       
   185   in
       
   186     case test_fun of NONE => (NONE, ([comp_time], NONE))
   182       | SOME test_fun =>
   187       | SOME test_fun =>
   183         TimeLimit.timeLimit (seconds (Config.get ctxt timeout)) (fn () =>
   188         TimeLimit.timeLimit (seconds (Config.get ctxt timeout)) (fn () =>
   184           let
   189           let
   185             val ((result, reports), exec_time) =
   190             val ((result, reports), exec_time) =
   186               cpu_time "quickcheck execution" (fn () => with_size test_fun 1 [])
   191               cpu_time "quickcheck execution" (fn () => with_size test_fun 1 [])
   223         TimeLimit.timeLimit (seconds (Config.get ctxt timeout)) (fn () =>
   228         TimeLimit.timeLimit (seconds (Config.get ctxt timeout)) (fn () =>
   224           (get_first test_card_size enumeration_card_size, ([comp_time], NONE))) ()
   229           (get_first test_card_size enumeration_card_size, ([comp_time], NONE))) ()
   225         handle TimeLimit.TimeOut =>
   230         handle TimeLimit.TimeOut =>
   226           if is_interactive then error ("Quickcheck ran out of time") else raise TimeLimit.TimeOut
   231           if is_interactive then error ("Quickcheck ran out of time") else raise TimeLimit.TimeOut
   227       else
   232       else
   228         error "Unexpectedly, testers of some cardinalities are executable but others are not"       
   233         error "Unexpectedly, testers of some cardinalities are executable but others are not"
   229     end
   234     end
   230 
   235 
   231 fun get_finite_types ctxt =
   236 fun get_finite_types ctxt =
   232   fst (chop (Config.get ctxt finite_type_size)
   237   fst (chop (Config.get ctxt finite_type_size)
   233     (map (Type o rpair []) ["Enum.finite_1", "Enum.finite_2", "Enum.finite_3",
   238     (map (Type o rpair []) ["Enum.finite_1", "Enum.finite_2", "Enum.finite_3",
   234      "Enum.finite_4", "Enum.finite_5"]))  
   239      "Enum.finite_4", "Enum.finite_5"]))
   235 
   240 
   236 exception WELLSORTED of string
   241 exception WELLSORTED of string
   237 
   242 
   238 fun monomorphic_term thy insts default_T = 
   243 fun monomorphic_term thy insts default_T =
   239   let
   244   let
   240     fun subst (T as TFree (v, S)) =
   245     fun subst (T as TFree (v, S)) =
   241       let
   246       let
   242         val T' = AList.lookup (op =) insts v
   247         val T' = AList.lookup (op =) insts v
   243           |> the_default default_T
   248           |> the_default default_T
   244       in if Sign.of_sort thy (T', S) then T'
   249       in if Sign.of_sort thy (T', S) then T'
   245         else raise (WELLSORTED ("For instantiation with default_type " ^ Syntax.string_of_typ_global thy default_T ^
   250         else raise (WELLSORTED ("For instantiation with default_type " ^
       
   251           Syntax.string_of_typ_global thy default_T ^
   246           ":\n" ^ Syntax.string_of_typ_global thy T' ^
   252           ":\n" ^ Syntax.string_of_typ_global thy T' ^
   247           " to be substituted for variable " ^
   253           " to be substituted for variable " ^
   248           Syntax.string_of_typ_global thy T ^ " does not have sort " ^
   254           Syntax.string_of_typ_global thy T ^ " does not have sort " ^
   249           Syntax.string_of_sort_global thy S))
   255           Syntax.string_of_sort_global thy S))
   250       end
   256       end
   253 
   259 
   254 datatype wellsorted_error = Wellsorted_Error of string | Term of term
   260 datatype wellsorted_error = Wellsorted_Error of string | Term of term
   255 
   261 
   256 fun test_goal_terms lthy is_interactive insts check_goals =
   262 fun test_goal_terms lthy is_interactive insts check_goals =
   257   let
   263   let
   258     val thy = ProofContext.theory_of lthy 
   264     val thy = ProofContext.theory_of lthy
   259     val default_insts =
   265     val default_insts =
   260       if Config.get lthy finite_types then (get_finite_types lthy) else (default_type lthy)
   266       if Config.get lthy finite_types then (get_finite_types lthy) else (default_type lthy)
   261     val inst_goals =
   267     val inst_goals =
   262       map (fn check_goal =>
   268       map (fn check_goal =>
   263         if not (null (Term.add_tfree_names check_goal [])) then
   269         if not (null (Term.add_tfree_names check_goal [])) then
   264           map (fn T =>
   270           map (fn T =>
   265             (pair (SOME T) o Term o Object_Logic.atomize_term thy o monomorphic_term thy insts T) check_goal
   271             (pair (SOME T) o Term o Object_Logic.atomize_term thy o monomorphic_term thy insts T)
       
   272               check_goal
   266               handle WELLSORTED s => (SOME T, Wellsorted_Error s)) default_insts
   273               handle WELLSORTED s => (SOME T, Wellsorted_Error s)) default_insts
   267         else
   274         else
   268           [(NONE, Term (Object_Logic.atomize_term thy check_goal))]) check_goals
   275           [(NONE, Term (Object_Logic.atomize_term thy check_goal))]) check_goals
   269     val error_msg = cat_lines (maps (map_filter (fn (_, Term t) => NONE | (_, Wellsorted_Error s) => SOME s)) inst_goals)
   276     val error_msg =
       
   277       cat_lines
       
   278         (maps (map_filter (fn (_, Term t) => NONE | (_, Wellsorted_Error s) => SOME s)) inst_goals)
   270     fun is_wellsorted_term (T, Term t) = SOME (T, t)
   279     fun is_wellsorted_term (T, Term t) = SOME (T, t)
   271       | is_wellsorted_term (_, Wellsorted_Error s) = NONE
   280       | is_wellsorted_term (_, Wellsorted_Error s) = NONE
   272     val correct_inst_goals =
   281     val correct_inst_goals =
   273       case map (map_filter is_wellsorted_term) inst_goals of
   282       case map (map_filter is_wellsorted_term) inst_goals of
   274         [[]] => error error_msg
   283         [[]] => error error_msg
   280           (SOME res, report) => (SOME res, rev (report :: reports))
   289           (SOME res, report) => (SOME res, rev (report :: reports))
   281         | (NONE, report) => collect_results f (report :: reports) ts
   290         | (NONE, report) => collect_results f (report :: reports) ts
   282     fun test_term' goal =
   291     fun test_term' goal =
   283       case goal of
   292       case goal of
   284         [(NONE, t)] => test_term lthy is_interactive t
   293         [(NONE, t)] => test_term lthy is_interactive t
   285       | ts => test_term_with_increasing_cardinality lthy is_interactive (map snd ts)                           
   294       | ts => test_term_with_increasing_cardinality lthy is_interactive (map snd ts)
   286   in
   295   in
   287     if Config.get lthy finite_types then
   296     if Config.get lthy finite_types then
   288       collect_results test_term' [] correct_inst_goals
   297       collect_results test_term' [] correct_inst_goals
   289     else
   298     else
   290       collect_results (test_term lthy is_interactive) [] (maps (map snd) correct_inst_goals)
   299       collect_results (test_term lthy is_interactive) [] (maps (map snd) correct_inst_goals)
   329   let
   338   let
   330     fun pretty_stat s i = Pretty.block ([Pretty.str (s ^ ": " ^ string_of_int i)])
   339     fun pretty_stat s i = Pretty.block ([Pretty.str (s ^ ": " ^ string_of_int i)])
   331   in
   340   in
   332      ([pretty_stat "iterations" iterations,
   341      ([pretty_stat "iterations" iterations,
   333      pretty_stat "match exceptions" raised_match_errors]
   342      pretty_stat "match exceptions" raised_match_errors]
   334      @ map_index (fn (i, n) => pretty_stat ("satisfied " ^ string_of_int (i + 1) ^ ". assumption") n)
   343      @ map_index
       
   344        (fn (i, n) => pretty_stat ("satisfied " ^ string_of_int (i + 1) ^ ". assumption") n)
   335        satisfied_assms
   345        satisfied_assms
   336      @ [pretty_stat "positive conclusion tests" positive_concl_tests])
   346      @ [pretty_stat "positive conclusion tests" positive_concl_tests])
   337   end
   347   end
   338 
   348 
   339 fun pretty_reports ctxt (SOME reports) =
   349 fun pretty_reports ctxt (SOME reports) =
   382   case (Real.fromString s) of
   392   case (Real.fromString s) of
   383     SOME s => s
   393     SOME s => s
   384   | NONE => error ("Not a real number: " ^ s)
   394   | NONE => error ("Not a real number: " ^ s)
   385 
   395 
   386 fun read_expectation "no_expectation" = No_Expectation
   396 fun read_expectation "no_expectation" = No_Expectation
   387   | read_expectation "no_counterexample" = No_Counterexample 
   397   | read_expectation "no_counterexample" = No_Counterexample
   388   | read_expectation "counterexample" = Counterexample
   398   | read_expectation "counterexample" = Counterexample
   389   | read_expectation s = error ("Not an expectation value: " ^ s)  
   399   | read_expectation s = error ("Not an expectation value: " ^ s)
   390 
   400 
   391 fun valid_tester_name genctxt = AList.defined (op =) (fst (Data.get genctxt))
   401 fun valid_tester_name genctxt = AList.defined (op =) (fst (Data.get genctxt))
   392 
   402 
   393 fun parse_tester name genctxt =
   403 fun parse_tester name genctxt =
   394   if valid_tester_name genctxt name then
   404   if valid_tester_name genctxt name then
   398 
   408 
   399 fun parse_test_param ("tester", [arg]) = parse_tester arg
   409 fun parse_test_param ("tester", [arg]) = parse_tester arg
   400   | parse_test_param ("size", [arg]) = Config.put_generic size (read_nat arg)
   410   | parse_test_param ("size", [arg]) = Config.put_generic size (read_nat arg)
   401   | parse_test_param ("iterations", [arg]) = Config.put_generic iterations (read_nat arg)
   411   | parse_test_param ("iterations", [arg]) = Config.put_generic iterations (read_nat arg)
   402   | parse_test_param ("default_type", arg) = (fn gen_ctxt =>
   412   | parse_test_param ("default_type", arg) = (fn gen_ctxt =>
   403     map_test_params ((apfst o K) (map (ProofContext.read_typ (Context.proof_of gen_ctxt)) arg)) gen_ctxt)
   413     map_test_params
       
   414       ((apfst o K) (map (ProofContext.read_typ (Context.proof_of gen_ctxt)) arg)) gen_ctxt)
   404   | parse_test_param ("no_assms", [arg]) = Config.put_generic no_assms (read_bool arg)
   415   | parse_test_param ("no_assms", [arg]) = Config.put_generic no_assms (read_bool arg)
   405   | parse_test_param ("expect", [arg]) = map_test_params ((apsnd o K) (read_expectation arg))
   416   | parse_test_param ("expect", [arg]) = map_test_params ((apsnd o K) (read_expectation arg))
   406   | parse_test_param ("report", [arg]) = Config.put_generic report (read_bool arg)
   417   | parse_test_param ("report", [arg]) = Config.put_generic report (read_bool arg)
   407   | parse_test_param ("quiet", [arg]) = Config.put_generic quiet (read_bool arg)
   418   | parse_test_param ("quiet", [arg]) = Config.put_generic quiet (read_bool arg)
   408   | parse_test_param ("timeout", [arg]) = Config.put_generic timeout (read_real arg)
   419   | parse_test_param ("timeout", [arg]) = Config.put_generic timeout (read_real arg)
   409   | parse_test_param ("finite_types", [arg]) = Config.put_generic finite_types (read_bool arg)
   420   | parse_test_param ("finite_types", [arg]) = Config.put_generic finite_types (read_bool arg)
   410   | parse_test_param ("finite_type_size", [arg]) = Config.put_generic finite_type_size (read_nat arg)
   421   | parse_test_param ("finite_type_size", [arg]) =
       
   422     Config.put_generic finite_type_size (read_nat arg)
   411   | parse_test_param (name, _) = fn genctxt =>
   423   | parse_test_param (name, _) = fn genctxt =>
   412     if valid_tester_name genctxt name then
   424     if valid_tester_name genctxt name then
   413       Config.put_generic tester name genctxt
   425       Config.put_generic tester name genctxt
   414     else error ("Unknown tester or test parameter: " ^ name);
   426     else error ("Unknown tester or test parameter: " ^ name);
   415 
   427 
   418        of SOME (TFree (v, _)) => (apfst o AList.update (op =))
   430        of SOME (TFree (v, _)) => (apfst o AList.update (op =))
   419               (v, ProofContext.read_typ ctxt (the_single arg)) (insts, ctxt)
   431               (v, ProofContext.read_typ ctxt (the_single arg)) (insts, ctxt)
   420         | _ => (apsnd o Context.proof_map o parse_test_param) (name, arg) (insts, ctxt);
   432         | _ => (apsnd o Context.proof_map o parse_test_param) (name, arg) (insts, ctxt);
   421 
   433 
   422 fun quickcheck_params_cmd args = Context.theory_map (fold parse_test_param args);
   434 fun quickcheck_params_cmd args = Context.theory_map (fold parse_test_param args);
   423   
   435 
   424 fun gen_quickcheck args i state =
   436 fun gen_quickcheck args i state =
   425   state
   437   state
   426   |> Proof.map_context_result (fn ctxt => fold parse_test_param_inst args ([], ctxt))
   438   |> Proof.map_context_result (fn ctxt => fold parse_test_param_inst args ([], ctxt))
   427   |> (fn (insts, state') => test_goal insts i state'
   439   |> (fn (insts, state') => test_goal insts i state'
   428   |> tap (fn (SOME x, _) => if expect (Proof.context_of state') = No_Counterexample then
   440   |> tap (fn (SOME x, _) => if expect (Proof.context_of state') = No_Counterexample then
   434 
   446 
   435 fun quickcheck_cmd args i state =
   447 fun quickcheck_cmd args i state =
   436   gen_quickcheck args i (Toplevel.proof_of state)
   448   gen_quickcheck args i (Toplevel.proof_of state)
   437   |> Pretty.writeln o pretty_counterex_and_reports (Toplevel.context_of state) false;
   449   |> Pretty.writeln o pretty_counterex_and_reports (Toplevel.context_of state) false;
   438 
   450 
   439 val parse_arg = Parse.name -- (Scan.optional (Parse.$$$ "=" |-- 
   451 val parse_arg =
   440   (((Parse.name || Parse.float_number) >> single) || (Parse.$$$ "[" |-- Parse.list1 Parse.name --| Parse.$$$ "]"))) ["true"]);
   452   Parse.name --
   441 
   453     (Scan.optional (Parse.$$$ "=" |--
   442 val parse_args = Parse.$$$ "[" |-- Parse.list1 parse_arg --| Parse.$$$ "]"
   454       (((Parse.name || Parse.float_number) >> single) ||
   443   || Scan.succeed [];
   455         (Parse.$$$ "[" |-- Parse.list1 Parse.name --| Parse.$$$ "]"))) ["true"]);
       
   456 
       
   457 val parse_args =
       
   458   Parse.$$$ "[" |-- Parse.list1 parse_arg --| Parse.$$$ "]" || Scan.succeed [];
   444 
   459 
   445 val _ =
   460 val _ =
   446   Outer_Syntax.command "quickcheck_params" "set parameters for random testing" Keyword.thy_decl
   461   Outer_Syntax.command "quickcheck_params" "set parameters for random testing" Keyword.thy_decl
   447     (parse_args >> (fn args => Toplevel.theory (quickcheck_params_cmd args)));
   462     (parse_args >> (fn args => Toplevel.theory (quickcheck_params_cmd args)));
   448 
   463