src/Tools/quickcheck.ML
changeset 40648 1598ec648b0d
parent 40647 6e92ca8e981b
child 40656 36ca3fad1f31
equal deleted inserted replaced
40647:6e92ca8e981b 40648:1598ec648b0d
    14   val iterations : int Config.T
    14   val iterations : int Config.T
    15   val no_assms : bool Config.T
    15   val no_assms : bool Config.T
    16   val report : bool Config.T
    16   val report : bool Config.T
    17   val quiet : bool Config.T
    17   val quiet : bool Config.T
    18   val timeout : real Config.T
    18   val timeout : real Config.T
       
    19   val finite_types : bool Config.T
       
    20   val finite_type_size : int Config.T
    19   datatype report = Report of
    21   datatype report = Report of
    20     { iterations : int, raised_match_errors : int,
    22     { iterations : int, raised_match_errors : int,
    21       satisfied_assms : int list, positive_concl_tests : int }
    23       satisfied_assms : int list, positive_concl_tests : int }
    22   datatype expectation = No_Expectation | No_Counterexample | Counterexample;  
    24   datatype expectation = No_Expectation | No_Counterexample | Counterexample;  
    23   datatype test_params = Test_Params of {default_type: typ list, expect : expectation};
    25   datatype test_params = Test_Params of {default_type: typ list, expect : expectation};
    30   (* testing terms and proof states *)
    32   (* testing terms and proof states *)
    31   val test_term_small: Proof.context -> term ->
    33   val test_term_small: Proof.context -> term ->
    32     (string * term) list option * ((string * int) list * ((int * report list) list) option)
    34     (string * term) list option * ((string * int) list * ((int * report list) list) option)
    33   val test_term: Proof.context -> string option -> term ->
    35   val test_term: Proof.context -> string option -> term ->
    34     (string * term) list option * ((string * int) list * ((int * report list) list) option)
    36     (string * term) list option * ((string * int) list * ((int * report list) list) option)
       
    37   val test_goal_terms:
       
    38     Proof.context -> string option * (string * typ) list -> term list
       
    39       -> (string * term) list option * ((string * int) list * ((int * report list) list) option) list
    35   val quickcheck: (string * string list) list -> int -> Proof.state -> (string * term) list option
    40   val quickcheck: (string * string list) list -> int -> Proof.state -> (string * term) list option
    36 end;
    41 end;
    37 
    42 
    38 structure Quickcheck : QUICKCHECK =
    43 structure Quickcheck : QUICKCHECK =
    39 struct
    44 struct
   197      val result = with_size 1 [comp_time]
   202      val result = with_size 1 [comp_time]
   198    in
   203    in
   199      apsnd (rpair NONE) (apfst (Option.map (curry (op ~~) names)) result)
   204      apsnd (rpair NONE) (apfst (Option.map (curry (op ~~) names)) result)
   200    end
   205    end
   201 
   206 
       
   207 (* we actually assume we know the generators and its behaviour *)
       
   208 fun is_iteratable "small" = false
       
   209   | is_iteratable "random" = true
       
   210   
   202 fun test_term ctxt generator_name t =
   211 fun test_term ctxt generator_name t =
   203   let
   212   let
   204     val (names, t') = prep_test_term t;
   213     val (names, t') = prep_test_term t;
   205     val current_size = Unsynchronized.ref 0
   214     val current_size = Unsynchronized.ref 0
   206     fun excipit s =
   215     fun excipit s =
   221         end
   230         end
   222     val empty_report = Report { iterations = 0, raised_match_errors = 0,
   231     val empty_report = Report { iterations = 0, raised_match_errors = 0,
   223       satisfied_assms = [], positive_concl_tests = 0 }
   232       satisfied_assms = [], positive_concl_tests = 0 }
   224     fun with_testers k [] = (NONE, [])
   233     fun with_testers k [] = (NONE, [])
   225       | with_testers k (tester :: testers) =
   234       | with_testers k (tester :: testers) =
   226           case iterate (fn () => tester (k - 1)) (Config.get ctxt iterations) empty_report
   235         let
       
   236           val niterations = case generator_name of
       
   237             SOME generator_name =>
       
   238               if is_iteratable generator_name then Config.get ctxt iterations else 1
       
   239           | NONE => Config.get ctxt iterations
       
   240           val (result, timing) = cpu_time ("size " ^ string_of_int k)
       
   241             (fn () => iterate (fn () => tester (k - 1)) niterations empty_report)
       
   242         in
       
   243           case result
   227            of (NONE, report) => apsnd (cons report) (with_testers k testers)
   244            of (NONE, report) => apsnd (cons report) (with_testers k testers)
   228             | (SOME q, report) => (SOME q, [report]);
   245             | (SOME q, report) => (SOME q, [report])
       
   246         end
   229     fun with_size k reports =
   247     fun with_size k reports =
   230       if k > Config.get ctxt size then (NONE, reports)
   248       if k > Config.get ctxt size then (NONE, reports)
   231       else
   249       else
   232        (if Config.get ctxt quiet then () else Output.urgent_message ("Test data size: " ^ string_of_int k);
   250        (if Config.get ctxt quiet then () else Output.urgent_message ("Test data size: " ^ string_of_int k);
   233         let
   251         let
   234           val _ = current_size := k  
   252           val _ = current_size := k
   235           val (result, new_report) = with_testers k testers
   253           val (result, new_report) = with_testers k testers
   236           val reports = ((k, new_report) :: reports)
   254           val reports = ((k, new_report) :: reports)
   237         in case result of NONE => with_size (k + 1) reports | SOME q => (SOME q, reports) end);
   255         in case result of NONE => with_size (k + 1) reports | SOME q => (SOME q, reports) end);
   238     val ((result, reports), exec_time) =
   256     val ((result, reports), exec_time) =
   239       TimeLimit.timeLimit (seconds (Config.get ctxt timeout)) (fn () => cpu_time "quickcheck execution"
   257       TimeLimit.timeLimit (seconds (Config.get ctxt timeout)) (fn () => cpu_time "quickcheck execution"
   270       | subst T = T;
   288       | subst T = T;
   271   in (map_types o map_atyps) subst end;
   289   in (map_types o map_atyps) subst end;
   272 
   290 
   273 datatype wellsorted_error = Wellsorted_Error of string | Term of term
   291 datatype wellsorted_error = Wellsorted_Error of string | Term of term
   274 
   292 
   275 fun test_goal (generator_name, insts) i state =
   293 fun test_goal_terms lthy (generator_name, insts) check_goals =
   276   let
   294   let
   277     val lthy = Proof.context_of state;
   295     val thy = ProofContext.theory_of lthy 
   278     val thy = Proof.theory_of state;
       
   279     fun strip (Const ("all", _) $ Abs (_, _, t)) = strip t
       
   280       | strip t = t;
       
   281     val {goal = st, ...} = Proof.raw_goal state;
       
   282     val (gi, frees) = Logic.goal_params (prop_of st) i;
       
   283     val some_locale = case (Option.map #target o Named_Target.peek) lthy
       
   284      of NONE => NONE
       
   285       | SOME "" => NONE
       
   286       | SOME locale => SOME locale;
       
   287     val assms = if Config.get lthy no_assms then [] else case some_locale
       
   288      of NONE => Assumption.all_assms_of lthy
       
   289       | SOME locale => Assumption.local_assms_of lthy (Locale.init locale thy);
       
   290     val proto_goal = Logic.list_implies (map Thm.term_of assms, subst_bounds (frees, strip gi));
       
   291     val check_goals = case some_locale
       
   292      of NONE => [proto_goal]
       
   293       | SOME locale => map (fn (_, phi) => Morphism.term phi proto_goal)
       
   294         (Locale.registrations_of (Context.Theory thy) (*FIXME*) locale);
       
   295     val inst_goals =
   296     val inst_goals =
   296       if Config.get lthy finite_types then 
   297       if Config.get lthy finite_types then 
   297         maps (fn check_goal => map (fn T =>
   298         maps (fn check_goal => map (fn T =>
   298           Term ((Object_Logic.atomize_term thy o monomorphic_term thy insts T) check_goal)
   299           Term ((Object_Logic.atomize_term thy o monomorphic_term thy insts T) check_goal)
   299             handle WELLSORTED s => Wellsorted_Error s) (get_finite_types lthy)) check_goals
   300             handle WELLSORTED s => Wellsorted_Error s) (get_finite_types lthy)) check_goals
   312         case f t of
   313         case f t of
   313           (SOME res, report) => (SOME res, rev (report :: reports))
   314           (SOME res, report) => (SOME res, rev (report :: reports))
   314         | (NONE, report) => collect_results f (report :: reports) ts
   315         | (NONE, report) => collect_results f (report :: reports) ts
   315   in collect_results (test_term lthy generator_name) [] correct_inst_goals end;
   316   in collect_results (test_term lthy generator_name) [] correct_inst_goals end;
   316 
   317 
       
   318 fun test_goal (generator_name, insts) i state =
       
   319   let
       
   320     val lthy = Proof.context_of state;
       
   321     val thy = Proof.theory_of state;
       
   322     fun strip (Const ("all", _) $ Abs (_, _, t)) = strip t
       
   323       | strip t = t;
       
   324     val {goal = st, ...} = Proof.raw_goal state;
       
   325     val (gi, frees) = Logic.goal_params (prop_of st) i;
       
   326     val some_locale = case (Option.map #target o Named_Target.peek) lthy
       
   327      of NONE => NONE
       
   328       | SOME "" => NONE
       
   329       | SOME locale => SOME locale;
       
   330     val assms = if Config.get lthy no_assms then [] else case some_locale
       
   331      of NONE => Assumption.all_assms_of lthy
       
   332       | SOME locale => Assumption.local_assms_of lthy (Locale.init locale thy);
       
   333     val proto_goal = Logic.list_implies (map Thm.term_of assms, subst_bounds (frees, strip gi));
       
   334     val check_goals = case some_locale
       
   335      of NONE => [proto_goal]
       
   336       | SOME locale => map (fn (_, phi) => Morphism.term phi proto_goal)
       
   337         (Locale.registrations_of (Context.Theory thy) (*FIXME*) locale);
       
   338   in
       
   339     test_goal_terms lthy (generator_name, insts) check_goals
       
   340   end
       
   341 
   317 (* pretty printing *)
   342 (* pretty printing *)
   318 
   343 
   319 fun tool_name auto = (if auto then "Auto " else "") ^ "Quickcheck"
   344 fun tool_name auto = (if auto then "Auto " else "") ^ "Quickcheck"
   320 
   345 
   321 fun pretty_counterex ctxt auto NONE = Pretty.str (tool_name auto ^ " found no counterexample.")
   346 fun pretty_counterex ctxt auto NONE = Pretty.str (tool_name auto ^ " found no counterexample.")