src/Tools/quickcheck.ML
changeset 43876 b8fa7287ee4c
parent 43875 485d2ad43528
child 43877 abd1f074cb98
equal deleted inserted replaced
43875:485d2ad43528 43876:b8fa7287ee4c
    54       -> Context.generic -> Context.generic
    54       -> Context.generic -> Context.generic
    55   val add_batch_validator:
    55   val add_batch_validator:
    56     string * (Proof.context -> term list -> (int -> bool) list)
    56     string * (Proof.context -> term list -> (int -> bool) list)
    57       -> Context.generic -> Context.generic
    57       -> Context.generic -> Context.generic
    58   (* basic operations *)
    58   (* basic operations *)
       
    59   type compile_generator =
       
    60     Proof.context -> (term * term list) list -> int list -> term list option * report option
    59   val limit : Time.time -> (bool * bool) -> (unit -> 'a) -> (unit -> 'a) -> unit -> 'a  
    61   val limit : Time.time -> (bool * bool) -> (unit -> 'a) -> (unit -> 'a) -> unit -> 'a  
    60   val instantiate_goals: Proof.context -> (string * typ) list -> (term * term list) list
    62   val instantiate_goals: Proof.context -> (string * typ) list -> (term * term list) list
    61     -> (typ option * (term * term list)) list list
    63     -> (typ option * (term * term list)) list list
    62   val collect_results: ('a -> result) -> 'a list -> result list -> result list
    64   val collect_results: ('a -> result) -> 'a list -> result list -> result list
    63   val generator_test_goal_terms: Proof.context -> bool * bool -> (string * typ) list ->
    65   val generator_test_goal_terms: compile_generator ->
    64     (term * term list) list -> result list
    66     Proof.context -> bool * bool -> (string * typ) list -> (term * term list) list -> result list
    65   (* testing terms and proof states *)
    67   (* testing terms and proof states *)
    66   val test_term: Proof.context -> bool * bool -> term * term list -> result
    68   val test_term: compile_generator -> Proof.context -> bool * bool -> term * term list -> result
    67   val test_goal_terms:
    69   val test_goal_terms:
    68     Proof.context -> bool * bool -> (string * typ) list -> (term * term list) list  
    70     Proof.context -> bool * bool -> (string * typ) list -> (term * term list) list -> result list
    69       -> result list
       
    70   val quickcheck: (string * string list) list -> int -> Proof.state -> (string * term) list option
    71   val quickcheck: (string * string list) list -> int -> Proof.state -> (string * term) list option
    71   (* testing a batch of terms *)
    72   (* testing a batch of terms *)
    72   val test_terms:
    73   val test_terms:
    73     Proof.context -> term list -> (string * term) list option list option * (string * int) list
    74     Proof.context -> term list -> (string * term) list option list option * (string * int) list
    74   val validate_terms: Proof.context -> term list -> bool list option * (string * int) list
    75   val validate_terms: Proof.context -> term list -> bool list option * (string * int) list
   244       in case Exn.get_res tester of
   245       in case Exn.get_res tester of
   245           NONE => SOME (Exn.release tester)
   246           NONE => SOME (Exn.release tester)
   246         | SOME tester => SOME tester
   247         | SOME tester => SOME tester
   247       end
   248       end
   248   end
   249   end
   249 
   250 (*
   250 val mk_tester =
   251 val mk_tester =
   251   gen_mk_tester (fn ctxt => AList.lookup (op =) ((fst o fst o fst o Data.get o Context.Proof) ctxt))
   252   gen_mk_tester (fn ctxt => AList.lookup (op =) ((fst o fst o fst o Data.get o Context.Proof) ctxt))
       
   253 *)
   252 val mk_batch_tester =
   254 val mk_batch_tester =
   253   gen_mk_tester (fn ctxt => AList.lookup (op =) ((fst o snd o fst o Data.get o Context.Proof) ctxt))
   255   gen_mk_tester (fn ctxt => AList.lookup (op =) ((fst o snd o fst o Data.get o Context.Proof) ctxt))
   254 val mk_batch_validator =
   256 val mk_batch_validator =
   255   gen_mk_tester (fn ctxt => AList.lookup (op =) ((snd o snd o fst o Data.get o Context.Proof) ctxt))
   257   gen_mk_tester (fn ctxt => AList.lookup (op =) ((snd o snd o fst o Data.get o Context.Proof) ctxt))
   256 
   258 
   257   
   259   
   258 fun lookup_tester ctxt = AList.lookup (op =) ((snd o fst o fst o Data.get o Context.Proof) ctxt)
   260 fun lookup_tester ctxt = AList.lookup (op =) ((snd o fst o fst o Data.get o Context.Proof) ctxt)
   259 
   261 
   260 (* testing propositions *)
   262 (* testing propositions *)
       
   263 
       
   264 type compile_generator =
       
   265   Proof.context -> (term * term list) list -> int list -> term list option * report option
   261 
   266 
   262 fun check_test_term t =
   267 fun check_test_term t =
   263   let
   268   let
   264     val _ = (null (Term.add_tvars t []) andalso null (Term.add_tfrees t [])) orelse
   269     val _ = (null (Term.add_tvars t []) andalso null (Term.add_tfrees t [])) orelse
   265       error "Term to be tested contains type variables";
   270       error "Term to be tested contains type variables";
   277     handle TimeLimit.TimeOut =>
   282     handle TimeLimit.TimeOut =>
   278       if is_interactive then exc () else raise TimeLimit.TimeOut
   283       if is_interactive then exc () else raise TimeLimit.TimeOut
   279   else
   284   else
   280     f ()
   285     f ()
   281 
   286 
   282 fun test_term ctxt (limit_time, is_interactive) (t, eval_terms) =
   287 fun test_term compile ctxt (limit_time, is_interactive) (t, eval_terms) =
   283   let
   288   let
   284     fun message s = if Config.get ctxt quiet then () else Output.urgent_message s
   289     fun message s = if Config.get ctxt quiet then () else Output.urgent_message s
   285     val _ = check_test_term t
   290     val _ = check_test_term t
   286     val names = Term.add_free_names t []
   291     val names = Term.add_free_names t []
   287     val current_size = Unsynchronized.ref 0
   292     val current_size = Unsynchronized.ref 0
   304         end;
   309         end;
   305   in
   310   in
   306     limit (seconds (Config.get ctxt timeout)) (limit_time, is_interactive) (fn () =>
   311     limit (seconds (Config.get ctxt timeout)) (limit_time, is_interactive) (fn () =>
   307       let
   312       let
   308         val (test_fun, comp_time) = cpu_time "quickcheck compilation"
   313         val (test_fun, comp_time) = cpu_time "quickcheck compilation"
   309           (fn () => mk_tester ctxt [(t, eval_terms)]);
   314           (fn () => compile ctxt [(t, eval_terms)]);
   310         val _ = add_timing comp_time current_result
   315         val _ = add_timing comp_time current_result
   311       in
   316         val (response, exec_time) =
   312         case test_fun of NONE => !current_result
   317           cpu_time "quickcheck execution" (fn () => with_size test_fun 1)
   313           | SOME test_fun =>
   318         val _ = add_response names eval_terms response current_result
   314             let
   319         val _ = add_timing exec_time current_result
   315               val (response, exec_time) =
   320       in !current_result end)
   316                 cpu_time "quickcheck execution" (fn () => with_size test_fun 1)
   321     (fn () => (message (excipit ()); !current_result)) ()
   317               val _ = add_response names eval_terms response current_result
       
   318               val _ = add_timing exec_time current_result
       
   319             in
       
   320               !current_result
       
   321             end
       
   322        end)
       
   323      (fn () => (message (excipit ()); !current_result)) ()
       
   324   end;
   322   end;
   325 
   323 
   326 fun validate_terms ctxt ts =
   324 fun validate_terms ctxt ts =
   327   let
   325   let
   328     val _ = map check_test_term ts
   326     val _ = map check_test_term ts
   359   end
   357   end
   360 
   358 
   361 (* FIXME: this function shows that many assumptions are made upon the generation *)
   359 (* FIXME: this function shows that many assumptions are made upon the generation *)
   362 (* In the end there is probably no generic quickcheck interface left... *)
   360 (* In the end there is probably no generic quickcheck interface left... *)
   363 
   361 
   364 fun test_term_with_increasing_cardinality ctxt (limit_time, is_interactive) ts =
   362 fun test_term_with_increasing_cardinality compile ctxt (limit_time, is_interactive) ts =
   365   let
   363   let
   366     val thy = Proof_Context.theory_of ctxt
   364     val thy = Proof_Context.theory_of ctxt
   367     fun message s = if Config.get ctxt quiet then () else Output.urgent_message s
   365     fun message s = if Config.get ctxt quiet then () else Output.urgent_message s
   368     val (ts', eval_terms) = split_list ts
   366     val (ts', eval_terms) = split_list ts
   369     val _ = map check_test_term ts'
   367     val _ = map check_test_term ts'
   388         map_product pair (1 upto (length ts)) (1 upto (Config.get ctxt size))
   386         map_product pair (1 upto (length ts)) (1 upto (Config.get ctxt size))
   389         |> sort (fn ((c1, s1), (c2, s2)) => int_ord ((c1 + s1), (c2 + s2)))
   387         |> sort (fn ((c1, s1), (c2, s2)) => int_ord ((c1 + s1), (c2 + s2)))
   390   in
   388   in
   391     limit (seconds (Config.get ctxt timeout)) (limit_time, is_interactive) (fn () =>
   389     limit (seconds (Config.get ctxt timeout)) (limit_time, is_interactive) (fn () =>
   392       let
   390       let
   393         val (test_fun, comp_time) = cpu_time "quickcheck compilation" (fn () => mk_tester ctxt ts)
   391         val (test_fun, comp_time) = cpu_time "quickcheck compilation" (fn () => compile ctxt ts)
   394         val _ = add_timing comp_time current_result
   392         val _ = add_timing comp_time current_result     
   395       in
   393         val _ = case get_first (test_card_size test_fun) enumeration_card_size of
   396         case test_fun of
   394           SOME (card, ts) => add_response names (nth eval_terms (card - 1)) (SOME ts) current_result
   397           NONE => !current_result
   395         | NONE => ()
   398         | SOME test_fun =>
   396       in !current_result end)
   399           let
       
   400             val _ = case get_first (test_card_size test_fun) enumeration_card_size of
       
   401               SOME (card, ts) => add_response names (nth eval_terms (card - 1)) (SOME ts) current_result
       
   402             | NONE => ()
       
   403           in !current_result end
       
   404       end)
       
   405       (fn () => (message "Quickcheck ran out of time"; !current_result)) ()
   397       (fn () => (message "Quickcheck ran out of time"; !current_result)) ()
   406   end
   398   end
   407 
   399 
   408 fun get_finite_types ctxt =
   400 fun get_finite_types ctxt =
   409   fst (chop (Config.get ctxt finite_type_size)
   401   fst (chop (Config.get ctxt finite_type_size)
   469         (result :: results)
   461         (result :: results)
   470       else
   462       else
   471         collect_results f ts (result :: results)
   463         collect_results f ts (result :: results)
   472     end  
   464     end  
   473 
   465 
   474 fun generator_test_goal_terms ctxt (limit_time, is_interactive) insts goals =
   466 fun generator_test_goal_terms compile ctxt (limit_time, is_interactive) insts goals =
   475   let
   467   let
   476     fun test_term' goal =
   468     fun test_term' goal =
   477       case goal of
   469       case goal of
   478         [(NONE, t)] => test_term ctxt (limit_time, is_interactive) t
   470         [(NONE, t)] => test_term compile ctxt (limit_time, is_interactive) t
   479       | ts => test_term_with_increasing_cardinality ctxt (limit_time, is_interactive) (map snd ts)
   471       | ts => test_term_with_increasing_cardinality compile ctxt (limit_time, is_interactive) (map snd ts)
   480     val correct_inst_goals = instantiate_goals ctxt insts goals
   472     val correct_inst_goals = instantiate_goals ctxt insts goals
   481   in
   473   in
   482     if Config.get ctxt finite_types then
   474     if Config.get ctxt finite_types then
   483       collect_results test_term' correct_inst_goals []
   475       collect_results test_term' correct_inst_goals []
   484     else
   476     else
   485       collect_results (test_term ctxt (limit_time, is_interactive))
   477       collect_results (test_term compile ctxt (limit_time, is_interactive))
   486         (maps (map snd) correct_inst_goals) []
   478         (maps (map snd) correct_inst_goals) []
   487   end;
   479   end;
   488 
   480 
   489 fun test_goal_terms ctxt (limit_time, is_interactive) insts goals =
   481 fun test_goal_terms ctxt (limit_time, is_interactive) insts goals =
   490   case lookup_tester ctxt (Config.get ctxt tester) of
   482   case lookup_tester ctxt (Config.get ctxt tester) of