author  bulwahn 
Tue, 31 May 2011 15:45:24 +0200  
changeset 43112  3117573292b8 
parent 43029  3e060b1c844b 
child 43113  7226051e8b89 
permissions  rwrr 
30824  1 
(* Title: Tools/quickcheck.ML 
40246
c03fc7d3fa97
changed global fixed timeout to a configurable timeout for quickcheck; test parameters in quickcheck are now fully passed around with the context
bulwahn
parents:
40225
diff
changeset

2 
Author: Stefan Berghofer, Florian Haftmann, Lukas Bulwahn, TU Muenchen 
28256  3 

4 
Generic counterexample search engine. 

5 
*) 

6 

7 
signature QUICKCHECK = 

8 
sig 

43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

9 
val quickcheckN: string 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

10 
val genuineN: string 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

11 
val noneN: string 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

12 
val unknownN: string 
37910
555287ba8d8d
reordering quickcheck signature; exporting test_params and inspection function
bulwahn
parents:
37909
diff
changeset

13 
val setup: theory > theory 
555287ba8d8d
reordering quickcheck signature; exporting test_params and inspection function
bulwahn
parents:
37909
diff
changeset

14 
(* configuration *) 
32740  15 
val auto: bool Unsynchronized.ref 
41517  16 
val tester : string Config.T 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

17 
val size : int Config.T 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

18 
val iterations : int Config.T 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

19 
val no_assms : bool Config.T 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

20 
val report : bool Config.T 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

21 
val timing : bool Config.T 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

22 
val quiet : bool Config.T 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

23 
val timeout : real Config.T 
40648
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

24 
val finite_types : bool Config.T 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

25 
val finite_type_size : int Config.T 
41517  26 
datatype expectation = No_Expectation  No_Counterexample  Counterexample; 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

27 
datatype test_params = Test_Params of {default_type: typ list, expect : expectation}; 
40246
c03fc7d3fa97
changed global fixed timeout to a configurable timeout for quickcheck; test parameters in quickcheck are now fully passed around with the context
bulwahn
parents:
40225
diff
changeset

28 
val test_params_of : Proof.context > test_params 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

29 
val map_test_params : (typ list * expectation > typ list * expectation) 
40246
c03fc7d3fa97
changed global fixed timeout to a configurable timeout for quickcheck; test parameters in quickcheck are now fully passed around with the context
bulwahn
parents:
40225
diff
changeset

30 
> Context.generic > Context.generic 
42089
904897d0c5bd
adapting mutabelle; exporting more Quickcheck functions
bulwahn
parents:
42088
diff
changeset

31 
datatype report = Report of 
904897d0c5bd
adapting mutabelle; exporting more Quickcheck functions
bulwahn
parents:
42088
diff
changeset

32 
{ iterations : int, raised_match_errors : int, 
904897d0c5bd
adapting mutabelle; exporting more Quickcheck functions
bulwahn
parents:
42088
diff
changeset

33 
satisfied_assms : int list, positive_concl_tests : int } 
904897d0c5bd
adapting mutabelle; exporting more Quickcheck functions
bulwahn
parents:
42088
diff
changeset

34 
(* quickcheck's result *) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

35 
datatype result = 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

36 
Result of 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

37 
{counterexample : (string * term) list option, 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

38 
evaluation_terms : (term * term) list option, 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

39 
timings : (string * int) list, 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

40 
reports : (int * report) list} 
42089
904897d0c5bd
adapting mutabelle; exporting more Quickcheck functions
bulwahn
parents:
42088
diff
changeset

41 
val counterexample_of : result > (string * term) list option 
904897d0c5bd
adapting mutabelle; exporting more Quickcheck functions
bulwahn
parents:
42088
diff
changeset

42 
val timings_of : result > (string * int) list 
43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

43 
(* registering generators *) 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

44 
val add_generator: 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

45 
string * (Proof.context > (term * term list) list > int list > term list option * report option) 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

46 
> Context.generic > Context.generic 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

47 
val add_tester: 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

48 
string * (Proof.context > bool * bool > (string * typ) list > (term * term list) list > result list) 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

49 
> Context.generic > Context.generic 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

50 
val add_batch_generator: 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

51 
string * (Proof.context > term list > (int > term list option) list) 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

52 
> Context.generic > Context.generic 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

53 
val add_batch_validator: 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

54 
string * (Proof.context > term list > (int > bool) list) 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

55 
> Context.generic > Context.generic 
37910
555287ba8d8d
reordering quickcheck signature; exporting test_params and inspection function
bulwahn
parents:
37909
diff
changeset

56 
(* testing terms and proof states *) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

57 
val test_term: Proof.context > bool * bool > term * term list > result 
40648
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

58 
val test_goal_terms: 
42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

59 
Proof.context > bool * bool > (string * typ) list > (term * term list) list 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

60 
> result list 
37909
583543ad6ad1
changed default types to a list of types; extended quickcheck parameters to be a list of values to parse a list of default types
bulwahn
parents:
36960
diff
changeset

61 
val quickcheck: (string * string list) list > int > Proof.state > (string * term) list option 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

62 
(* testing a batch of terms *) 
42188
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

63 
val test_terms: 
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

64 
Proof.context > term list > (string * term) list option list option * (string * int) list 
42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

65 
val validate_terms: Proof.context > term list > bool list option * (string * int) list 
28256  66 
end; 
67 

68 
structure Quickcheck : QUICKCHECK = 

69 
struct 

70 

43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

71 
val quickcheckN = "quickcheck" 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

72 
val quickcheck_paramsN = "quickcheck_params" 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

73 

abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

74 
val genuineN = "genuine" 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

75 
val noneN = "none" 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

76 
val unknownN = "unknown" 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

77 

30980  78 
(* preferences *) 
79 

32740  80 
val auto = Unsynchronized.ref false; 
30980  81 

82 
val _ = 

83 
ProofGeneralPgip.add_preference Preferences.category_tracing 

39616
8052101883c3
renamed setmp_noncritical to Unsynchronized.setmp to emphasize its meaning;
wenzelm
parents:
39329
diff
changeset

84 
(Unsynchronized.setmp auto true (fn () => 
30980  85 
Preferences.bool_pref auto 
86 
"autoquickcheck" 

39329  87 
"Run Quickcheck automatically.") ()); 
30980  88 

35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

89 
(* quickcheck report *) 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

90 

95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

91 
datatype report = Report of 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

92 
{ iterations : int, raised_match_errors : int, 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

93 
satisfied_assms : int list, positive_concl_tests : int } 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

94 

42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

95 
(* Quickcheck Result *) 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

96 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

97 
datatype result = Result of 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

98 
{ counterexample : (string * term) list option, evaluation_terms : (term * term) list option, 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

99 
timings : (string * int) list, reports : (int * report) list} 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

100 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

101 
val empty_result = 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

102 
Result {counterexample = NONE, evaluation_terms = NONE, timings = [], reports = []} 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

103 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

104 
fun counterexample_of (Result r) = #counterexample r 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

105 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

106 
fun found_counterexample (Result r) = is_some (#counterexample r) 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

107 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

108 
fun response_of (Result r) = case (#counterexample r, #evaluation_terms r) of 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

109 
(SOME ts, SOME evals) => SOME (ts, evals) 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

110 
 (NONE, NONE) => NONE 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

111 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

112 
fun timings_of (Result r) = #timings r 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

113 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

114 
fun set_reponse names eval_terms (SOME ts) (Result r) = 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

115 
let 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

116 
val (ts1, ts2) = chop (length names) ts 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

117 
in 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

118 
Result {counterexample = SOME (names ~~ ts1), evaluation_terms = SOME (eval_terms ~~ ts2), 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

119 
timings = #timings r, reports = #reports r} 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

120 
end 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

121 
 set_reponse _ _ NONE result = result 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

122 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

123 
fun cons_timing timing (Result r) = 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

124 
Result {counterexample = #counterexample r, evaluation_terms = #evaluation_terms r, 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

125 
timings = cons timing (#timings r), reports = #reports r} 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

126 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

127 
fun cons_report size (SOME report) (Result r) = 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

128 
Result {counterexample = #counterexample r, evaluation_terms = #evaluation_terms r, 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

129 
timings = #timings r, reports = cons (size, report) (#reports r)} 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

130 
 cons_report _ NONE result = result 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

131 

42198
ded5ba6b7bac
use Unsynchronized.change convenience, which also emphasizes the raw access to these references (which happen to be local here);
wenzelm
parents:
42194
diff
changeset

132 
fun add_timing timing result_ref = 
ded5ba6b7bac
use Unsynchronized.change convenience, which also emphasizes the raw access to these references (which happen to be local here);
wenzelm
parents:
42194
diff
changeset

133 
Unsynchronized.change result_ref (cons_timing timing) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

134 

42198
ded5ba6b7bac
use Unsynchronized.change convenience, which also emphasizes the raw access to these references (which happen to be local here);
wenzelm
parents:
42194
diff
changeset

135 
fun add_report size report result_ref = 
ded5ba6b7bac
use Unsynchronized.change convenience, which also emphasizes the raw access to these references (which happen to be local here);
wenzelm
parents:
42194
diff
changeset

136 
Unsynchronized.change result_ref (cons_report size report) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

137 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

138 
fun add_response names eval_terms response result_ref = 
42198
ded5ba6b7bac
use Unsynchronized.change convenience, which also emphasizes the raw access to these references (which happen to be local here);
wenzelm
parents:
42194
diff
changeset

139 
Unsynchronized.change result_ref (set_reponse names eval_terms response) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

140 

37929
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

141 
(* expectation *) 
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

142 

41517  143 
datatype expectation = No_Expectation  No_Counterexample  Counterexample; 
37929
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

144 

22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

145 
fun merge_expectation (expect1, expect2) = 
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

146 
if expect1 = expect2 then expect1 else No_Expectation 
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

147 

28315  148 
(* quickcheck configuration  default parameters, test generators *) 
42616
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

149 
val tester = Attrib.setup_config_string @{binding quickcheck_tester} (K "") 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

150 
val size = Attrib.setup_config_int @{binding quickcheck_size} (K 10) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

151 
val iterations = Attrib.setup_config_int @{binding quickcheck_iterations} (K 100) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

152 
val no_assms = Attrib.setup_config_bool @{binding quickcheck_no_assms} (K false) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

153 
val report = Attrib.setup_config_bool @{binding quickcheck_report} (K true) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

154 
val timing = Attrib.setup_config_bool @{binding quickcheck_timing} (K false) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

155 
val quiet = Attrib.setup_config_bool @{binding quickcheck_quiet} (K false) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

156 
val timeout = Attrib.setup_config_real @{binding quickcheck_timeout} (K 30.0) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

157 
val finite_types = Attrib.setup_config_bool @{binding quickcheck_finite_types} (K true) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

158 
val finite_type_size = Attrib.setup_config_int @{binding quickcheck_finite_type_size} (K 3) 
40646  159 

40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

160 
datatype test_params = Test_Params of 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

161 
{default_type: typ list, expect : expectation}; 
38759
37a9092de102
simplification/standardization of some theory data;
wenzelm
parents:
38390
diff
changeset

162 

40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

163 
fun dest_test_params (Test_Params {default_type, expect}) = (default_type, expect); 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

164 

41517  165 
fun make_test_params (default_type, expect) = 
166 
Test_Params {default_type = default_type, expect = expect}; 

40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

167 

41517  168 
fun map_test_params' f (Test_Params {default_type, expect}) = 
169 
make_test_params (f (default_type, expect)); 

38759
37a9092de102
simplification/standardization of some theory data;
wenzelm
parents:
38390
diff
changeset

170 

37a9092de102
simplification/standardization of some theory data;
wenzelm
parents:
38390
diff
changeset

171 
fun merge_test_params 
41472
f6ab14e61604
misc tuning and comments based on review of Theory_Data, Proof_Data, Generic_Data usage;
wenzelm
parents:
41086
diff
changeset

172 
(Test_Params {default_type = default_type1, expect = expect1}, 
f6ab14e61604
misc tuning and comments based on review of Theory_Data, Proof_Data, Generic_Data usage;
wenzelm
parents:
41086
diff
changeset

173 
Test_Params {default_type = default_type2, expect = expect2}) = 
f6ab14e61604
misc tuning and comments based on review of Theory_Data, Proof_Data, Generic_Data usage;
wenzelm
parents:
41086
diff
changeset

174 
make_test_params 
f6ab14e61604
misc tuning and comments based on review of Theory_Data, Proof_Data, Generic_Data usage;
wenzelm
parents:
41086
diff
changeset

175 
(merge (op =) (default_type1, default_type2), merge_expectation (expect1, expect2)); 
28309  176 

39252
8f176e575a49
changing the container for the quickcheck options to a generic data
bulwahn
parents:
39138
diff
changeset

177 
structure Data = Generic_Data 
33522  178 
( 
38759
37a9092de102
simplification/standardization of some theory data;
wenzelm
parents:
38390
diff
changeset

179 
type T = 
43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

180 
(((string * (Proof.context > (term * term list) list > int list > term list option * report option)) list 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

181 
* (string * (Proof.context > bool * bool > (string * typ) list > (term * term list) list > result list)) list) 
42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

182 
* ((string * (Proof.context > term list > (int > term list option) list)) list 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

183 
* ((string * (Proof.context > term list > (int > bool) list)) list))) 
38759
37a9092de102
simplification/standardization of some theory data;
wenzelm
parents:
38390
diff
changeset

184 
* test_params; 
43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

185 
val empty = ((([], []), ([], [])), Test_Params {default_type = [], expect = No_Expectation}); 
28256  186 
val extend = I; 
43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

187 
fun merge ((((generators1, testers1), (batch_generators1, batch_validators1)), params1), 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

188 
(((generators2, testers2), (batch_generators2, batch_validators2)), params2)) : T = 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

189 
(((AList.merge (op =) (K true) (generators1, generators2), 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

190 
AList.merge (op =) (K true) (testers1, testers2)), 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

191 
(AList.merge (op =) (K true) (batch_generators1, batch_generators2), 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

192 
AList.merge (op =) (K true) (batch_validators1, batch_validators2))), 
28309  193 
merge_test_params (params1, params2)); 
33522  194 
); 
28256  195 

39252
8f176e575a49
changing the container for the quickcheck options to a generic data
bulwahn
parents:
39138
diff
changeset

196 
val test_params_of = snd o Data.get o Context.Proof; 
37910
555287ba8d8d
reordering quickcheck signature; exporting test_params and inspection function
bulwahn
parents:
37909
diff
changeset

197 

40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

198 
val default_type = fst o dest_test_params o test_params_of 
40246
c03fc7d3fa97
changed global fixed timeout to a configurable timeout for quickcheck; test parameters in quickcheck are now fully passed around with the context
bulwahn
parents:
40225
diff
changeset

199 

40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

200 
val expect = snd o dest_test_params o test_params_of 
40246
c03fc7d3fa97
changed global fixed timeout to a configurable timeout for quickcheck; test parameters in quickcheck are now fully passed around with the context
bulwahn
parents:
40225
diff
changeset

201 

c03fc7d3fa97
changed global fixed timeout to a configurable timeout for quickcheck; test parameters in quickcheck are now fully passed around with the context
bulwahn
parents:
40225
diff
changeset

202 
val map_test_params = Data.map o apsnd o map_test_params' 
39253
0c47d615a69b
removing report from the arguments of the quickcheck functions and refering to it by picking it from the context
bulwahn
parents:
39252
diff
changeset

203 

43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

204 
val add_generator = Data.map o apfst o apfst o apfst o AList.update (op =); 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

205 

3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

206 
val add_tester = Data.map o apfst o apfst o apsnd o AList.update (op =); 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

207 

42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

208 
val add_batch_generator = Data.map o apfst o apsnd o apfst o AList.update (op =); 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

209 

bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

210 
val add_batch_validator = Data.map o apfst o apsnd o apsnd o AList.update (op =); 
28309  211 

28315  212 
(* generating tests *) 
213 

41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

214 
fun gen_mk_tester lookup ctxt v = 
28309  215 
let 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

216 
val name = Config.get ctxt tester 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

217 
val tester = case lookup ctxt name 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

218 
of NONE => error ("No such quickcheck tester: " ^ name) 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

219 
 SOME tester => tester ctxt; 
40235
87998864284e
use Exn.interruptible_capture to keep usercode interruptible (Exn.capture not immediately followed by Exn.release here);
wenzelm
parents:
40225
diff
changeset

220 
in 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

221 
if Config.get ctxt quiet then 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

222 
try tester v 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

223 
else 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

224 
let 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

225 
val tester = Exn.interruptible_capture tester v 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

226 
in case Exn.get_result tester of 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

227 
NONE => SOME (Exn.release tester) 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

228 
 SOME tester => SOME tester 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

229 
end 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

230 
end 
28315  231 

42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

232 
val mk_tester = 
43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

233 
gen_mk_tester (fn ctxt => AList.lookup (op =) ((fst o fst o fst o Data.get o Context.Proof) ctxt)) 
42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

234 
val mk_batch_tester = 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

235 
gen_mk_tester (fn ctxt => AList.lookup (op =) ((fst o snd o fst o Data.get o Context.Proof) ctxt)) 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

236 
val mk_batch_validator = 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

237 
gen_mk_tester (fn ctxt => AList.lookup (op =) ((snd o snd o fst o Data.get o Context.Proof) ctxt)) 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

238 

43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

239 

3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

240 
fun lookup_tester ctxt = AList.lookup (op =) ((snd o fst o fst o Data.get o Context.Proof) ctxt) 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

241 

28315  242 
(* testing propositions *) 
243 

42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

244 
fun check_test_term t = 
28309  245 
let 
29266  246 
val _ = (null (Term.add_tvars t []) andalso null (Term.add_tfrees t [])) orelse 
28309  247 
error "Term to be tested contains type variables"; 
29266  248 
val _ = null (Term.add_vars t []) orelse 
28309  249 
error "Term to be tested contains schematic variables"; 
42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

250 
in () end 
28256  251 

42014
75417ef605ba
simplified various cpu_time clones (!): eliminated odd Exn.capture/Exn.release (no need to "stop" timing);
wenzelm
parents:
42012
diff
changeset

252 
fun cpu_time description e = 
75417ef605ba
simplified various cpu_time clones (!): eliminated odd Exn.capture/Exn.release (no need to "stop" timing);
wenzelm
parents:
42012
diff
changeset

253 
let val ({cpu, ...}, result) = Timing.timing e () 
75417ef605ba
simplified various cpu_time clones (!): eliminated odd Exn.capture/Exn.release (no need to "stop" timing);
wenzelm
parents:
42012
diff
changeset

254 
in (result, (description, Time.toMilliseconds cpu)) end 
41517  255 

41754
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

256 
fun limit ctxt (limit_time, is_interactive) f exc () = 
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

257 
if limit_time then 
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

258 
TimeLimit.timeLimit (seconds (Config.get ctxt timeout)) f () 
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

259 
handle TimeLimit.TimeOut => 
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

260 
if is_interactive then exc () else raise TimeLimit.TimeOut 
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

261 
else 
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

262 
f () 
41753
dbd00d8a4784
quickcheck invokes TimeLimit.timeLimit only in one separate function
bulwahn
parents:
41735
diff
changeset

263 

42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

264 
fun test_term ctxt (limit_time, is_interactive) (t, eval_terms) = 
28309  265 
let 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

266 
fun message s = if Config.get ctxt quiet then () else Output.urgent_message s 
42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

267 
val _ = check_test_term t 
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

268 
val names = Term.add_free_names t [] 
40366
a2866dbfbe6b
changing timeout to real value; handling Interrupt and Timeout more like nitpick does
bulwahn
parents:
40253
diff
changeset

269 
val current_size = Unsynchronized.ref 0 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

270 
val current_result = Unsynchronized.ref empty_result 
42087
5e236f6ef04f
changing timeout behaviour of quickcheck to proceed after command rather than failing; adding a test case for timeout
bulwahn
parents:
42032
diff
changeset

271 
fun excipit () = 
5e236f6ef04f
changing timeout behaviour of quickcheck to proceed after command rather than failing; adding a test case for timeout
bulwahn
parents:
42032
diff
changeset

272 
"Quickcheck ran out of time while testing at size " ^ string_of_int (!current_size) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

273 
fun with_size test_fun k = 
40911
7febf76e0a69
moving iteration of tests to the testers in quickcheck
bulwahn
parents:
40910
diff
changeset

274 
if k > Config.get ctxt size then 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

275 
NONE 
40911
7febf76e0a69
moving iteration of tests to the testers in quickcheck
bulwahn
parents:
40910
diff
changeset

276 
else 
35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

277 
let 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

278 
val _ = message ("Test data size: " ^ string_of_int k) 
40648
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

279 
val _ = current_size := k 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

280 
val ((result, report), timing) = 
42159
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

281 
cpu_time ("size " ^ string_of_int k) (fn () => test_fun [1, k  1]) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

282 
val _ = add_timing timing current_result 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

283 
val _ = add_report k report current_result 
41517  284 
in 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

285 
case result of NONE => with_size test_fun (k + 1)  SOME q => SOME q 
41517  286 
end; 
34948
2d5f2a9f7601
refactoring the predicate compiler; adding theories for Sequences; adding retrieval to Spec_Rules; adding timing to Quickcheck
bulwahn
parents:
34128
diff
changeset

287 
in 
42162
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

288 
limit ctxt (limit_time, is_interactive) (fn () => 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

289 
let 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

290 
val (test_fun, comp_time) = cpu_time "quickcheck compilation" 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

291 
(fn () => mk_tester ctxt [(t, eval_terms)]); 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

292 
val _ = add_timing comp_time current_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

293 
in 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

294 
case test_fun of NONE => !current_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

295 
 SOME test_fun => 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

296 
let 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

297 
val (response, exec_time) = 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

298 
cpu_time "quickcheck execution" (fn () => with_size test_fun 1) 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

299 
val _ = add_response names eval_terms response current_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

300 
val _ = add_timing exec_time current_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

301 
in 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

302 
!current_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

303 
end 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

304 
end) 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

305 
(fn () => (message (excipit ()); !current_result)) () 
28309  306 
end; 
307 

42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

308 
fun validate_terms ctxt ts = 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

309 
let 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

310 
val _ = map check_test_term ts 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

311 
val size = Config.get ctxt size 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

312 
val (test_funs, comp_time) = cpu_time "quickcheck batch compilation" 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

313 
(fn () => mk_batch_validator ctxt ts) 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

314 
fun with_size tester k = 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

315 
if k > size then true 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

316 
else if tester k then with_size tester (k + 1) else false 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

317 
val (results, exec_time) = cpu_time "quickcheck batch execution" (fn () => 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

318 
Option.map (map (fn test_fun => TimeLimit.timeLimit (seconds (Config.get ctxt timeout)) 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

319 
(fn () => with_size test_fun 1) () 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

320 
handle TimeLimit.TimeOut => true)) test_funs) 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

321 
in 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

322 
(results, [comp_time, exec_time]) 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

323 
end 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

324 

41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

325 
fun test_terms ctxt ts = 
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

326 
let 
42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

327 
val _ = map check_test_term ts 
42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

328 
val size = Config.get ctxt size 
42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

329 
val namess = map (fn t => Term.add_free_names t []) ts 
42188
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

330 
val (test_funs, comp_time) = cpu_time "quickcheck batch compilation" (fn () => mk_batch_tester ctxt ts) 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

331 
fun with_size tester k = 
42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

332 
if k > size then NONE 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

333 
else case tester k of SOME ts => SOME ts  NONE => with_size tester (k + 1) 
42188
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

334 
val (results, exec_time) = cpu_time "quickcheck batch execution" (fn () => 
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

335 
Option.map (map (fn test_fun => TimeLimit.timeLimit (seconds (Config.get ctxt timeout)) 
42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

336 
(fn () => with_size test_fun 1) () 
42188
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

337 
handle TimeLimit.TimeOut => NONE)) test_funs) 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

338 
in 
42188
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

339 
(Option.map (map2 (fn names => Option.map (fn ts => names ~~ ts)) namess) results, 
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

340 
[comp_time, exec_time]) 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

341 
end 
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

342 

41086
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

343 
(* FIXME: this function shows that many assumptions are made upon the generation *) 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

344 
(* In the end there is probably no generic quickcheck interface left... *) 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

345 

41754
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

346 
fun test_term_with_increasing_cardinality ctxt (limit_time, is_interactive) ts = 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

347 
let 
42361  348 
val thy = Proof_Context.theory_of ctxt 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

349 
fun message s = if Config.get ctxt quiet then () else Output.urgent_message s 
42159
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

350 
val (ts', eval_terms) = split_list ts 
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

351 
val _ = map check_test_term ts' 
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

352 
val names = Term.add_free_names (hd ts') [] 
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

353 
val Ts = map snd (Term.add_frees (hd ts') []) 
42162
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

354 
val current_result = Unsynchronized.ref empty_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

355 
fun test_card_size test_fun (card, size) = 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

356 
(* FIXME: why decrement size by one? *) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

357 
let 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

358 
val (ts, timing) = cpu_time ("size " ^ string_of_int size ^ " and card " ^ string_of_int card) 
42275
79be89e07589
removing decrement of cardinality in quickcheck  counting cardinalities starts at 1
bulwahn
parents:
42198
diff
changeset

359 
(fn () => fst (test_fun [card, size  1])) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

360 
val _ = add_timing timing current_result 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

361 
in 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

362 
Option.map (pair card) ts 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

363 
end 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

364 
val enumeration_card_size = 
41086
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

365 
if forall (fn T => Sign.of_sort thy (T, ["Enum.enum"])) Ts then 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

366 
(* size does not matter *) 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

367 
map (rpair 0) (1 upto (length ts)) 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

368 
else 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

369 
(* size does matter *) 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

370 
map_product pair (1 upto (length ts)) (1 upto (Config.get ctxt size)) 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

371 
> sort (fn ((c1, s1), (c2, s2)) => int_ord ((c1 + s1), (c2 + s2))) 
42162
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

372 
in 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

373 
limit ctxt (limit_time, is_interactive) (fn () => 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

374 
let 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

375 
val (test_fun, comp_time) = cpu_time "quickcheck compilation" (fn () => mk_tester ctxt ts) 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

376 
val _ = add_timing comp_time current_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

377 
in 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

378 
case test_fun of 
42159
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

379 
NONE => !current_result 
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

380 
 SOME test_fun => 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

381 
let 
42162
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

382 
val _ = case get_first (test_card_size test_fun) enumeration_card_size of 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

383 
SOME (card, ts) => add_response names (nth eval_terms (card  1)) (SOME ts) current_result 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

384 
 NONE => () 
42162
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

385 
in !current_result end 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

386 
end) 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

387 
(fn () => (message "Quickcheck ran out of time"; !current_result)) () 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

388 
end 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

389 

40647  390 
fun get_finite_types ctxt = 
391 
fst (chop (Config.get ctxt finite_type_size) 

392 
(map (Type o rpair []) ["Enum.finite_1", "Enum.finite_2", "Enum.finite_3", 

41517  393 
"Enum.finite_4", "Enum.finite_5"])) 
40647  394 

37913
e85f5ad02a8f
correcting wellsortedness check and improving error message
bulwahn
parents:
37912
diff
changeset

395 
exception WELLSORTED of string 
e85f5ad02a8f
correcting wellsortedness check and improving error message
bulwahn
parents:
37912
diff
changeset

396 

41517  397 
fun monomorphic_term thy insts default_T = 
28309  398 
let 
399 
fun subst (T as TFree (v, S)) = 

40903  400 
let 
401 
val T' = AList.lookup (op =) insts v 

402 
> the_default default_T 

403 
in if Sign.of_sort thy (T', S) then T' 

41517  404 
else raise (WELLSORTED ("For instantiation with default_type " ^ 
405 
Syntax.string_of_typ_global thy default_T ^ 

40903  406 
":\n" ^ Syntax.string_of_typ_global thy T' ^ 
407 
" to be substituted for variable " ^ 

408 
Syntax.string_of_typ_global thy T ^ " does not have sort " ^ 

409 
Syntax.string_of_sort_global thy S)) 

410 
end 

28309  411 
 subst T = T; 
412 
in (map_types o map_atyps) subst end; 

413 

42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

414 
datatype wellsorted_error = Wellsorted_Error of string  Term of term * term list 
37913
e85f5ad02a8f
correcting wellsortedness check and improving error message
bulwahn
parents:
37912
diff
changeset

415 

41754
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

416 
fun test_goal_terms lthy (limit_time, is_interactive) insts check_goals = 
28309  417 
let 
42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

418 
fun map_goal_and_eval_terms f (check_goal, eval_terms) = (f check_goal, map f eval_terms) 
42361  419 
val thy = Proof_Context.theory_of lthy 
40926
c600f6ae4b09
only instantiate type variable if there exists some in quickcheck
bulwahn
parents:
40916
diff
changeset

420 
val default_insts = 
c600f6ae4b09
only instantiate type variable if there exists some in quickcheck
bulwahn
parents:
40916
diff
changeset

421 
if Config.get lthy finite_types then (get_finite_types lthy) else (default_type lthy) 
40647  422 
val inst_goals = 
42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

423 
map (fn (check_goal, eval_terms) => 
40926
c600f6ae4b09
only instantiate type variable if there exists some in quickcheck
bulwahn
parents:
40916
diff
changeset

424 
if not (null (Term.add_tfree_names check_goal [])) then 
c600f6ae4b09
only instantiate type variable if there exists some in quickcheck
bulwahn
parents:
40916
diff
changeset

425 
map (fn T => 
42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

426 
(pair (SOME T) o Term o apfst (Object_Logic.atomize_term thy)) 
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

427 
(map_goal_and_eval_terms (monomorphic_term thy insts T) (check_goal, eval_terms)) 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

428 
handle WELLSORTED s => (SOME T, Wellsorted_Error s)) default_insts 
40926
c600f6ae4b09
only instantiate type variable if there exists some in quickcheck
bulwahn
parents:
40916
diff
changeset

429 
else 
42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

430 
[(NONE, Term (Object_Logic.atomize_term thy check_goal, eval_terms))]) check_goals 
41517  431 
val error_msg = 
432 
cat_lines 

433 
(maps (map_filter (fn (_, Term t) => NONE  (_, Wellsorted_Error s) => SOME s)) inst_goals) 

41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

434 
fun is_wellsorted_term (T, Term t) = SOME (T, t) 
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

435 
 is_wellsorted_term (_, Wellsorted_Error s) = NONE 
37913
e85f5ad02a8f
correcting wellsortedness check and improving error message
bulwahn
parents:
37912
diff
changeset

436 
val correct_inst_goals = 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

437 
case map (map_filter is_wellsorted_term) inst_goals of 
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

438 
[[]] => error error_msg 
37913
e85f5ad02a8f
correcting wellsortedness check and improving error message
bulwahn
parents:
37912
diff
changeset

439 
 xs => xs 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

440 
val _ = if Config.get lthy quiet then () else warning error_msg 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

441 
fun collect_results f [] results = results 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

442 
 collect_results f (t :: ts) results = 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

443 
let 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

444 
val result = f t 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

445 
in 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

446 
if found_counterexample result then 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

447 
(result :: results) 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

448 
else 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

449 
collect_results f ts (result :: results) 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

450 
end 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

451 
fun test_term' goal = 
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

452 
case goal of 
41754
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

453 
[(NONE, t)] => test_term lthy (limit_time, is_interactive) t 
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

454 
 ts => test_term_with_increasing_cardinality lthy (limit_time, is_interactive) (map snd ts) 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

455 
in 
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

456 
if Config.get lthy finite_types then 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

457 
collect_results test_term' correct_inst_goals [] 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

458 
else 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

459 
collect_results (test_term lthy (limit_time, is_interactive)) (maps (map snd) correct_inst_goals) [] 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

460 
end; 
37912  461 

42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

462 
fun test_goal (time_limit, is_interactive) (insts, eval_terms) i state = 
40648
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

463 
let 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

464 
val lthy = Proof.context_of state; 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

465 
val thy = Proof.theory_of state; 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

466 
fun strip (Const ("all", _) $ Abs (_, _, t)) = strip t 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

467 
 strip t = t; 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

468 
val {goal = st, ...} = Proof.raw_goal state; 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

469 
val (gi, frees) = Logic.goal_params (prop_of st) i; 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

470 
val some_locale = case (Option.map #target o Named_Target.peek) lthy 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

471 
of NONE => NONE 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

472 
 SOME "" => NONE 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

473 
 SOME locale => SOME locale; 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

474 
val assms = if Config.get lthy no_assms then [] else case some_locale 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

475 
of NONE => Assumption.all_assms_of lthy 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

476 
 SOME locale => Assumption.local_assms_of lthy (Locale.init locale thy); 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

477 
val proto_goal = Logic.list_implies (map Thm.term_of assms, subst_bounds (frees, strip gi)); 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

478 
val check_goals = case some_locale 
42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

479 
of NONE => [(proto_goal, eval_terms)] 
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

480 
 SOME locale => 
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

481 
map (fn (_, phi) => (Morphism.term phi proto_goal, map (Morphism.term phi) eval_terms)) 
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

482 
(Locale.registrations_of (Context.Theory thy) (*FIXME*) locale); 
43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

483 
val test_goals = the_default test_goal_terms (lookup_tester lthy (Config.get lthy tester)) 
40648
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

484 
in 
43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

485 
test_goals lthy (time_limit, is_interactive) insts check_goals 
40648
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

486 
end 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

487 

37912  488 
(* pretty printing *) 
28315  489 

40225  490 
fun tool_name auto = (if auto then "Auto " else "") ^ "Quickcheck" 
491 

492 
fun pretty_counterex ctxt auto NONE = Pretty.str (tool_name auto ^ " found no counterexample.") 

42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

493 
 pretty_counterex ctxt auto (SOME (cex, eval_terms)) = 
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

494 
Pretty.chunks ((Pretty.str (tool_name auto ^ " found a counterexample:\n") :: 
28315  495 
map (fn (s, t) => 
42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

496 
Pretty.block [Pretty.str (s ^ " ="), Pretty.brk 1, Syntax.pretty_term ctxt t]) (rev cex)) 
43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

497 
@ (if null eval_terms then [] 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

498 
else (Pretty.str ("Evaluated terms:\n") :: 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

499 
map (fn (t, u) => 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

500 
Pretty.block [Syntax.pretty_term ctxt t, Pretty.str " =", Pretty.brk 1, Syntax.pretty_term ctxt u]) 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

501 
(rev eval_terms)))); 
28315  502 

35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

503 
fun pretty_report (Report {iterations = iterations, raised_match_errors = raised_match_errors, 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

504 
satisfied_assms = satisfied_assms, positive_concl_tests = positive_concl_tests}) = 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

505 
let 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

506 
fun pretty_stat s i = Pretty.block ([Pretty.str (s ^ ": " ^ string_of_int i)]) 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

507 
in 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

508 
([pretty_stat "iterations" iterations, 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

509 
pretty_stat "match exceptions" raised_match_errors] 
41517  510 
@ map_index 
511 
(fn (i, n) => pretty_stat ("satisfied " ^ string_of_int (i + 1) ^ ". assumption") n) 

35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

512 
satisfied_assms 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

513 
@ [pretty_stat "positive conclusion tests" positive_concl_tests]) 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

514 
end 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

515 

35380
6ac5b81a763d
adopting Mutabelle to quickcheck reporting; improving quickcheck reporting
bulwahn
parents:
35379
diff
changeset

516 
fun pretty_reports ctxt (SOME reports) = 
40916  517 
Pretty.chunks (Pretty.fbrk :: Pretty.str "Quickcheck report:" :: 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

518 
maps (fn (size, report) => 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

519 
Pretty.str ("size " ^ string_of_int size ^ ":") :: pretty_report report @ [Pretty.brk 1]) 
35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

520 
(rev reports)) 
35380
6ac5b81a763d
adopting Mutabelle to quickcheck reporting; improving quickcheck reporting
bulwahn
parents:
35379
diff
changeset

521 
 pretty_reports ctxt NONE = Pretty.str "" 
35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

522 

42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

523 
fun pretty_timings timings = 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

524 
Pretty.chunks (Pretty.fbrk :: Pretty.str "timings:" :: 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

525 
maps (fn (label, time) => 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

526 
Pretty.str (label ^ ": " ^ string_of_int time ^ " ms") :: Pretty.brk 1 :: []) (rev timings)) 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

527 

42433
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

528 
fun pretty_counterex_and_reports ctxt auto [] = 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

529 
Pretty.chunks [Pretty.strs (tool_name auto :: 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

530 
space_explode " " "is used in a locale where no interpretation is provided."), 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

531 
Pretty.strs (space_explode " " "Please provide an executable interpretation for the locale.")] 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

532 
 pretty_counterex_and_reports ctxt auto (result :: _) = 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

533 
Pretty.chunks (pretty_counterex ctxt auto (response_of result) :: 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

534 
(* map (pretty_reports ctxt) (reports_of result) :: *) 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

535 
(if Config.get ctxt timing then cons (pretty_timings (timings_of result)) else I) []) 
28315  536 

30980  537 
(* Isar commands *) 
28315  538 

28336  539 
fun read_nat s = case (Library.read_int o Symbol.explode) s 
540 
of (k, []) => if k >= 0 then k 

541 
else error ("Not a natural number: " ^ s) 

542 
 (_, _ :: _) => error ("Not a natural number: " ^ s); 

37909
583543ad6ad1
changed default types to a list of types; extended quickcheck parameters to be a list of values to parse a list of default types
bulwahn
parents:
36960
diff
changeset

543 

34128
8650a073dd9b
made Quickcheck take structured proof assumptions into account (like Refute and Nitpick) by default;
blanchet
parents:
33583
diff
changeset

544 
fun read_bool "false" = false 
8650a073dd9b
made Quickcheck take structured proof assumptions into account (like Refute and Nitpick) by default;
blanchet
parents:
33583
diff
changeset

545 
 read_bool "true" = true 
8650a073dd9b
made Quickcheck take structured proof assumptions into account (like Refute and Nitpick) by default;
blanchet
parents:
33583
diff
changeset

546 
 read_bool s = error ("Not a Boolean value: " ^ s) 
28315  547 

40366
a2866dbfbe6b
changing timeout to real value; handling Interrupt and Timeout more like nitpick does
bulwahn
parents:
40253
diff
changeset

548 
fun read_real s = 
a2866dbfbe6b
changing timeout to real value; handling Interrupt and Timeout more like nitpick does
bulwahn
parents:
40253
diff
changeset

549 
case (Real.fromString s) of 
a2866dbfbe6b
changing timeout to real value; handling Interrupt and Timeout more like nitpick does
bulwahn
parents:
40253
diff
changeset

550 
SOME s => s 
a2866dbfbe6b
changing timeout to real value; handling Interrupt and Timeout more like nitpick does
bulwahn
parents:
40253
diff
changeset

551 
 NONE => error ("Not a real number: " ^ s) 
a2866dbfbe6b
changing timeout to real value; handling Interrupt and Timeout more like nitpick does
bulwahn
parents:
40253
diff
changeset

552 

37929
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

553 
fun read_expectation "no_expectation" = No_Expectation 
41517  554 
 read_expectation "no_counterexample" = No_Counterexample 
37929
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

555 
 read_expectation "counterexample" = Counterexample 
41517  556 
 read_expectation s = error ("Not an expectation value: " ^ s) 
37929
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

557 

43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

558 
fun valid_tester_name genctxt = AList.defined (op =) (fst (fst (fst (Data.get genctxt)))) 
40912
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

559 

1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

560 
fun parse_tester name genctxt = 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

561 
if valid_tester_name genctxt name then 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

562 
Config.put_generic tester name genctxt 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

563 
else 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

564 
error ("Unknown tester: " ^ name) 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

565 

1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

566 
fun parse_test_param ("tester", [arg]) = parse_tester arg 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

567 
 parse_test_param ("size", [arg]) = Config.put_generic size (read_nat arg) 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

568 
 parse_test_param ("iterations", [arg]) = Config.put_generic iterations (read_nat arg) 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

569 
 parse_test_param ("default_type", arg) = (fn gen_ctxt => 
41517  570 
map_test_params 
42361  571 
((apfst o K) (map (Proof_Context.read_typ (Context.proof_of gen_ctxt)) arg)) gen_ctxt) 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

572 
 parse_test_param ("no_assms", [arg]) = Config.put_generic no_assms (read_bool arg) 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

573 
 parse_test_param ("expect", [arg]) = map_test_params ((apsnd o K) (read_expectation arg)) 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

574 
 parse_test_param ("report", [arg]) = Config.put_generic report (read_bool arg) 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

575 
 parse_test_param ("quiet", [arg]) = Config.put_generic quiet (read_bool arg) 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

576 
 parse_test_param ("timeout", [arg]) = Config.put_generic timeout (read_real arg) 
40646  577 
 parse_test_param ("finite_types", [arg]) = Config.put_generic finite_types (read_bool arg) 
41517  578 
 parse_test_param ("finite_type_size", [arg]) = 
579 
Config.put_generic finite_type_size (read_nat arg) 

40912
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

580 
 parse_test_param (name, _) = fn genctxt => 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

581 
if valid_tester_name genctxt name then 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

582 
Config.put_generic tester name genctxt 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

583 
else error ("Unknown tester or test parameter: " ^ name); 
28315  584 

42025  585 
fun parse_test_param_inst (name, arg) ((insts, eval_terms), ctxt) = 
42361  586 
case try (Proof_Context.read_typ ctxt) name 
42025  587 
of SOME (TFree (v, _)) => 
42361  588 
((AList.update (op =) (v, Proof_Context.read_typ ctxt (the_single arg)) insts, eval_terms), ctxt) 
42025  589 
 NONE => (case name of 
590 
"eval" => ((insts, eval_terms @ map (Syntax.read_term ctxt) arg), ctxt) 

591 
 _ => ((insts, eval_terms), Context.proof_map (parse_test_param (name, arg)) ctxt)); 

28309  592 

40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

593 
fun quickcheck_params_cmd args = Context.theory_map (fold parse_test_param args); 
41517  594 

42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

595 
fun check_expectation state results = 
42433
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

596 
(if exists found_counterexample results andalso expect (Proof.context_of state) = No_Counterexample 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

597 
then 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

598 
error ("quickcheck expected to find no counterexample but found one") 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

599 
else 
42433
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

600 
(if not (exists found_counterexample results) andalso expect (Proof.context_of state) = Counterexample 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

601 
then 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

602 
error ("quickcheck expected to find a counterexample but did not find one") 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

603 
else ())) 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

604 

35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

605 
fun gen_quickcheck args i state = 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

606 
state 
42025  607 
> Proof.map_context_result (fn ctxt => fold parse_test_param_inst args (([], []), ctxt)) 
42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

608 
> (fn ((insts, eval_terms), state') => test_goal (true, true) (insts, eval_terms) i state' 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

609 
> tap (check_expectation state')) 
32297  610 

42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

611 
fun quickcheck args i state = counterexample_of (hd (gen_quickcheck args i state)) 
35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

612 

32297  613 
fun quickcheck_cmd args i state = 
35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

614 
gen_quickcheck args i (Toplevel.proof_of state) 
43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

615 
> Output.urgent_message o Pretty.string_of 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

616 
o pretty_counterex_and_reports (Toplevel.context_of state) false; 
28309  617 

41517  618 
val parse_arg = 
619 
Parse.name  

620 
(Scan.optional (Parse.$$$ "="  

621 
(((Parse.name  Parse.float_number) >> single)  

622 
(Parse.$$$ "["  Parse.list1 Parse.name  Parse.$$$ "]"))) ["true"]); 

28309  623 

41517  624 
val parse_args = 
625 
Parse.$$$ "["  Parse.list1 parse_arg  Parse.$$$ "]"  Scan.succeed []; 

28336  626 

36960
01594f816e3a
prefer structure Keyword, Parse, Parse_Spec, Outer_Syntax;
wenzelm
parents:
36610
diff
changeset

627 
val _ = 
43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

628 
Outer_Syntax.command quickcheck_paramsN "set parameters for random testing" Keyword.thy_decl 
36960
01594f816e3a
prefer structure Keyword, Parse, Parse_Spec, Outer_Syntax;
wenzelm
parents:
36610
diff
changeset

629 
(parse_args >> (fn args => Toplevel.theory (quickcheck_params_cmd args))); 
28309  630 

36960
01594f816e3a
prefer structure Keyword, Parse, Parse_Spec, Outer_Syntax;
wenzelm
parents:
36610
diff
changeset

631 
val _ = 
43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

632 
Outer_Syntax.improper_command quickcheckN "try to find counterexample for subgoal" Keyword.diag 
36960
01594f816e3a
prefer structure Keyword, Parse, Parse_Spec, Outer_Syntax;
wenzelm
parents:
36610
diff
changeset

633 
(parse_args  Scan.optional Parse.nat 1 
01594f816e3a
prefer structure Keyword, Parse, Parse_Spec, Outer_Syntax;
wenzelm
parents:
36610
diff
changeset

634 
>> (fn (args, i) => Toplevel.no_timing o Toplevel.keep (quickcheck_cmd args i))); 
28309  635 

43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

636 
(* automatic testing *) 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

637 

abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

638 
fun try_quickcheck auto state = 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

639 
let 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

640 
val ctxt = Proof.context_of state; 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

641 
val i = 1; 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

642 
val res = 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

643 
state 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

644 
> Proof.map_context (Config.put report false #> Config.put quiet true) 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

645 
> try (test_goal (false, false) ([], []) i); 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

646 
in 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

647 
case res of 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

648 
NONE => (unknownN, state) 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

649 
 SOME results => 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

650 
let 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

651 
val msg = pretty_counterex_and_reports ctxt auto results 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

652 
in 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

653 
if exists found_counterexample results then 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

654 
(genuineN, 
43029
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

655 
state 
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

656 
> (if auto then 
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

657 
Proof.goal_message (K (Pretty.chunks [Pretty.str "", 
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

658 
Pretty.mark Markup.hilite msg])) 
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

659 
else 
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

660 
tap (fn _ => Output.urgent_message (Pretty.string_of msg)))) 
43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

661 
else 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

662 
(noneN, state) 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

663 
end 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

664 
end 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

665 
> `(fn (outcome_code, _) => outcome_code = genuineN) 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

666 

43029
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

667 
val setup = Try.register_tool (quickcheckN, (20, auto, try_quickcheck)) 
43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

668 

28315  669 
end; 
28309  670 

28315  671 
val auto_quickcheck = Quickcheck.auto; 