author  wenzelm 
Sat, 09 Jul 2011 21:53:27 +0200  
changeset 43721  fad8634cee62 
parent 43585  ea959ab7bbe3 
child 43761  e72ba84ae58f 
permissions  rwrr 
30824  1 
(* Title: Tools/quickcheck.ML 
40246
c03fc7d3fa97
changed global fixed timeout to a configurable timeout for quickcheck; test parameters in quickcheck are now fully passed around with the context
bulwahn
parents:
40225
diff
changeset

2 
Author: Stefan Berghofer, Florian Haftmann, Lukas Bulwahn, TU Muenchen 
28256  3 

4 
Generic counterexample search engine. 

5 
*) 

6 

7 
signature QUICKCHECK = 

8 
sig 

43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

9 
val quickcheckN: string 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

10 
val genuineN: string 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

11 
val noneN: string 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

12 
val unknownN: string 
37910
555287ba8d8d
reordering quickcheck signature; exporting test_params and inspection function
bulwahn
parents:
37909
diff
changeset

13 
val setup: theory > theory 
555287ba8d8d
reordering quickcheck signature; exporting test_params and inspection function
bulwahn
parents:
37909
diff
changeset

14 
(* configuration *) 
32740  15 
val auto: bool Unsynchronized.ref 
41517  16 
val tester : string Config.T 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

17 
val size : int Config.T 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

18 
val iterations : int Config.T 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

19 
val no_assms : bool Config.T 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

20 
val report : bool Config.T 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

21 
val timing : bool Config.T 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

22 
val quiet : bool Config.T 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

23 
val timeout : real Config.T 
40648
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

24 
val finite_types : bool Config.T 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

25 
val finite_type_size : int Config.T 
41517  26 
datatype expectation = No_Expectation  No_Counterexample  Counterexample; 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

27 
datatype test_params = Test_Params of {default_type: typ list, expect : expectation}; 
40246
c03fc7d3fa97
changed global fixed timeout to a configurable timeout for quickcheck; test parameters in quickcheck are now fully passed around with the context
bulwahn
parents:
40225
diff
changeset

28 
val test_params_of : Proof.context > test_params 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

29 
val map_test_params : (typ list * expectation > typ list * expectation) 
40246
c03fc7d3fa97
changed global fixed timeout to a configurable timeout for quickcheck; test parameters in quickcheck are now fully passed around with the context
bulwahn
parents:
40225
diff
changeset

30 
> Context.generic > Context.generic 
42089
904897d0c5bd
adapting mutabelle; exporting more Quickcheck functions
bulwahn
parents:
42088
diff
changeset

31 
datatype report = Report of 
904897d0c5bd
adapting mutabelle; exporting more Quickcheck functions
bulwahn
parents:
42088
diff
changeset

32 
{ iterations : int, raised_match_errors : int, 
904897d0c5bd
adapting mutabelle; exporting more Quickcheck functions
bulwahn
parents:
42088
diff
changeset

33 
satisfied_assms : int list, positive_concl_tests : int } 
904897d0c5bd
adapting mutabelle; exporting more Quickcheck functions
bulwahn
parents:
42088
diff
changeset

34 
(* quickcheck's result *) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

35 
datatype result = 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

36 
Result of 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

37 
{counterexample : (string * term) list option, 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

38 
evaluation_terms : (term * term) list option, 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

39 
timings : (string * int) list, 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

40 
reports : (int * report) list} 
43314
a9090cabca14
adding a nicer error message for quickcheck_narrowing; hiding fact empty_def
bulwahn
parents:
43147
diff
changeset

41 
val empty_result : result 
43585  42 
val add_timing : (string * int) > result Unsynchronized.ref > unit 
42089
904897d0c5bd
adapting mutabelle; exporting more Quickcheck functions
bulwahn
parents:
42088
diff
changeset

43 
val counterexample_of : result > (string * term) list option 
904897d0c5bd
adapting mutabelle; exporting more Quickcheck functions
bulwahn
parents:
42088
diff
changeset

44 
val timings_of : result > (string * int) list 
43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

45 
(* registering generators *) 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

46 
val add_generator: 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

47 
string * (Proof.context > (term * term list) list > int list > term list option * report option) 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

48 
> Context.generic > Context.generic 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

49 
val add_tester: 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

50 
string * (Proof.context > bool * bool > (string * typ) list > (term * term list) list > result list) 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

51 
> Context.generic > Context.generic 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

52 
val add_batch_generator: 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

53 
string * (Proof.context > term list > (int > term list option) list) 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

54 
> Context.generic > Context.generic 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

55 
val add_batch_validator: 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

56 
string * (Proof.context > term list > (int > bool) list) 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

57 
> Context.generic > Context.generic 
43113
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

58 
(* basic operations *) 
43585  59 
val limit : Time.time > (bool * bool) > (unit > 'a) > (unit > 'a) > unit > 'a 
43113
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

60 
val instantiate_goals: Proof.context > (string * typ) list > (term * term list) list 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

61 
> (typ option * (term * term list)) list list 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

62 
val collect_results: ('a > result) > 'a list > result list > result list 
37910
555287ba8d8d
reordering quickcheck signature; exporting test_params and inspection function
bulwahn
parents:
37909
diff
changeset

63 
(* testing terms and proof states *) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

64 
val test_term: Proof.context > bool * bool > term * term list > result 
40648
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

65 
val test_goal_terms: 
42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

66 
Proof.context > bool * bool > (string * typ) list > (term * term list) list 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

67 
> result list 
37909
583543ad6ad1
changed default types to a list of types; extended quickcheck parameters to be a list of values to parse a list of default types
bulwahn
parents:
36960
diff
changeset

68 
val quickcheck: (string * string list) list > int > Proof.state > (string * term) list option 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

69 
(* testing a batch of terms *) 
42188
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

70 
val test_terms: 
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

71 
Proof.context > term list > (string * term) list option list option * (string * int) list 
42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

72 
val validate_terms: Proof.context > term list > bool list option * (string * int) list 
28256  73 
end; 
74 

75 
structure Quickcheck : QUICKCHECK = 

76 
struct 

77 

43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

78 
val quickcheckN = "quickcheck" 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

79 
val quickcheck_paramsN = "quickcheck_params" 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

80 

abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

81 
val genuineN = "genuine" 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

82 
val noneN = "none" 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

83 
val unknownN = "unknown" 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

84 

30980  85 
(* preferences *) 
86 

32740  87 
val auto = Unsynchronized.ref false; 
30980  88 

89 
val _ = 

90 
ProofGeneralPgip.add_preference Preferences.category_tracing 

39616
8052101883c3
renamed setmp_noncritical to Unsynchronized.setmp to emphasize its meaning;
wenzelm
parents:
39329
diff
changeset

91 
(Unsynchronized.setmp auto true (fn () => 
30980  92 
Preferences.bool_pref auto 
93 
"autoquickcheck" 

39329  94 
"Run Quickcheck automatically.") ()); 
30980  95 

35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

96 
(* quickcheck report *) 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

97 

95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

98 
datatype report = Report of 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

99 
{ iterations : int, raised_match_errors : int, 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

100 
satisfied_assms : int list, positive_concl_tests : int } 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

101 

42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

102 
(* Quickcheck Result *) 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

103 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

104 
datatype result = Result of 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

105 
{ counterexample : (string * term) list option, evaluation_terms : (term * term) list option, 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

106 
timings : (string * int) list, reports : (int * report) list} 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

107 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

108 
val empty_result = 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

109 
Result {counterexample = NONE, evaluation_terms = NONE, timings = [], reports = []} 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

110 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

111 
fun counterexample_of (Result r) = #counterexample r 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

112 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

113 
fun found_counterexample (Result r) = is_some (#counterexample r) 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

114 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

115 
fun response_of (Result r) = case (#counterexample r, #evaluation_terms r) of 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

116 
(SOME ts, SOME evals) => SOME (ts, evals) 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

117 
 (NONE, NONE) => NONE 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

118 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

119 
fun timings_of (Result r) = #timings r 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

120 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

121 
fun set_reponse names eval_terms (SOME ts) (Result r) = 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

122 
let 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

123 
val (ts1, ts2) = chop (length names) ts 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

124 
in 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

125 
Result {counterexample = SOME (names ~~ ts1), evaluation_terms = SOME (eval_terms ~~ ts2), 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

126 
timings = #timings r, reports = #reports r} 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

127 
end 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

128 
 set_reponse _ _ NONE result = result 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

129 

43585  130 

131 
fun set_counterexample counterexample (Result r) = 

132 
Result {counterexample = counterexample, evaluation_terms = #evaluation_terms r, 

133 
timings = #timings r, reports = #reports r} 

134 

135 
fun set_evaluation_terms evaluation_terms (Result r) = 

136 
Result {counterexample = #counterexample r, evaluation_terms = evaluation_terms, 

137 
timings = #timings r, reports = #reports r} 

138 

42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

139 
fun cons_timing timing (Result r) = 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

140 
Result {counterexample = #counterexample r, evaluation_terms = #evaluation_terms r, 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

141 
timings = cons timing (#timings r), reports = #reports r} 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

142 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

143 
fun cons_report size (SOME report) (Result r) = 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

144 
Result {counterexample = #counterexample r, evaluation_terms = #evaluation_terms r, 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

145 
timings = #timings r, reports = cons (size, report) (#reports r)} 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

146 
 cons_report _ NONE result = result 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

147 

42198
ded5ba6b7bac
use Unsynchronized.change convenience, which also emphasizes the raw access to these references (which happen to be local here);
wenzelm
parents:
42194
diff
changeset

148 
fun add_timing timing result_ref = 
ded5ba6b7bac
use Unsynchronized.change convenience, which also emphasizes the raw access to these references (which happen to be local here);
wenzelm
parents:
42194
diff
changeset

149 
Unsynchronized.change result_ref (cons_timing timing) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

150 

42198
ded5ba6b7bac
use Unsynchronized.change convenience, which also emphasizes the raw access to these references (which happen to be local here);
wenzelm
parents:
42194
diff
changeset

151 
fun add_report size report result_ref = 
ded5ba6b7bac
use Unsynchronized.change convenience, which also emphasizes the raw access to these references (which happen to be local here);
wenzelm
parents:
42194
diff
changeset

152 
Unsynchronized.change result_ref (cons_report size report) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

153 

8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

154 
fun add_response names eval_terms response result_ref = 
42198
ded5ba6b7bac
use Unsynchronized.change convenience, which also emphasizes the raw access to these references (which happen to be local here);
wenzelm
parents:
42194
diff
changeset

155 
Unsynchronized.change result_ref (set_reponse names eval_terms response) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

156 

37929
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

157 
(* expectation *) 
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

158 

41517  159 
datatype expectation = No_Expectation  No_Counterexample  Counterexample; 
37929
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

160 

22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

161 
fun merge_expectation (expect1, expect2) = 
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

162 
if expect1 = expect2 then expect1 else No_Expectation 
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

163 

28315  164 
(* quickcheck configuration  default parameters, test generators *) 
42616
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

165 
val tester = Attrib.setup_config_string @{binding quickcheck_tester} (K "") 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

166 
val size = Attrib.setup_config_int @{binding quickcheck_size} (K 10) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

167 
val iterations = Attrib.setup_config_int @{binding quickcheck_iterations} (K 100) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

168 
val no_assms = Attrib.setup_config_bool @{binding quickcheck_no_assms} (K false) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

169 
val report = Attrib.setup_config_bool @{binding quickcheck_report} (K true) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

170 
val timing = Attrib.setup_config_bool @{binding quickcheck_timing} (K false) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

171 
val quiet = Attrib.setup_config_bool @{binding quickcheck_quiet} (K false) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

172 
val timeout = Attrib.setup_config_real @{binding quickcheck_timeout} (K 30.0) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

173 
val finite_types = Attrib.setup_config_bool @{binding quickcheck_finite_types} (K true) 
92715b528e78
added Attrib.setup_config_XXX conveniences, with implicit setup of the background theory;
wenzelm
parents:
42433
diff
changeset

174 
val finite_type_size = Attrib.setup_config_int @{binding quickcheck_finite_type_size} (K 3) 
40646  175 

40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

176 
datatype test_params = Test_Params of 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

177 
{default_type: typ list, expect : expectation}; 
38759
37a9092de102
simplification/standardization of some theory data;
wenzelm
parents:
38390
diff
changeset

178 

40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

179 
fun dest_test_params (Test_Params {default_type, expect}) = (default_type, expect); 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

180 

41517  181 
fun make_test_params (default_type, expect) = 
182 
Test_Params {default_type = default_type, expect = expect}; 

40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

183 

41517  184 
fun map_test_params' f (Test_Params {default_type, expect}) = 
185 
make_test_params (f (default_type, expect)); 

38759
37a9092de102
simplification/standardization of some theory data;
wenzelm
parents:
38390
diff
changeset

186 

37a9092de102
simplification/standardization of some theory data;
wenzelm
parents:
38390
diff
changeset

187 
fun merge_test_params 
41472
f6ab14e61604
misc tuning and comments based on review of Theory_Data, Proof_Data, Generic_Data usage;
wenzelm
parents:
41086
diff
changeset

188 
(Test_Params {default_type = default_type1, expect = expect1}, 
f6ab14e61604
misc tuning and comments based on review of Theory_Data, Proof_Data, Generic_Data usage;
wenzelm
parents:
41086
diff
changeset

189 
Test_Params {default_type = default_type2, expect = expect2}) = 
f6ab14e61604
misc tuning and comments based on review of Theory_Data, Proof_Data, Generic_Data usage;
wenzelm
parents:
41086
diff
changeset

190 
make_test_params 
f6ab14e61604
misc tuning and comments based on review of Theory_Data, Proof_Data, Generic_Data usage;
wenzelm
parents:
41086
diff
changeset

191 
(merge (op =) (default_type1, default_type2), merge_expectation (expect1, expect2)); 
28309  192 

39252
8f176e575a49
changing the container for the quickcheck options to a generic data
bulwahn
parents:
39138
diff
changeset

193 
structure Data = Generic_Data 
33522  194 
( 
38759
37a9092de102
simplification/standardization of some theory data;
wenzelm
parents:
38390
diff
changeset

195 
type T = 
43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

196 
(((string * (Proof.context > (term * term list) list > int list > term list option * report option)) list 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

197 
* (string * (Proof.context > bool * bool > (string * typ) list > (term * term list) list > result list)) list) 
42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

198 
* ((string * (Proof.context > term list > (int > term list option) list)) list 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

199 
* ((string * (Proof.context > term list > (int > bool) list)) list))) 
38759
37a9092de102
simplification/standardization of some theory data;
wenzelm
parents:
38390
diff
changeset

200 
* test_params; 
43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

201 
val empty = ((([], []), ([], [])), Test_Params {default_type = [], expect = No_Expectation}); 
28256  202 
val extend = I; 
43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

203 
fun merge ((((generators1, testers1), (batch_generators1, batch_validators1)), params1), 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

204 
(((generators2, testers2), (batch_generators2, batch_validators2)), params2)) : T = 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

205 
(((AList.merge (op =) (K true) (generators1, generators2), 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

206 
AList.merge (op =) (K true) (testers1, testers2)), 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

207 
(AList.merge (op =) (K true) (batch_generators1, batch_generators2), 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

208 
AList.merge (op =) (K true) (batch_validators1, batch_validators2))), 
28309  209 
merge_test_params (params1, params2)); 
33522  210 
); 
28256  211 

39252
8f176e575a49
changing the container for the quickcheck options to a generic data
bulwahn
parents:
39138
diff
changeset

212 
val test_params_of = snd o Data.get o Context.Proof; 
37910
555287ba8d8d
reordering quickcheck signature; exporting test_params and inspection function
bulwahn
parents:
37909
diff
changeset

213 

40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

214 
val default_type = fst o dest_test_params o test_params_of 
40246
c03fc7d3fa97
changed global fixed timeout to a configurable timeout for quickcheck; test parameters in quickcheck are now fully passed around with the context
bulwahn
parents:
40225
diff
changeset

215 

40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

216 
val expect = snd o dest_test_params o test_params_of 
40246
c03fc7d3fa97
changed global fixed timeout to a configurable timeout for quickcheck; test parameters in quickcheck are now fully passed around with the context
bulwahn
parents:
40225
diff
changeset

217 

c03fc7d3fa97
changed global fixed timeout to a configurable timeout for quickcheck; test parameters in quickcheck are now fully passed around with the context
bulwahn
parents:
40225
diff
changeset

218 
val map_test_params = Data.map o apsnd o map_test_params' 
39253
0c47d615a69b
removing report from the arguments of the quickcheck functions and refering to it by picking it from the context
bulwahn
parents:
39252
diff
changeset

219 

43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

220 
val add_generator = Data.map o apfst o apfst o apfst o AList.update (op =); 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

221 

3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

222 
val add_tester = Data.map o apfst o apfst o apsnd o AList.update (op =); 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

223 

42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

224 
val add_batch_generator = Data.map o apfst o apsnd o apfst o AList.update (op =); 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

225 

bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

226 
val add_batch_validator = Data.map o apfst o apsnd o apsnd o AList.update (op =); 
28309  227 

28315  228 
(* generating tests *) 
229 

41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

230 
fun gen_mk_tester lookup ctxt v = 
28309  231 
let 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

232 
val name = Config.get ctxt tester 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

233 
val tester = case lookup ctxt name 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

234 
of NONE => error ("No such quickcheck tester: " ^ name) 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

235 
 SOME tester => tester ctxt; 
40235
87998864284e
use Exn.interruptible_capture to keep usercode interruptible (Exn.capture not immediately followed by Exn.release here);
wenzelm
parents:
40225
diff
changeset

236 
in 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

237 
if Config.get ctxt quiet then 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

238 
try tester v 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

239 
else 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

240 
let 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

241 
val tester = Exn.interruptible_capture tester v 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

242 
in case Exn.get_result tester of 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

243 
NONE => SOME (Exn.release tester) 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

244 
 SOME tester => SOME tester 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

245 
end 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

246 
end 
28315  247 

42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

248 
val mk_tester = 
43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

249 
gen_mk_tester (fn ctxt => AList.lookup (op =) ((fst o fst o fst o Data.get o Context.Proof) ctxt)) 
42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

250 
val mk_batch_tester = 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

251 
gen_mk_tester (fn ctxt => AList.lookup (op =) ((fst o snd o fst o Data.get o Context.Proof) ctxt)) 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

252 
val mk_batch_validator = 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

253 
gen_mk_tester (fn ctxt => AList.lookup (op =) ((snd o snd o fst o Data.get o Context.Proof) ctxt)) 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

254 

43112
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

255 

3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

256 
fun lookup_tester ctxt = AList.lookup (op =) ((snd o fst o fst o Data.get o Context.Proof) ctxt) 
3117573292b8
adding registration of testers in Quickcheck for its use in Quickcheck_Narrowing
bulwahn
parents:
43029
diff
changeset

257 

28315  258 
(* testing propositions *) 
259 

42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

260 
fun check_test_term t = 
28309  261 
let 
29266  262 
val _ = (null (Term.add_tvars t []) andalso null (Term.add_tfrees t [])) orelse 
28309  263 
error "Term to be tested contains type variables"; 
29266  264 
val _ = null (Term.add_vars t []) orelse 
28309  265 
error "Term to be tested contains schematic variables"; 
42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

266 
in () end 
28256  267 

42014
75417ef605ba
simplified various cpu_time clones (!): eliminated odd Exn.capture/Exn.release (no need to "stop" timing);
wenzelm
parents:
42012
diff
changeset

268 
fun cpu_time description e = 
75417ef605ba
simplified various cpu_time clones (!): eliminated odd Exn.capture/Exn.release (no need to "stop" timing);
wenzelm
parents:
42012
diff
changeset

269 
let val ({cpu, ...}, result) = Timing.timing e () 
75417ef605ba
simplified various cpu_time clones (!): eliminated odd Exn.capture/Exn.release (no need to "stop" timing);
wenzelm
parents:
42012
diff
changeset

270 
in (result, (description, Time.toMilliseconds cpu)) end 
41517  271 

43585  272 
fun limit timeout (limit_time, is_interactive) f exc () = 
41754
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

273 
if limit_time then 
43585  274 
TimeLimit.timeLimit timeout f () 
41754
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

275 
handle TimeLimit.TimeOut => 
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

276 
if is_interactive then exc () else raise TimeLimit.TimeOut 
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

277 
else 
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

278 
f () 
41753
dbd00d8a4784
quickcheck invokes TimeLimit.timeLimit only in one separate function
bulwahn
parents:
41735
diff
changeset

279 

42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

280 
fun test_term ctxt (limit_time, is_interactive) (t, eval_terms) = 
28309  281 
let 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

282 
fun message s = if Config.get ctxt quiet then () else Output.urgent_message s 
42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

283 
val _ = check_test_term t 
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

284 
val names = Term.add_free_names t [] 
40366
a2866dbfbe6b
changing timeout to real value; handling Interrupt and Timeout more like nitpick does
bulwahn
parents:
40253
diff
changeset

285 
val current_size = Unsynchronized.ref 0 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

286 
val current_result = Unsynchronized.ref empty_result 
42087
5e236f6ef04f
changing timeout behaviour of quickcheck to proceed after command rather than failing; adding a test case for timeout
bulwahn
parents:
42032
diff
changeset

287 
fun excipit () = 
5e236f6ef04f
changing timeout behaviour of quickcheck to proceed after command rather than failing; adding a test case for timeout
bulwahn
parents:
42032
diff
changeset

288 
"Quickcheck ran out of time while testing at size " ^ string_of_int (!current_size) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

289 
fun with_size test_fun k = 
40911
7febf76e0a69
moving iteration of tests to the testers in quickcheck
bulwahn
parents:
40910
diff
changeset

290 
if k > Config.get ctxt size then 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

291 
NONE 
40911
7febf76e0a69
moving iteration of tests to the testers in quickcheck
bulwahn
parents:
40910
diff
changeset

292 
else 
35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

293 
let 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

294 
val _ = message ("Test data size: " ^ string_of_int k) 
40648
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

295 
val _ = current_size := k 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

296 
val ((result, report), timing) = 
42159
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

297 
cpu_time ("size " ^ string_of_int k) (fn () => test_fun [1, k  1]) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

298 
val _ = add_timing timing current_result 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

299 
val _ = add_report k report current_result 
41517  300 
in 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

301 
case result of NONE => with_size test_fun (k + 1)  SOME q => SOME q 
41517  302 
end; 
34948
2d5f2a9f7601
refactoring the predicate compiler; adding theories for Sequences; adding retrieval to Spec_Rules; adding timing to Quickcheck
bulwahn
parents:
34128
diff
changeset

303 
in 
43585  304 
limit (seconds (Config.get ctxt timeout)) (limit_time, is_interactive) (fn () => 
42162
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

305 
let 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

306 
val (test_fun, comp_time) = cpu_time "quickcheck compilation" 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

307 
(fn () => mk_tester ctxt [(t, eval_terms)]); 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

308 
val _ = add_timing comp_time current_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

309 
in 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

310 
case test_fun of NONE => !current_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

311 
 SOME test_fun => 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

312 
let 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

313 
val (response, exec_time) = 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

314 
cpu_time "quickcheck execution" (fn () => with_size test_fun 1) 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

315 
val _ = add_response names eval_terms response current_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

316 
val _ = add_timing exec_time current_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

317 
in 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

318 
!current_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

319 
end 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

320 
end) 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

321 
(fn () => (message (excipit ()); !current_result)) () 
28309  322 
end; 
323 

42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

324 
fun validate_terms ctxt ts = 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

325 
let 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

326 
val _ = map check_test_term ts 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

327 
val size = Config.get ctxt size 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

328 
val (test_funs, comp_time) = cpu_time "quickcheck batch compilation" 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

329 
(fn () => mk_batch_validator ctxt ts) 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

330 
fun with_size tester k = 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

331 
if k > size then true 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

332 
else if tester k then with_size tester (k + 1) else false 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

333 
val (results, exec_time) = cpu_time "quickcheck batch execution" (fn () => 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

334 
Option.map (map (fn test_fun => TimeLimit.timeLimit (seconds (Config.get ctxt timeout)) 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

335 
(fn () => with_size test_fun 1) () 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

336 
handle TimeLimit.TimeOut => true)) test_funs) 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

337 
in 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

338 
(results, [comp_time, exec_time]) 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

339 
end 
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

340 

41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

341 
fun test_terms ctxt ts = 
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

342 
let 
42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

343 
val _ = map check_test_term ts 
42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

344 
val size = Config.get ctxt size 
42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

345 
val namess = map (fn t => Term.add_free_names t []) ts 
42188
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

346 
val (test_funs, comp_time) = cpu_time "quickcheck batch compilation" (fn () => mk_batch_tester ctxt ts) 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

347 
fun with_size tester k = 
42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

348 
if k > size then NONE 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

349 
else case tester k of SOME ts => SOME ts  NONE => with_size tester (k + 1) 
42188
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

350 
val (results, exec_time) = cpu_time "quickcheck batch execution" (fn () => 
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

351 
Option.map (map (fn test_fun => TimeLimit.timeLimit (seconds (Config.get ctxt timeout)) 
42194
bd416284a432
adding general interface for batch validators in quickcheck
bulwahn
parents:
42188
diff
changeset

352 
(fn () => with_size test_fun 1) () 
42188
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

353 
handle TimeLimit.TimeOut => NONE)) test_funs) 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

354 
in 
42188
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

355 
(Option.map (map2 (fn names => Option.map (fn ts => names ~~ ts)) namess) results, 
f6bc441fbf19
adding time profiling in quickcheck's batch testing for further diagnosis in IsaCoSy
bulwahn
parents:
42162
diff
changeset

356 
[comp_time, exec_time]) 
41862
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

357 
end 
a38536bf2736
adding function Quickcheck.test_terms to provide checking a batch of terms
bulwahn
parents:
41754
diff
changeset

358 

41086
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

359 
(* FIXME: this function shows that many assumptions are made upon the generation *) 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

360 
(* In the end there is probably no generic quickcheck interface left... *) 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

361 

41754
aa94a003dcdf
quickcheck can be invoked with its internal timelimit or without
bulwahn
parents:
41753
diff
changeset

362 
fun test_term_with_increasing_cardinality ctxt (limit_time, is_interactive) ts = 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

363 
let 
42361  364 
val thy = Proof_Context.theory_of ctxt 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

365 
fun message s = if Config.get ctxt quiet then () else Output.urgent_message s 
42159
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

366 
val (ts', eval_terms) = split_list ts 
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

367 
val _ = map check_test_term ts' 
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

368 
val names = Term.add_free_names (hd ts') [] 
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

369 
val Ts = map snd (Term.add_frees (hd ts') []) 
42162
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

370 
val current_result = Unsynchronized.ref empty_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

371 
fun test_card_size test_fun (card, size) = 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

372 
(* FIXME: why decrement size by one? *) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

373 
let 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

374 
val (ts, timing) = cpu_time ("size " ^ string_of_int size ^ " and card " ^ string_of_int card) 
42275
79be89e07589
removing decrement of cardinality in quickcheck  counting cardinalities starts at 1
bulwahn
parents:
42198
diff
changeset

375 
(fn () => fst (test_fun [card, size  1])) 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

376 
val _ = add_timing timing current_result 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

377 
in 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

378 
Option.map (pair card) ts 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

379 
end 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

380 
val enumeration_card_size = 
41086
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

381 
if forall (fn T => Sign.of_sort thy (T, ["Enum.enum"])) Ts then 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

382 
(* size does not matter *) 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

383 
map (rpair 0) (1 upto (length ts)) 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

384 
else 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

385 
(* size does matter *) 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

386 
map_product pair (1 upto (length ts)) (1 upto (Config.get ctxt size)) 
b4cccce25d9a
if only finite types and no real datatypes occur in the quantifiers only enumerate cardinality not size in quickcheck
bulwahn
parents:
41043
diff
changeset

387 
> sort (fn ((c1, s1), (c2, s2)) => int_ord ((c1 + s1), (c2 + s2))) 
42162
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

388 
in 
43585  389 
limit (seconds (Config.get ctxt timeout)) (limit_time, is_interactive) (fn () => 
42162
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

390 
let 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

391 
val (test_fun, comp_time) = cpu_time "quickcheck compilation" (fn () => mk_tester ctxt ts) 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

392 
val _ = add_timing comp_time current_result 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

393 
in 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

394 
case test_fun of 
42159
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

395 
NONE => !current_result 
234ec7011e5d
generalizing compilation scheme of quickcheck generators to multiple arguments; changing random and exhaustive tester to use one code invocation for polymorphic instances with multiple cardinalities
bulwahn
parents:
42089
diff
changeset

396 
 SOME test_fun => 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

397 
let 
42162
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

398 
val _ = case get_first (test_card_size test_fun) enumeration_card_size of 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

399 
SOME (card, ts) => add_response names (nth eval_terms (card  1)) (SOME ts) current_result 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

400 
 NONE => () 
42162
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

401 
in !current_result end 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

402 
end) 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

403 
(fn () => (message "Quickcheck ran out of time"; !current_result)) () 
00899500c6ca
moved TimeLimit.timeLimit closure to limit time of compilation and execution to avoid the strange, occasional occuring message Exception trace for exception  Interrupt  probably due to race conditions of a fast execution within the TimeLimit.timelimit closure
bulwahn
parents:
42159
diff
changeset

404 
end 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

405 

40647  406 
fun get_finite_types ctxt = 
407 
fst (chop (Config.get ctxt finite_type_size) 

408 
(map (Type o rpair []) ["Enum.finite_1", "Enum.finite_2", "Enum.finite_3", 

41517  409 
"Enum.finite_4", "Enum.finite_5"])) 
40647  410 

37913
e85f5ad02a8f
correcting wellsortedness check and improving error message
bulwahn
parents:
37912
diff
changeset

411 
exception WELLSORTED of string 
e85f5ad02a8f
correcting wellsortedness check and improving error message
bulwahn
parents:
37912
diff
changeset

412 

41517  413 
fun monomorphic_term thy insts default_T = 
28309  414 
let 
415 
fun subst (T as TFree (v, S)) = 

40903  416 
let 
417 
val T' = AList.lookup (op =) insts v 

418 
> the_default default_T 

419 
in if Sign.of_sort thy (T', S) then T' 

41517  420 
else raise (WELLSORTED ("For instantiation with default_type " ^ 
421 
Syntax.string_of_typ_global thy default_T ^ 

40903  422 
":\n" ^ Syntax.string_of_typ_global thy T' ^ 
423 
" to be substituted for variable " ^ 

424 
Syntax.string_of_typ_global thy T ^ " does not have sort " ^ 

425 
Syntax.string_of_sort_global thy S)) 

426 
end 

28309  427 
 subst T = T; 
428 
in (map_types o map_atyps) subst end; 

429 

42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

430 
datatype wellsorted_error = Wellsorted_Error of string  Term of term * term list 
37913
e85f5ad02a8f
correcting wellsortedness check and improving error message
bulwahn
parents:
37912
diff
changeset

431 

43113
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

432 
fun instantiate_goals lthy insts goals = 
28309  433 
let 
42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

434 
fun map_goal_and_eval_terms f (check_goal, eval_terms) = (f check_goal, map f eval_terms) 
42361  435 
val thy = Proof_Context.theory_of lthy 
40926
c600f6ae4b09
only instantiate type variable if there exists some in quickcheck
bulwahn
parents:
40916
diff
changeset

436 
val default_insts = 
c600f6ae4b09
only instantiate type variable if there exists some in quickcheck
bulwahn
parents:
40916
diff
changeset

437 
if Config.get lthy finite_types then (get_finite_types lthy) else (default_type lthy) 
40647  438 
val inst_goals = 
42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

439 
map (fn (check_goal, eval_terms) => 
40926
c600f6ae4b09
only instantiate type variable if there exists some in quickcheck
bulwahn
parents:
40916
diff
changeset

440 
if not (null (Term.add_tfree_names check_goal [])) then 
c600f6ae4b09
only instantiate type variable if there exists some in quickcheck
bulwahn
parents:
40916
diff
changeset

441 
map (fn T => 
42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

442 
(pair (SOME T) o Term o apfst (Object_Logic.atomize_term thy)) 
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

443 
(map_goal_and_eval_terms (monomorphic_term thy insts T) (check_goal, eval_terms)) 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

444 
handle WELLSORTED s => (SOME T, Wellsorted_Error s)) default_insts 
40926
c600f6ae4b09
only instantiate type variable if there exists some in quickcheck
bulwahn
parents:
40916
diff
changeset

445 
else 
43113
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

446 
[(NONE, Term (Object_Logic.atomize_term thy check_goal, eval_terms))]) goals 
41517  447 
val error_msg = 
448 
cat_lines 

449 
(maps (map_filter (fn (_, Term t) => NONE  (_, Wellsorted_Error s) => SOME s)) inst_goals) 

41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

450 
fun is_wellsorted_term (T, Term t) = SOME (T, t) 
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

451 
 is_wellsorted_term (_, Wellsorted_Error s) = NONE 
37913
e85f5ad02a8f
correcting wellsortedness check and improving error message
bulwahn
parents:
37912
diff
changeset

452 
val correct_inst_goals = 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

453 
case map (map_filter is_wellsorted_term) inst_goals of 
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

454 
[[]] => error error_msg 
37913
e85f5ad02a8f
correcting wellsortedness check and improving error message
bulwahn
parents:
37912
diff
changeset

455 
 xs => xs 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

456 
val _ = if Config.get lthy quiet then () else warning error_msg 
43113
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

457 
in 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

458 
correct_inst_goals 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

459 
end 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

460 

7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

461 
fun collect_results f [] results = results 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

462 
 collect_results f (t :: ts) results = 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

463 
let 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

464 
val result = f t 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

465 
in 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

466 
if found_counterexample result then 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

467 
(result :: results) 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

468 
else 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

469 
collect_results f ts (result :: results) 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

470 
end 
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

471 

43147
70337ff0352d
moving the distinction between invocation of testers and generators into test_goal_terms function in quickcheck for its usage with mutabelle
bulwahn
parents:
43114
diff
changeset

472 
fun test_goal_terms ctxt (limit_time, is_interactive) insts goals = 
43113
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

473 
let 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

474 
fun test_term' goal = 
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

475 
case goal of 
43147
70337ff0352d
moving the distinction between invocation of testers and generators into test_goal_terms function in quickcheck for its usage with mutabelle
bulwahn
parents:
43114
diff
changeset

476 
[(NONE, t)] => test_term ctxt (limit_time, is_interactive) t 
70337ff0352d
moving the distinction between invocation of testers and generators into test_goal_terms function in quickcheck for its usage with mutabelle
bulwahn
parents:
43114
diff
changeset

477 
 ts => test_term_with_increasing_cardinality ctxt (limit_time, is_interactive) (map snd ts) 
70337ff0352d
moving the distinction between invocation of testers and generators into test_goal_terms function in quickcheck for its usage with mutabelle
bulwahn
parents:
43114
diff
changeset

478 
val correct_inst_goals = instantiate_goals ctxt insts goals 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

479 
in 
43147
70337ff0352d
moving the distinction between invocation of testers and generators into test_goal_terms function in quickcheck for its usage with mutabelle
bulwahn
parents:
43114
diff
changeset

480 
case lookup_tester ctxt (Config.get ctxt tester) of 
70337ff0352d
moving the distinction between invocation of testers and generators into test_goal_terms function in quickcheck for its usage with mutabelle
bulwahn
parents:
43114
diff
changeset

481 
SOME test_goal_terms => test_goal_terms ctxt (limit_time, is_interactive) insts goals 
70337ff0352d
moving the distinction between invocation of testers and generators into test_goal_terms function in quickcheck for its usage with mutabelle
bulwahn
parents:
43114
diff
changeset

482 
 NONE => 
70337ff0352d
moving the distinction between invocation of testers and generators into test_goal_terms function in quickcheck for its usage with mutabelle
bulwahn
parents:
43114
diff
changeset

483 
if Config.get ctxt finite_types then 
70337ff0352d
moving the distinction between invocation of testers and generators into test_goal_terms function in quickcheck for its usage with mutabelle
bulwahn
parents:
43114
diff
changeset

484 
collect_results test_term' correct_inst_goals [] 
70337ff0352d
moving the distinction between invocation of testers and generators into test_goal_terms function in quickcheck for its usage with mutabelle
bulwahn
parents:
43114
diff
changeset

485 
else 
70337ff0352d
moving the distinction between invocation of testers and generators into test_goal_terms function in quickcheck for its usage with mutabelle
bulwahn
parents:
43114
diff
changeset

486 
collect_results (test_term ctxt (limit_time, is_interactive)) (maps (map snd) correct_inst_goals) [] 
41043
3750bdac1327
testing smartly in two dimensions (cardinality and size) in quickcheck
bulwahn
parents:
40931
diff
changeset

487 
end; 
37912  488 

42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

489 
fun test_goal (time_limit, is_interactive) (insts, eval_terms) i state = 
40648
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

490 
let 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

491 
val lthy = Proof.context_of state; 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

492 
val thy = Proof.theory_of state; 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

493 
fun strip (Const ("all", _) $ Abs (_, _, t)) = strip t 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

494 
 strip t = t; 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

495 
val {goal = st, ...} = Proof.raw_goal state; 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

496 
val (gi, frees) = Logic.goal_params (prop_of st) i; 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

497 
val some_locale = case (Option.map #target o Named_Target.peek) lthy 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

498 
of NONE => NONE 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

499 
 SOME "" => NONE 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

500 
 SOME locale => SOME locale; 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

501 
val assms = if Config.get lthy no_assms then [] else case some_locale 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

502 
of NONE => Assumption.all_assms_of lthy 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

503 
 SOME locale => Assumption.local_assms_of lthy (Locale.init locale thy); 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

504 
val proto_goal = Logic.list_implies (map Thm.term_of assms, subst_bounds (frees, strip gi)); 
43113
7226051e8b89
splitting test_goal_terms in Quickcheck into smaller basic functions
bulwahn
parents:
43112
diff
changeset

505 
val goals = case some_locale 
42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

506 
of NONE => [(proto_goal, eval_terms)] 
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

507 
 SOME locale => 
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

508 
map (fn (_, phi) => (Morphism.term phi proto_goal, map (Morphism.term phi) eval_terms)) 
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

509 
(Locale.registrations_of (Context.Theory thy) (*FIXME*) locale); 
40648
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

510 
in 
43147
70337ff0352d
moving the distinction between invocation of testers and generators into test_goal_terms function in quickcheck for its usage with mutabelle
bulwahn
parents:
43114
diff
changeset

511 
test_goal_terms lthy (time_limit, is_interactive) insts goals 
40648
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

512 
end 
1598ec648b0d
splitting test_goal function in two functions; exporting new configurations in quickcheck; iterations depend on generator_name in quickcheck
bulwahn
parents:
40647
diff
changeset

513 

37912  514 
(* pretty printing *) 
28315  515 

40225  516 
fun tool_name auto = (if auto then "Auto " else "") ^ "Quickcheck" 
517 

518 
fun pretty_counterex ctxt auto NONE = Pretty.str (tool_name auto ^ " found no counterexample.") 

42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

519 
 pretty_counterex ctxt auto (SOME (cex, eval_terms)) = 
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

520 
Pretty.chunks ((Pretty.str (tool_name auto ^ " found a counterexample:\n") :: 
28315  521 
map (fn (s, t) => 
42028
bd6515e113b2
passing a term with free variables to the quickcheck tester functions instead of an lambda expression because this is more natural with passing further evaluation terms; added output of evaluation terms; added evaluation of terms in the exhaustive testing
bulwahn
parents:
42026
diff
changeset

522 
Pretty.block [Pretty.str (s ^ " ="), Pretty.brk 1, Syntax.pretty_term ctxt t]) (rev cex)) 
43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

523 
@ (if null eval_terms then [] 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

524 
else (Pretty.str ("Evaluated terms:\n") :: 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

525 
map (fn (t, u) => 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

526 
Pretty.block [Syntax.pretty_term ctxt t, Pretty.str " =", Pretty.brk 1, Syntax.pretty_term ctxt u]) 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

527 
(rev eval_terms)))); 
28315  528 

35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

529 
fun pretty_report (Report {iterations = iterations, raised_match_errors = raised_match_errors, 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

530 
satisfied_assms = satisfied_assms, positive_concl_tests = positive_concl_tests}) = 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

531 
let 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

532 
fun pretty_stat s i = Pretty.block ([Pretty.str (s ^ ": " ^ string_of_int i)]) 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

533 
in 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

534 
([pretty_stat "iterations" iterations, 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

535 
pretty_stat "match exceptions" raised_match_errors] 
41517  536 
@ map_index 
537 
(fn (i, n) => pretty_stat ("satisfied " ^ string_of_int (i + 1) ^ ". assumption") n) 

35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

538 
satisfied_assms 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

539 
@ [pretty_stat "positive conclusion tests" positive_concl_tests]) 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

540 
end 
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

541 

35380
6ac5b81a763d
adopting Mutabelle to quickcheck reporting; improving quickcheck reporting
bulwahn
parents:
35379
diff
changeset

542 
fun pretty_reports ctxt (SOME reports) = 
40916  543 
Pretty.chunks (Pretty.fbrk :: Pretty.str "Quickcheck report:" :: 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

544 
maps (fn (size, report) => 
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

545 
Pretty.str ("size " ^ string_of_int size ^ ":") :: pretty_report report @ [Pretty.brk 1]) 
35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

546 
(rev reports)) 
35380
6ac5b81a763d
adopting Mutabelle to quickcheck reporting; improving quickcheck reporting
bulwahn
parents:
35379
diff
changeset

547 
 pretty_reports ctxt NONE = Pretty.str "" 
35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

548 

42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

549 
fun pretty_timings timings = 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

550 
Pretty.chunks (Pretty.fbrk :: Pretty.str "timings:" :: 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

551 
maps (fn (label, time) => 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

552 
Pretty.str (label ^ ": " ^ string_of_int time ^ " ms") :: Pretty.brk 1 :: []) (rev timings)) 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

553 

42433
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

554 
fun pretty_counterex_and_reports ctxt auto [] = 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

555 
Pretty.chunks [Pretty.strs (tool_name auto :: 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

556 
space_explode " " "is used in a locale where no interpretation is provided."), 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

557 
Pretty.strs (space_explode " " "Please provide an executable interpretation for the locale.")] 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

558 
 pretty_counterex_and_reports ctxt auto (result :: _) = 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

559 
Pretty.chunks (pretty_counterex ctxt auto (response_of result) :: 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

560 
(* map (pretty_reports ctxt) (reports_of result) :: *) 
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

561 
(if Config.get ctxt timing then cons (pretty_timings (timings_of result)) else I) []) 
28315  562 

30980  563 
(* Isar commands *) 
28315  564 

28336  565 
fun read_nat s = case (Library.read_int o Symbol.explode) s 
566 
of (k, []) => if k >= 0 then k 

567 
else error ("Not a natural number: " ^ s) 

568 
 (_, _ :: _) => error ("Not a natural number: " ^ s); 

37909
583543ad6ad1
changed default types to a list of types; extended quickcheck parameters to be a list of values to parse a list of default types
bulwahn
parents:
36960
diff
changeset

569 

34128
8650a073dd9b
made Quickcheck take structured proof assumptions into account (like Refute and Nitpick) by default;
blanchet
parents:
33583
diff
changeset

570 
fun read_bool "false" = false 
8650a073dd9b
made Quickcheck take structured proof assumptions into account (like Refute and Nitpick) by default;
blanchet
parents:
33583
diff
changeset

571 
 read_bool "true" = true 
8650a073dd9b
made Quickcheck take structured proof assumptions into account (like Refute and Nitpick) by default;
blanchet
parents:
33583
diff
changeset

572 
 read_bool s = error ("Not a Boolean value: " ^ s) 
28315  573 

40366
a2866dbfbe6b
changing timeout to real value; handling Interrupt and Timeout more like nitpick does
bulwahn
parents:
40253
diff
changeset

574 
fun read_real s = 
a2866dbfbe6b
changing timeout to real value; handling Interrupt and Timeout more like nitpick does
bulwahn
parents:
40253
diff
changeset

575 
case (Real.fromString s) of 
a2866dbfbe6b
changing timeout to real value; handling Interrupt and Timeout more like nitpick does
bulwahn
parents:
40253
diff
changeset

576 
SOME s => s 
a2866dbfbe6b
changing timeout to real value; handling Interrupt and Timeout more like nitpick does
bulwahn
parents:
40253
diff
changeset

577 
 NONE => error ("Not a real number: " ^ s) 
a2866dbfbe6b
changing timeout to real value; handling Interrupt and Timeout more like nitpick does
bulwahn
parents:
40253
diff
changeset

578 

37929
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

579 
fun read_expectation "no_expectation" = No_Expectation 
41517  580 
 read_expectation "no_counterexample" = No_Counterexample 
37929
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

581 
 read_expectation "counterexample" = Counterexample 
41517  582 
 read_expectation s = error ("Not an expectation value: " ^ s) 
37929
22e0797857e6
adding checking of expected result for the tool quickcheck; annotated a few quickcheck examples
bulwahn
parents:
37913
diff
changeset

583 

43114
b9fca691addd
Quickcheck Narrowing only requires one compilation with GHC now
bulwahn
parents:
43113
diff
changeset

584 
fun valid_tester_name genctxt name = 
b9fca691addd
Quickcheck Narrowing only requires one compilation with GHC now
bulwahn
parents:
43113
diff
changeset

585 
AList.defined (op =) (fst (fst (fst (Data.get genctxt)))) name orelse 
b9fca691addd
Quickcheck Narrowing only requires one compilation with GHC now
bulwahn
parents:
43113
diff
changeset

586 
AList.defined (op =) (snd (fst (fst (Data.get genctxt)))) name 
40912
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

587 

1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

588 
fun parse_tester name genctxt = 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

589 
if valid_tester_name genctxt name then 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

590 
Config.put_generic tester name genctxt 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

591 
else 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

592 
error ("Unknown tester: " ^ name) 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

593 

1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

594 
fun parse_test_param ("tester", [arg]) = parse_tester arg 
40909
e006d1e06920
renamed parameter from generator to tester; quickcheck only applies one tester on invocation
bulwahn
parents:
40908
diff
changeset

595 
 parse_test_param ("size", [arg]) = Config.put_generic size (read_nat arg) 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

596 
 parse_test_param ("iterations", [arg]) = Config.put_generic iterations (read_nat arg) 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

597 
 parse_test_param ("default_type", arg) = (fn gen_ctxt => 
41517  598 
map_test_params 
42361  599 
((apfst o K) (map (Proof_Context.read_typ (Context.proof_of gen_ctxt)) arg)) gen_ctxt) 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

600 
 parse_test_param ("no_assms", [arg]) = Config.put_generic no_assms (read_bool arg) 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

601 
 parse_test_param ("expect", [arg]) = map_test_params ((apsnd o K) (read_expectation arg)) 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

602 
 parse_test_param ("report", [arg]) = Config.put_generic report (read_bool arg) 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

603 
 parse_test_param ("quiet", [arg]) = Config.put_generic quiet (read_bool arg) 
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

604 
 parse_test_param ("timeout", [arg]) = Config.put_generic timeout (read_real arg) 
40646  605 
 parse_test_param ("finite_types", [arg]) = Config.put_generic finite_types (read_bool arg) 
41517  606 
 parse_test_param ("finite_type_size", [arg]) = 
607 
Config.put_generic finite_type_size (read_nat arg) 

40912
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

608 
 parse_test_param (name, _) = fn genctxt => 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

609 
if valid_tester_name genctxt name then 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

610 
Config.put_generic tester name genctxt 
1108529100ce
checking if parameter is name of a tester which allows e.g. quickcheck[random]
bulwahn
parents:
40911
diff
changeset

611 
else error ("Unknown tester or test parameter: " ^ name); 
28315  612 

42025  613 
fun parse_test_param_inst (name, arg) ((insts, eval_terms), ctxt) = 
42361  614 
case try (Proof_Context.read_typ ctxt) name 
42025  615 
of SOME (TFree (v, _)) => 
42361  616 
((AList.update (op =) (v, Proof_Context.read_typ ctxt (the_single arg)) insts, eval_terms), ctxt) 
42025  617 
 NONE => (case name of 
618 
"eval" => ((insts, eval_terms @ map (Syntax.read_term ctxt) arg), ctxt) 

619 
 _ => ((insts, eval_terms), Context.proof_map (parse_test_param (name, arg)) ctxt)); 

28309  620 

40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

621 
fun quickcheck_params_cmd args = Context.theory_map (fold parse_test_param args); 
41517  622 

42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

623 
fun check_expectation state results = 
42433
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

624 
(if exists found_counterexample results andalso expect (Proof.context_of state) = No_Counterexample 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

625 
then 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

626 
error ("quickcheck expected to find no counterexample but found one") 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

627 
else 
42433
b48d9186e883
handling the case where quickcheck is used in a locale with no known interpretation userfriendly
bulwahn
parents:
42361
diff
changeset

628 
(if not (exists found_counterexample results) andalso expect (Proof.context_of state) = Counterexample 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

629 
then 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

630 
error ("quickcheck expected to find a counterexample but did not find one") 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

631 
else ())) 
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

632 

35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

633 
fun gen_quickcheck args i state = 
40644
0850a2a16dce
changed oldstyle quickcheck configurations to new Config.T configurations
bulwahn
parents:
40643
diff
changeset

634 
state 
42025  635 
> Proof.map_context_result (fn ctxt => fold parse_test_param_inst args (([], []), ctxt)) 
42026
da9b58f1db42
adding option of evaluating terms after invocation in quickcheck
bulwahn
parents:
42025
diff
changeset

636 
> (fn ((insts, eval_terms), state') => test_goal (true, true) (insts, eval_terms) i state' 
42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

637 
> tap (check_expectation state')) 
32297  638 

42088
8d00484551fe
making quickcheck's result value more formal; allowing more result information to be returned after timeout; adding output of timing information in quickcheck
bulwahn
parents:
42087
diff
changeset

639 
fun quickcheck args i state = counterexample_of (hd (gen_quickcheck args i state)) 
35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

640 

32297  641 
fun quickcheck_cmd args i state = 
35378
95d0e3adf38e
added basic reporting of test cases to quickcheck
bulwahn
parents:
35324
diff
changeset

642 
gen_quickcheck args i (Toplevel.proof_of state) 
43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

643 
> Output.urgent_message o Pretty.string_of 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

644 
o pretty_counterex_and_reports (Toplevel.context_of state) false; 
28309  645 

41517  646 
val parse_arg = 
647 
Parse.name  

648 
(Scan.optional (Parse.$$$ "="  

649 
(((Parse.name  Parse.float_number) >> single)  

650 
(Parse.$$$ "["  Parse.list1 Parse.name  Parse.$$$ "]"))) ["true"]); 

28309  651 

41517  652 
val parse_args = 
653 
Parse.$$$ "["  Parse.list1 parse_arg  Parse.$$$ "]"  Scan.succeed []; 

28336  654 

36960
01594f816e3a
prefer structure Keyword, Parse, Parse_Spec, Outer_Syntax;
wenzelm
parents:
36610
diff
changeset

655 
val _ = 
43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

656 
Outer_Syntax.command quickcheck_paramsN "set parameters for random testing" Keyword.thy_decl 
36960
01594f816e3a
prefer structure Keyword, Parse, Parse_Spec, Outer_Syntax;
wenzelm
parents:
36610
diff
changeset

657 
(parse_args >> (fn args => Toplevel.theory (quickcheck_params_cmd args))); 
28309  658 

36960
01594f816e3a
prefer structure Keyword, Parse, Parse_Spec, Outer_Syntax;
wenzelm
parents:
36610
diff
changeset

659 
val _ = 
43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

660 
Outer_Syntax.improper_command quickcheckN "try to find counterexample for subgoal" Keyword.diag 
36960
01594f816e3a
prefer structure Keyword, Parse, Parse_Spec, Outer_Syntax;
wenzelm
parents:
36610
diff
changeset

661 
(parse_args  Scan.optional Parse.nat 1 
01594f816e3a
prefer structure Keyword, Parse, Parse_Spec, Outer_Syntax;
wenzelm
parents:
36610
diff
changeset

662 
>> (fn (args, i) => Toplevel.no_timing o Toplevel.keep (quickcheck_cmd args i))); 
28309  663 

43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

664 
(* automatic testing *) 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

665 

abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

666 
fun try_quickcheck auto state = 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

667 
let 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

668 
val ctxt = Proof.context_of state; 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

669 
val i = 1; 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

670 
val res = 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

671 
state 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

672 
> Proof.map_context (Config.put report false #> Config.put quiet true) 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

673 
> try (test_goal (false, false) ([], []) i); 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

674 
in 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

675 
case res of 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

676 
NONE => (unknownN, state) 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

677 
 SOME results => 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

678 
let 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

679 
val msg = pretty_counterex_and_reports ctxt auto results 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

680 
in 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

681 
if exists found_counterexample results then 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

682 
(genuineN, 
43029
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

683 
state 
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

684 
> (if auto then 
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

685 
Proof.goal_message (K (Pretty.chunks [Pretty.str "", 
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

686 
Pretty.mark Markup.hilite msg])) 
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

687 
else 
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

688 
tap (fn _ => Output.urgent_message (Pretty.string_of msg)))) 
43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

689 
else 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

690 
(noneN, state) 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

691 
end 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

692 
end 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

693 
> `(fn (outcome_code, _) => outcome_code = genuineN) 
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

694 

43029
3e060b1c844b
use helpers and tweak Quickcheck's priority to it comes second (to give Solve Direct slightly more time before another prover runs)
blanchet
parents:
43024
diff
changeset

695 
val setup = Try.register_tool (quickcheckN, (20, auto, try_quickcheck)) 
43020
abb5d1f907e4
added "try" command, to launch Solve Direct, Quickcheck, Nitpick, Sledgehammer, and Try Methods
blanchet
parents:
43018
diff
changeset

696 

28315  697 
end; 
28309  698 

28315  699 
val auto_quickcheck = Quickcheck.auto; 