author | wenzelm |
Sun, 13 Mar 2011 16:52:59 +0100 | |
changeset 41949 | f9a2e10c49cb |
parent 41309 | 2e9bf718a7a1 |
child 42119 | 21714b0de625 |
permissions | -rwxr-xr-x |
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
1 |
#!/usr/bin/env bash |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
2 |
# |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
3 |
# Author: Lukas Bulwahn |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
4 |
# |
41309
2e9bf718a7a1
some attempts to fit diagnostic output into regular TTY (75-80 characters per line);
wenzelm
parents:
41191
diff
changeset
|
5 |
# DESCRIPTION: mutant-testing for counterexample generators and automated tools |
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
6 |
|
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
7 |
|
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
8 |
PRG="$(basename "$0")" |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
9 |
|
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
10 |
function usage() { |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
11 |
echo |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
12 |
echo "Usage: isabelle $PRG [OPTIONS] THEORY" |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
13 |
echo |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
14 |
echo " Options are:" |
41949
f9a2e10c49cb
more conventional Mutabelle settings -- similar to Mirabelle;
wenzelm
parents:
41309
diff
changeset
|
15 |
echo " -L LOGIC parent logic to use (default $MUTABELLE_LOGIC)" |
f9a2e10c49cb
more conventional Mutabelle settings -- similar to Mirabelle;
wenzelm
parents:
41309
diff
changeset
|
16 |
echo " -T THEORY parent theory to use (default $MUTABELLE_IMPORT_THEORY)" |
f9a2e10c49cb
more conventional Mutabelle settings -- similar to Mirabelle;
wenzelm
parents:
41309
diff
changeset
|
17 |
echo " -O DIR output directory for test data (default $MUTABELLE_OUTPUT_PATH)" |
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
18 |
echo |
41309
2e9bf718a7a1
some attempts to fit diagnostic output into regular TTY (75-80 characters per line);
wenzelm
parents:
41191
diff
changeset
|
19 |
echo " THEORY is the name of the theory of which all theorems should be" |
2e9bf718a7a1
some attempts to fit diagnostic output into regular TTY (75-80 characters per line);
wenzelm
parents:
41191
diff
changeset
|
20 |
echo " mutated and tested." |
41077 | 21 |
echo |
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
22 |
exit 1 |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
23 |
} |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
24 |
|
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
25 |
|
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
26 |
## process command line |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
27 |
|
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
28 |
# options |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
29 |
|
41949
f9a2e10c49cb
more conventional Mutabelle settings -- similar to Mirabelle;
wenzelm
parents:
41309
diff
changeset
|
30 |
MUTABELLE_IMPORTS="" |
41077 | 31 |
|
32 |
while getopts "L:T:O:" OPT |
|
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
33 |
do |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
34 |
case "$OPT" in |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
35 |
L) |
41021 | 36 |
MUTABELLE_LOGIC="$OPTARG" |
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
37 |
;; |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
38 |
T) |
41949
f9a2e10c49cb
more conventional Mutabelle settings -- similar to Mirabelle;
wenzelm
parents:
41309
diff
changeset
|
39 |
MUTABELLE_IMPORTS="$MUTABELLE_IMPORTS \"$OPTARG\"" |
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
40 |
;; |
41077 | 41 |
O) |
41021 | 42 |
MUTABELLE_OUTPUT_PATH="$OPTARG" |
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
43 |
;; |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
44 |
\?) |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
45 |
usage |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
46 |
;; |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
47 |
esac |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
48 |
done |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
49 |
|
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
50 |
shift $(($OPTIND - 1)) |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
51 |
|
41949
f9a2e10c49cb
more conventional Mutabelle settings -- similar to Mirabelle;
wenzelm
parents:
41309
diff
changeset
|
52 |
if [ "$MUTABELLE_IMPORTS" = "" ] |
41077 | 53 |
then |
41949
f9a2e10c49cb
more conventional Mutabelle settings -- similar to Mirabelle;
wenzelm
parents:
41309
diff
changeset
|
54 |
MUTABELLE_IMPORTS="$MUTABELLE_IMPORT_THEORY" |
41077 | 55 |
fi |
56 |
||
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
57 |
[ "$#" -ne 1 ] && usage |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
58 |
|
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
59 |
MUTABELLE_TEST_THEORY="$1" |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
60 |
|
41077 | 61 |
export MUTABELLE_OUTPUT_PATH |
62 |
||
41949
f9a2e10c49cb
more conventional Mutabelle settings -- similar to Mirabelle;
wenzelm
parents:
41309
diff
changeset
|
63 |
|
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
64 |
## main |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
65 |
|
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
66 |
echo "Starting Mutabelle..." |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
67 |
|
41949
f9a2e10c49cb
more conventional Mutabelle settings -- similar to Mirabelle;
wenzelm
parents:
41309
diff
changeset
|
68 |
|
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
69 |
# setup |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
70 |
|
41077 | 71 |
mkdir -p "$MUTABELLE_OUTPUT_PATH" |
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
72 |
|
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
73 |
echo "theory Mutabelle_Test |
41949
f9a2e10c49cb
more conventional Mutabelle settings -- similar to Mirabelle;
wenzelm
parents:
41309
diff
changeset
|
74 |
imports $MUTABELLE_IMPORTS |
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
75 |
uses |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
76 |
\"$MUTABELLE_HOME/mutabelle.ML\" |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
77 |
\"$MUTABELLE_HOME/mutabelle_extra.ML\" |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
78 |
begin |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
79 |
|
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
80 |
ML {* |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
81 |
val mtds = [ |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
82 |
MutabelleExtra.quickcheck_mtd (Config.put Quickcheck.tester \"random\") \"random\", |
41191 | 83 |
MutabelleExtra.quickcheck_mtd (Config.put Quickcheck.tester \"exhaustive\") \"exhaustive\", |
84 |
MutabelleExtra.nitpick_mtd |
|
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
85 |
] |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
86 |
*} |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
87 |
|
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
88 |
ML {* |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
89 |
fun mutation_testing_of thy = |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
90 |
(MutabelleExtra.random_seed := 1.0; |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
91 |
MutabelleExtra.thms_of false thy |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
92 |
|> (fn thms => MutabelleExtra.mutate_theorems_and_write_report |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
93 |
@{theory} mtds thms (\"$MUTABELLE_OUTPUT_PATH/log\"))) |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
94 |
*} |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
95 |
|
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
96 |
ML {* |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
97 |
mutation_testing_of @{theory $MUTABELLE_TEST_THEORY} |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
98 |
*} |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
99 |
|
41077 | 100 |
end" > "$MUTABELLE_OUTPUT_PATH/Mutabelle_Test.thy" |
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
101 |
|
41949
f9a2e10c49cb
more conventional Mutabelle settings -- similar to Mirabelle;
wenzelm
parents:
41309
diff
changeset
|
102 |
|
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
103 |
# execution |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
104 |
|
41949
f9a2e10c49cb
more conventional Mutabelle settings -- similar to Mirabelle;
wenzelm
parents:
41309
diff
changeset
|
105 |
"$ISABELLE_PROCESS" -e 'use_thy "$MUTABELLE_OUTPUT_PATH/Mutabelle_Test"' -q "$MUTABELLE_LOGIC" > /dev/null 2>&1 |
41077 | 106 |
|
107 |
||
108 |
[ $? -ne 0 ] && echo "isabelle processing of mutabelle failed" |
|
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
109 |
|
41949
f9a2e10c49cb
more conventional Mutabelle settings -- similar to Mirabelle;
wenzelm
parents:
41309
diff
changeset
|
110 |
|
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
111 |
# make statistics |
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
112 |
|
41077 | 113 |
function count() { |
41191 | 114 |
cat "$MUTABELLE_OUTPUT_PATH/log" | grep "$1: $2" | wc -l |
41077 | 115 |
} |
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
116 |
|
41191 | 117 |
echo "random : C: $(count "quickcheck_random" "GenuineCex") N: $(count "quickcheck_random" "NoCex") \ |
118 |
T: $(count "quickcheck_random" "Timeout") E: $(count "quickcheck_random" "Error")" |
|
119 |
echo "exhaustive : C: $(count "quickcheck_exhaustive" "GenuineCex") N: $(count "quickcheck_exhaustive" "NoCex") \ |
|
120 |
T: $(count "quickcheck_exhaustive" "Timeout") E: $(count "quickcheck_exhaustive" "Error")" |
|
121 |
echo "nitpick : C: $(count "nitpick" "GenuineCex") N: $(count "nitpick" "NoCex") \ |
|
122 |
T: $(count "nitpick" "Timeout") E: $(count "nitpick" "Error")" |
|
40975
498f272b4bcb
adding mutabelle as a component and an isabelle tool to be used in regression testing
bulwahn
parents:
diff
changeset
|
123 |