src/HOL/TPTP/mash_eval.ML
changeset 57432 78d7fbe9b203
parent 57431 02c408aed5ee
child 57434 6ea8b8592787
equal deleted inserted replaced
57431:02c408aed5ee 57432:78d7fbe9b203
    45   let
    45   let
    46     val thy = Proof_Context.theory_of ctxt
    46     val thy = Proof_Context.theory_of ctxt
    47     val zeros = [0, 0, 0, 0, 0, 0]
    47     val zeros = [0, 0, 0, 0, 0, 0]
    48     val report_path = report_file_name |> Path.explode
    48     val report_path = report_file_name |> Path.explode
    49     val _ = File.write report_path ""
    49     val _ = File.write report_path ""
       
    50 
    50     fun print s = File.append report_path (s ^ "\n")
    51     fun print s = File.append report_path (s ^ "\n")
       
    52 
    51     val {provers, max_facts, slice, type_enc, lam_trans, timeout, ...} = default_params thy []
    53     val {provers, max_facts, slice, type_enc, lam_trans, timeout, ...} = default_params thy []
    52     val prover = hd provers
    54     val prover = hd provers
    53     val max_suggs = generous_max_suggestions (the max_facts)
    55     val max_suggs = generous_max_suggestions (the max_facts)
    54     val lines_of = Path.explode #> try File.read_lines #> these
    56     val lines_of = Path.explode #> try File.read_lines #> these
    55     val file_names =
    57     val file_names =
    57        mesh_isar_file_name, mesh_prover_file_name]
    59        mesh_isar_file_name, mesh_prover_file_name]
    58     val lines as [mepo_lines, mash_isar_lines, mash_prover_lines,
    60     val lines as [mepo_lines, mash_isar_lines, mash_prover_lines,
    59                   mesh_isar_lines, mesh_prover_lines] =
    61                   mesh_isar_lines, mesh_prover_lines] =
    60       map lines_of file_names
    62       map lines_of file_names
    61     val num_lines = fold (Integer.max o length) lines 0
    63     val num_lines = fold (Integer.max o length) lines 0
       
    64 
    62     fun pad lines = lines @ replicate (num_lines - length lines) ""
    65     fun pad lines = lines @ replicate (num_lines - length lines) ""
       
    66 
    63     val lines =
    67     val lines =
    64       pad mepo_lines ~~ pad mash_isar_lines ~~ pad mash_prover_lines ~~
    68       pad mepo_lines ~~ pad mash_isar_lines ~~ pad mash_prover_lines ~~
    65       pad mesh_isar_lines ~~ pad mesh_prover_lines
    69       pad mesh_isar_lines ~~ pad mesh_prover_lines
    66     val css = clasimpset_rule_table_of ctxt
    70     val css = clasimpset_rule_table_of ctxt
    67     val facts = all_facts ctxt true false Symtab.empty [] [] css
    71     val facts = all_facts ctxt true false Symtab.empty [] [] css
    68     val name_tabs = build_name_tables nickname_of_thm facts
    72     val name_tabs = build_name_tables nickname_of_thm facts
       
    73 
    69     fun with_index facts s = (find_index (curry (op =) s) facts + 1, s)
    74     fun with_index facts s = (find_index (curry (op =) s) facts + 1, s)
    70     fun index_str (j, s) = s ^ "@" ^ string_of_int j
    75     fun index_str (j, s) = s ^ "@" ^ string_of_int j
    71     val str_of_method = enclose "  " ": "
    76     val str_of_method = enclose "  " ": "
    72     fun str_of_result method facts ({outcome, run_time, used_facts, ...}
    77 
    73                                      : prover_result) =
    78     fun str_of_result method facts ({outcome, run_time, used_facts, ...} : prover_result) =
    74       let val facts = facts |> map (fst o fst) in
    79       let val facts = facts |> map (fst o fst) in
    75         str_of_method method ^
    80         str_of_method method ^
    76         (if is_none outcome then
    81         (if is_none outcome then
    77            "Success (" ^ ATP_Util.string_of_time run_time ^ "): " ^
    82            "Success (" ^ ATP_Util.string_of_time run_time ^ "): " ^
    78            (used_facts |> map (with_index facts o fst)
    83            (used_facts |> map (with_index facts o fst)
    87            "Failure: " ^
    92            "Failure: " ^
    88            (facts |> take (the max_facts) |> tag_list 1
    93            (facts |> take (the max_facts) |> tag_list 1
    89                   |> map index_str
    94                   |> map index_str
    90                   |> space_implode " "))
    95                   |> space_implode " "))
    91       end
    96       end
    92     fun solve_goal (j, ((((mepo_line, mash_isar_line), mash_prover_line),
    97 
    93                          mesh_isar_line), mesh_prover_line)) =
    98     fun solve_goal (j, ((((mepo_line, mash_isar_line), mash_prover_line), mesh_isar_line),
       
    99         mesh_prover_line)) =
    94       if in_range range j then
   100       if in_range range j then
    95         let
   101         let
    96           val get_suggs = extract_suggestions ##> (take max_suggs #> map fst)
   102           val get_suggs = extract_suggestions ##> (take max_suggs #> map fst)
    97           val (name1, mepo_suggs) = get_suggs mepo_line
   103           val (name1, mepo_suggs) = get_suggs mepo_line
    98           val (name2, mash_isar_suggs) = get_suggs mash_isar_line
   104           val (name2, mash_isar_suggs) = get_suggs mash_isar_line
   109             | NONE => error ("No fact called \"" ^ name ^ "\".")
   115             | NONE => error ("No fact called \"" ^ name ^ "\".")
   110           val goal = goal_of_thm (Proof_Context.theory_of ctxt) th
   116           val goal = goal_of_thm (Proof_Context.theory_of ctxt) th
   111           val (_, hyp_ts, concl_t) = ATP_Util.strip_subgoal goal 1 ctxt
   117           val (_, hyp_ts, concl_t) = ATP_Util.strip_subgoal goal 1 ctxt
   112           val isar_deps = these (isar_dependencies_of name_tabs th)
   118           val isar_deps = these (isar_dependencies_of name_tabs th)
   113           val facts = facts |> filter (fn (_, th') => thm_less (th', th))
   119           val facts = facts |> filter (fn (_, th') => thm_less (th', th))
       
   120 
   114           (* adapted from "mirabelle_sledgehammer.ML" *)
   121           (* adapted from "mirabelle_sledgehammer.ML" *)
   115           fun set_file_name method (SOME dir) =
   122           fun set_file_name method (SOME dir) =
   116               let
   123               let
   117                 val prob_prefix =
   124                 val prob_prefix = "goal_" ^ string_of_int j ^ "__" ^ encode_str name ^ "__" ^ method
   118                   "goal_" ^ string_of_int j ^ "__" ^ encode_str name ^ "__" ^
       
   119                   method
       
   120               in
   125               in
   121                 Config.put atp_dest_dir dir
   126                 Config.put atp_dest_dir dir
   122                 #> Config.put atp_problem_prefix (prob_prefix ^ "__")
   127                 #> Config.put atp_problem_prefix (prob_prefix ^ "__")
   123                 #> Config.put SMT_Config.debug_files (dir ^ "/" ^ prob_prefix)
   128                 #> Config.put SMT_Config.debug_files (dir ^ "/" ^ prob_prefix)
   124               end
   129               end
   125             | set_file_name _ NONE = I
   130             | set_file_name _ NONE = I
       
   131 
   126           fun prove method suggs =
   132           fun prove method suggs =
   127             if not (member (op =) methods method) orelse
   133             if not (member (op =) methods method) orelse
   128                (null facts andalso method <> IsarN) then
   134                (null facts andalso method <> IsarN) then
   129               (str_of_method method ^ "Skipped", 0)
   135               (str_of_method method ^ "Skipped", 0)
   130             else
   136             else
   131               let
   137               let
   132                 fun nickify ((_, stature), th) =
   138                 fun nickify ((_, stature), th) =
   133                   ((K (encode_str (nickname_of_thm th)), stature), th)
   139                   ((K (encode_str (nickname_of_thm th)), stature), th)
       
   140 
   134                 val facts =
   141                 val facts =
   135                   suggs
   142                   suggs
   136                   |> find_suggested_facts ctxt facts
   143                   |> find_suggested_facts ctxt facts
   137                   |> map (fact_of_raw_fact #> nickify)
   144                   |> map (fact_of_raw_fact #> nickify)
   138                   |> maybe_instantiate_inducts ctxt hyp_ts concl_t
   145                   |> maybe_instantiate_inducts ctxt hyp_ts concl_t
   140                   |> map fact_of_raw_fact
   147                   |> map fact_of_raw_fact
   141                 val ctxt = ctxt |> set_file_name method prob_dir_name
   148                 val ctxt = ctxt |> set_file_name method prob_dir_name
   142                 val res as {outcome, ...} =
   149                 val res as {outcome, ...} =
   143                   run_prover_for_mash ctxt params prover name facts goal
   150                   run_prover_for_mash ctxt params prover name facts goal
   144                 val ok = if is_none outcome then 1 else 0
   151                 val ok = if is_none outcome then 1 else 0
   145               in (str_of_result method facts res, ok) end
   152               in
       
   153                 (str_of_result method facts res, ok)
       
   154               end
       
   155 
   146           val ress =
   156           val ress =
   147             [fn () => prove MePoN mepo_suggs,
   157             [fn () => prove MePoN mepo_suggs,
   148              fn () => prove MaSh_IsarN mash_isar_suggs,
   158              fn () => prove MaSh_IsarN mash_isar_suggs,
   149              fn () => prove MaSh_ProverN mash_prover_suggs,
   159              fn () => prove MaSh_ProverN mash_prover_suggs,
   150              fn () => prove MeSh_IsarN mesh_isar_suggs,
   160              fn () => prove MeSh_IsarN mesh_isar_suggs,
   156           |> cat_lines |> print;
   166           |> cat_lines |> print;
   157           map snd ress
   167           map snd ress
   158         end
   168         end
   159       else
   169       else
   160         zeros
   170         zeros
       
   171 
   161     fun total_of method ok n =
   172     fun total_of method ok n =
   162       str_of_method method ^ string_of_int ok ^ " (" ^
   173       str_of_method method ^ string_of_int ok ^ " (" ^
   163       Real.fmt (StringCvt.FIX (SOME 1))
   174       Real.fmt (StringCvt.FIX (SOME 1))
   164                (100.0 * Real.fromInt ok / Real.fromInt (Int.max (1, n))) ^ "%)"
   175                (100.0 * Real.fromInt ok / Real.fromInt (Int.max (1, n))) ^ "%)"
       
   176 
   165     val inst_inducts = Config.get ctxt instantiate_inducts
   177     val inst_inducts = Config.get ctxt instantiate_inducts
   166     val options =
   178     val options =
   167       ["prover = " ^ prover,
   179       ["prover = " ^ prover,
   168        "max_facts = " ^ string_of_int (the max_facts),
   180        "max_facts = " ^ string_of_int (the max_facts),
   169        "slice" |> not slice ? prefix "dont_",
   181        "slice" |> not slice ? prefix "dont_",
   173        "instantiate_inducts" |> not inst_inducts ? prefix "dont_"]
   185        "instantiate_inducts" |> not inst_inducts ? prefix "dont_"]
   174     val _ = print " * * *";
   186     val _ = print " * * *";
   175     val _ = print ("Options: " ^ commas options);
   187     val _ = print ("Options: " ^ commas options);
   176     val oks = Par_List.map solve_goal (tag_list 1 lines)
   188     val oks = Par_List.map solve_goal (tag_list 1 lines)
   177     val n = length oks
   189     val n = length oks
   178     val [mepo_ok, mash_isar_ok, mash_prover_ok, mesh_isar_ok, mesh_prover_ok,
   190     val [mepo_ok, mash_isar_ok, mash_prover_ok, mesh_isar_ok, mesh_prover_ok, isar_ok] =
   179          isar_ok] =
       
   180       if n = 0 then zeros else map Integer.sum (map_transpose I oks)
   191       if n = 0 then zeros else map Integer.sum (map_transpose I oks)
   181   in
   192   in
   182     ["Successes (of " ^ string_of_int n ^ " goals)",
   193     ["Successes (of " ^ string_of_int n ^ " goals)",
   183      total_of MePoN mepo_ok n,
   194      total_of MePoN mepo_ok n,
   184      total_of MaSh_IsarN mash_isar_ok n,
   195      total_of MaSh_IsarN mash_isar_ok n,