src/HOL/Tools/Sledgehammer/sledgehammer_reconstruct.ML

put annotate in own structure

(*  Title:      HOL/Tools/Sledgehammer/sledgehammer_reconstruct.ML
    Author:     Jasmin Blanchette, TU Muenchen
    Author:     Steffen Juilf Smolka, TU Muenchen

Isar proof reconstruction from ATP proofs.
*)

signature SLEDGEHAMMER_PROOF_RECONSTRUCT =
sig
  type 'a proof = 'a ATP_Proof.proof
  type stature = ATP_Problem_Generate.stature

  datatype reconstructor =
    Metis of string * string |
    SMT

  datatype play =
    Played of reconstructor * Time.time |
    Trust_Playable of reconstructor * Time.time option |
    Failed_to_Play of reconstructor

  type minimize_command = string list -> string
  type one_line_params =
    play * string * (string * stature) list * minimize_command * int * int
  type isar_params =
    bool * bool * Time.time * real * string Symtab.table
    * (string * stature) list vector * int Symtab.table * string proof * thm

  val smtN : string
  val string_for_reconstructor : reconstructor -> string
  val thms_of_name : Proof.context -> string -> thm list
  val lam_trans_from_atp_proof : string proof -> string -> string
  val is_typed_helper_used_in_atp_proof : string proof -> bool
  val used_facts_in_atp_proof :
    Proof.context -> (string * stature) list vector -> string proof ->
    (string * stature) list
  val used_facts_in_unsound_atp_proof :
    Proof.context -> (string * stature) list vector -> 'a proof ->
    string list option
  val one_line_proof_text : int -> one_line_params -> string
  val isar_proof_text :
    Proof.context -> bool -> isar_params -> one_line_params -> string
  val proof_text :
    Proof.context -> bool -> isar_params -> int -> one_line_params -> string
end;

structure Sledgehammer_Reconstruct : SLEDGEHAMMER_PROOF_RECONSTRUCT =
struct

open ATP_Util
open ATP_Problem
open ATP_Proof
open ATP_Problem_Generate
open ATP_Proof_Reconstruct
open Sledgehammer_Util
open Sledgehammer_Annotate

structure String_Redirect = ATP_Proof_Redirect(
  type key = step_name
  val ord = fn ((s, _ : string list), (s', _)) => fast_string_ord (s, s')
  val string_of = fst)

open String_Redirect


(** reconstructors **)

datatype reconstructor =
  Metis of string * string |
  SMT

datatype play =
  Played of reconstructor * Time.time |
  Trust_Playable of reconstructor * Time.time option |
  Failed_to_Play of reconstructor

val smtN = "smt"

fun string_for_reconstructor (Metis (type_enc, lam_trans)) =
    metis_call type_enc lam_trans
  | string_for_reconstructor SMT = smtN

fun thms_of_name ctxt name =
  let
    val lex = Keyword.get_lexicons
    val get = maps (Proof_Context.get_fact ctxt o fst)
  in
    Source.of_string name
    |> Symbol.source
    |> Token.source {do_recover = SOME false} lex Position.start
    |> Token.source_proper
    |> Source.source Token.stopper (Parse_Spec.xthms1 >> get) NONE
    |> Source.exhaust
  end


(** fact extraction from ATP proofs **)

fun find_first_in_list_vector vec key =
  Vector.foldl (fn (ps, NONE) => AList.lookup (op =) ps key
                 | (_, value) => value) NONE vec

val unprefix_fact_number = space_implode "_" o tl o space_explode "_"

fun resolve_one_named_fact fact_names s =
  case try (unprefix fact_prefix) s of
    SOME s' =>
    let val s' = s' |> unprefix_fact_number |> unascii_of in
      s' |> find_first_in_list_vector fact_names |> Option.map (pair s')
    end
  | NONE => NONE
fun resolve_fact fact_names = map_filter (resolve_one_named_fact fact_names)
fun is_fact fact_names = not o null o resolve_fact fact_names

fun resolve_one_named_conjecture s =
  case try (unprefix conjecture_prefix) s of
    SOME s' => Int.fromString s'
  | NONE => NONE

val resolve_conjecture = map_filter resolve_one_named_conjecture
val is_conjecture = not o null o resolve_conjecture

val ascii_of_lam_fact_prefix = ascii_of lam_fact_prefix

(* overapproximation (good enough) *)
fun is_lam_lifted s =
  String.isPrefix fact_prefix s andalso
  String.isSubstring ascii_of_lam_fact_prefix s

val is_combinator_def = String.isPrefix (helper_prefix ^ combinator_prefix)

fun is_axiom_used_in_proof pred =
  exists (fn Inference_Step ((_, ss), _, _, _, []) => exists pred ss
           | _ => false)

fun lam_trans_from_atp_proof atp_proof default =
  case (is_axiom_used_in_proof is_combinator_def atp_proof,
        is_axiom_used_in_proof is_lam_lifted atp_proof) of
    (false, false) => default
  | (false, true) => liftingN
(*  | (true, true) => combs_and_liftingN -- not supported by "metis" *)
  | (true, _) => combsN

val is_typed_helper_name =
  String.isPrefix helper_prefix andf String.isSuffix typed_helper_suffix
fun is_typed_helper_used_in_atp_proof atp_proof =
  is_axiom_used_in_proof is_typed_helper_name atp_proof

fun add_non_rec_defs fact_names accum =
  Vector.foldl (fn (facts, facts') =>
      union (op =) (filter (fn (_, (_, status)) => status = Non_Rec_Def) facts)
            facts')
    accum fact_names

val isa_ext = Thm.get_name_hint @{thm ext}
val isa_short_ext = Long_Name.base_name isa_ext

fun ext_name ctxt =
  if Thm.eq_thm_prop (@{thm ext},
         singleton (Attrib.eval_thms ctxt) (Facts.named isa_short_ext, [])) then
    isa_short_ext
  else
    isa_ext

val leo2_ext = "extcnf_equal_neg"
val leo2_unfold_def = "unfold_def"

fun add_fact ctxt fact_names (Inference_Step ((_, ss), _, _, rule, deps)) =
    (if rule = leo2_ext then
       insert (op =) (ext_name ctxt, (Global, General))
     else if rule = leo2_unfold_def then
       (* LEO 1.3.3 does not record definitions properly, leading to missing
         dependencies in the TSTP proof. Remove the next line once this is
         fixed. *)
       add_non_rec_defs fact_names
     else if rule = satallax_coreN then
       (fn [] =>
           (* Satallax doesn't include definitions in its unsatisfiable cores,
              so we assume the worst and include them all here. *)
           [(ext_name ctxt, (Global, General))] |> add_non_rec_defs fact_names
         | facts => facts)
     else
       I)
    #> (if null deps then union (op =) (resolve_fact fact_names ss)
        else I)
  | add_fact _ _ _ = I

fun used_facts_in_atp_proof ctxt fact_names atp_proof =
  if null atp_proof then Vector.foldl (uncurry (union (op =))) [] fact_names
  else fold (add_fact ctxt fact_names) atp_proof []

fun used_facts_in_unsound_atp_proof _ _ [] = NONE
  | used_facts_in_unsound_atp_proof ctxt fact_names atp_proof =
    let val used_facts = used_facts_in_atp_proof ctxt fact_names atp_proof in
      if forall (fn (_, (sc, _)) => sc = Global) used_facts andalso
         not (is_axiom_used_in_proof (is_conjecture o single) atp_proof) then
        SOME (map fst used_facts)
      else
        NONE
    end


(** one-liner reconstructor proofs **)

fun string_for_label (s, num) = s ^ string_of_int num

fun show_time NONE = ""
  | show_time (SOME ext_time) = " (" ^ string_from_ext_time ext_time ^ ")"

(* FIXME: Various bugs, esp. with "unfolding"
fun unusing_chained_facts _ 0 = ""
  | unusing_chained_facts used_chaineds num_chained =
    if length used_chaineds = num_chained then ""
    else if null used_chaineds then "(* using no facts *) "
    else "(* using only " ^ space_implode " " used_chaineds ^ " *) "
*)

fun apply_on_subgoal _ 1 = "by "
  | apply_on_subgoal 1 _ = "apply "
  | apply_on_subgoal i n =
    "prefer " ^ string_of_int i ^ " " ^ apply_on_subgoal 1 n

fun using_labels [] = ""
  | using_labels ls =
    "using " ^ space_implode " " (map string_for_label ls) ^ " "

fun command_call name [] =
    name |> not (Symbol_Pos.is_identifier name) ? enclose "(" ")"
  | command_call name args = "(" ^ name ^ " " ^ space_implode " " args ^ ")"

fun reconstructor_command reconstr i n used_chaineds num_chained (ls, ss) =
  (* unusing_chained_facts used_chaineds num_chained ^ *)
  using_labels ls ^ apply_on_subgoal i n ^
  command_call (string_for_reconstructor reconstr) ss

fun try_command_line banner time command =
  banner ^ ": " ^ Sendback.markup command ^ show_time time ^ "."

fun minimize_line _ [] = ""
  | minimize_line minimize_command ss =
    case minimize_command ss of
      "" => ""
    | command =>
      "\nTo minimize: " ^ Sendback.markup command ^ "."

fun split_used_facts facts =
  facts |> List.partition (fn (_, (sc, _)) => sc = Chained)
        |> pairself (sort_distinct (string_ord o pairself fst))

type minimize_command = string list -> string
type one_line_params =
  play * string * (string * stature) list * minimize_command * int * int

fun one_line_proof_text num_chained
        (preplay, banner, used_facts, minimize_command, subgoal,
         subgoal_count) =
  let
    val (chained, extra) = split_used_facts used_facts
    val (failed, reconstr, ext_time) =
      case preplay of
        Played (reconstr, time) => (false, reconstr, (SOME (false, time)))
      | Trust_Playable (reconstr, time) =>
        (false, reconstr,
         case time of
           NONE => NONE
         | SOME time =>
           if time = Time.zeroTime then NONE else SOME (true, time))
      | Failed_to_Play reconstr => (true, reconstr, NONE)
    val try_line =
      ([], map fst extra)
      |> reconstructor_command reconstr subgoal subgoal_count (map fst chained)
                               num_chained
      |> (if failed then
            enclose "One-line proof reconstruction failed: "
                     ".\n(Invoking \"sledgehammer\" with \"[strict]\" might \
                     \solve this.)"
          else
            try_command_line banner ext_time)
  in try_line ^ minimize_line minimize_command (map fst (extra @ chained)) end


(** Isar proof construction and manipulation **)

type label = string * int
type facts = label list * string list

datatype isar_qualifier = Show | Then | Moreover | Ultimately

datatype isar_step =
  Fix of (string * typ) list |
  Let of term * term |
  Assume of label * term |
  Prove of isar_qualifier list * label * term * byline
and byline =
  By_Metis of facts |
  Case_Split of isar_step list list * facts

val assume_prefix = "a"
val have_prefix = "f"
val raw_prefix = "x"

fun raw_label_for_name (num, ss) =
  case resolve_conjecture ss of
    [j] => (conjecture_prefix, j)
  | _ => (raw_prefix ^ ascii_of num, 0)

fun label_of_clause [name] = raw_label_for_name name
  | label_of_clause c = (space_implode "___" (map fst c), 0)

fun add_fact_from_dependencies fact_names (names as [(_, ss)]) =
    if is_fact fact_names ss then
      apsnd (union (op =) (map fst (resolve_fact fact_names ss)))
    else
      apfst (insert (op =) (label_of_clause names))
  | add_fact_from_dependencies fact_names names =
    apfst (insert (op =) (label_of_clause names))

fun repair_name "$true" = "c_True"
  | repair_name "$false" = "c_False"
  | repair_name "$$e" = tptp_equal (* seen in Vampire proofs *)
  | repair_name s =
    if is_tptp_equal s orelse
       (* seen in Vampire proofs *)
       (String.isPrefix "sQ" s andalso String.isSuffix "_eqProxy" s) then
      tptp_equal
    else
      s

fun unvarify_term (Var ((s, 0), T)) = Free (s, T)
  | unvarify_term t = raise TERM ("unvarify_term: non-Var", [t])

fun infer_formula_types ctxt =
  Type.constraint HOLogic.boolT
  #> Syntax.check_term
         (Proof_Context.set_mode Proof_Context.mode_schematic ctxt)

val combinator_table =
  [(@{const_name Meson.COMBI}, @{thm Meson.COMBI_def [abs_def]}),
   (@{const_name Meson.COMBK}, @{thm Meson.COMBK_def [abs_def]}),
   (@{const_name Meson.COMBB}, @{thm Meson.COMBB_def [abs_def]}),
   (@{const_name Meson.COMBC}, @{thm Meson.COMBC_def [abs_def]}),
   (@{const_name Meson.COMBS}, @{thm Meson.COMBS_def [abs_def]})]

fun uncombine_term thy =
  let
    fun aux (t1 $ t2) = betapply (pairself aux (t1, t2))
      | aux (Abs (s, T, t')) = Abs (s, T, aux t')
      | aux (t as Const (x as (s, _))) =
        (case AList.lookup (op =) combinator_table s of
           SOME thm => thm |> prop_of |> specialize_type thy x
                           |> Logic.dest_equals |> snd
         | NONE => t)
      | aux t = t
  in aux end

fun decode_line sym_tab (Definition_Step (name, phi1, phi2)) ctxt =
    let
      val thy = Proof_Context.theory_of ctxt
      val t1 = prop_from_atp ctxt true sym_tab phi1
      val vars = snd (strip_comb t1)
      val frees = map unvarify_term vars
      val unvarify_args = subst_atomic (vars ~~ frees)
      val t2 = prop_from_atp ctxt true sym_tab phi2
      val (t1, t2) =
        HOLogic.eq_const HOLogic.typeT $ t1 $ t2
        |> unvarify_args |> uncombine_term thy |> infer_formula_types ctxt
        |> HOLogic.dest_eq
    in
      (Definition_Step (name, t1, t2),
       fold Variable.declare_term (maps Misc_Legacy.term_frees [t1, t2]) ctxt)
    end
  | decode_line sym_tab (Inference_Step (name, role, u, rule, deps)) ctxt =
    let
      val thy = Proof_Context.theory_of ctxt
      val t = u |> prop_from_atp ctxt true sym_tab
                |> uncombine_term thy |> infer_formula_types ctxt
    in
      (Inference_Step (name, role, t, rule, deps),
       fold Variable.declare_term (Misc_Legacy.term_frees t) ctxt)
    end
fun decode_lines ctxt sym_tab lines =
  fst (fold_map (decode_line sym_tab) lines ctxt)

fun replace_one_dependency (old, new) dep =
  if is_same_atp_step dep old then new else [dep]
fun replace_dependencies_in_line _ (line as Definition_Step _) = line
  | replace_dependencies_in_line p
        (Inference_Step (name, role, t, rule, deps)) =
    Inference_Step (name, role, t, rule,
                    fold (union (op =) o replace_one_dependency p) deps [])

(* No "real" literals means only type information (tfree_tcs, clsrel, or
   clsarity). *)
fun is_only_type_information t = t aconv @{term True}

fun is_same_inference _ (Definition_Step _) = false
  | is_same_inference t (Inference_Step (_, _, t', _, _)) = t aconv t'

(* Discard facts; consolidate adjacent lines that prove the same formula, since
   they differ only in type information.*)
fun add_line _ (line as Definition_Step _) lines = line :: lines
  | add_line fact_names (Inference_Step (name as (_, ss), role, t, rule, []))
             lines =
    (* No dependencies: fact, conjecture, or (for Vampire) internal facts or
       definitions. *)
    if is_fact fact_names ss then
      (* Facts are not proof lines. *)
      if is_only_type_information t then
        map (replace_dependencies_in_line (name, [])) lines
      (* Is there a repetition? If so, replace later line by earlier one. *)
      else case take_prefix (not o is_same_inference t) lines of
        (_, []) => lines (* no repetition of proof line *)
      | (pre, Inference_Step (name', _, _, _, _) :: post) =>
        pre @ map (replace_dependencies_in_line (name', [name])) post
      | _ => raise Fail "unexpected inference"
    else if is_conjecture ss then
      Inference_Step (name, role, t, rule, []) :: lines
    else
      map (replace_dependencies_in_line (name, [])) lines
  | add_line _ (Inference_Step (name, role, t, rule, deps)) lines =
    (* Type information will be deleted later; skip repetition test. *)
    if is_only_type_information t then
      Inference_Step (name, role, t, rule, deps) :: lines
    (* Is there a repetition? If so, replace later line by earlier one. *)
    else case take_prefix (not o is_same_inference t) lines of
      (* FIXME: Doesn't this code risk conflating proofs involving different
         types? *)
       (_, []) => Inference_Step (name, role, t, rule, deps) :: lines
     | (pre, Inference_Step (name', role, t', rule, _) :: post) =>
       Inference_Step (name, role, t', rule, deps) ::
       pre @ map (replace_dependencies_in_line (name', [name])) post
     | _ => raise Fail "unexpected inference"

val waldmeister_conjecture_num = "1.0.0.0"

val repair_waldmeister_endgame =
  let
    fun do_tail (Inference_Step (name, role, t, rule, deps)) =
        Inference_Step (name, role, s_not t, rule, deps)
      | do_tail line = line
    fun do_body [] = []
      | do_body ((line as Inference_Step ((num, _), _, _, _, _)) :: lines) =
        if num = waldmeister_conjecture_num then map do_tail (line :: lines)
        else line :: do_body lines
      | do_body (line :: lines) = line :: do_body lines
  in do_body end

(* Recursively delete empty lines (type information) from the proof. *)
fun add_nontrivial_line (line as Inference_Step (name, _, t, _, [])) lines =
    if is_only_type_information t then delete_dependency name lines
    else line :: lines
  | add_nontrivial_line line lines = line :: lines
and delete_dependency name lines =
  fold_rev add_nontrivial_line
           (map (replace_dependencies_in_line (name, [])) lines) []

(* ATPs sometimes reuse free variable names in the strangest ways. Removing
   offending lines often does the trick. *)
fun is_bad_free frees (Free x) = not (member (op =) frees x)
  | is_bad_free _ _ = false

fun add_desired_line _ _ (line as Definition_Step (name, _, _)) (j, lines) =
    (j, line :: map (replace_dependencies_in_line (name, [])) lines)
  | add_desired_line fact_names frees
        (Inference_Step (name as (_, ss), role, t, rule, deps)) (j, lines) =
    (j + 1,
     if is_fact fact_names ss orelse
        is_conjecture ss orelse
        (* the last line must be kept *)
        j = 0 orelse
        (not (is_only_type_information t) andalso
         null (Term.add_tvars t []) andalso
         not (exists_subterm (is_bad_free frees) t) andalso
         length deps >= 2 andalso
         (* kill next to last line, which usually results in a trivial step *)
         j <> 1) then
       Inference_Step (name, role, t, rule, deps) :: lines  (* keep line *)
     else
       map (replace_dependencies_in_line (name, deps)) lines)  (* drop line *)

val indent_size = 2
val no_label = ("", ~1)

fun string_for_proof ctxt type_enc lam_trans i n =
  let
    fun do_indent ind = replicate_string (ind * indent_size) " "
    fun do_free (s, T) =
      maybe_quote s ^ " :: " ^
      maybe_quote (simplify_spaces (with_vanilla_print_mode
        (Syntax.string_of_typ ctxt) T))
    fun do_label l = if l = no_label then "" else string_for_label l ^ ": "
    fun do_have qs =
      (if member (op =) qs Moreover then "moreover " else "") ^
      (if member (op =) qs Ultimately then "ultimately " else "") ^
      (if member (op =) qs Then then
         if member (op =) qs Show then "thus" else "hence"
       else
         if member (op =) qs Show then "show" else "have")
    val do_term =
      annotate_types ctxt
      #> with_vanilla_print_mode (Syntax.string_of_term ctxt)
      #> simplify_spaces
      #> maybe_quote
    val reconstr = Metis (type_enc, lam_trans)
    fun do_facts ind (ls, ss) =
      "\n" ^ do_indent (ind + 1) ^
      reconstructor_command reconstr 1 1 [] 0
          (ls |> sort_distinct (prod_ord string_ord int_ord),
           ss |> sort_distinct string_ord)
    and do_step ind (Fix xs) =
        do_indent ind ^ "fix " ^ space_implode " and " (map do_free xs) ^ "\n"
      | do_step ind (Let (t1, t2)) =
        do_indent ind ^ "let " ^ do_term t1 ^ " = " ^ do_term t2 ^ "\n"
      | do_step ind (Assume (l, t)) =
        do_indent ind ^ "assume " ^ do_label l ^ do_term t ^ "\n"
      | do_step ind (Prove (qs, l, t, By_Metis facts)) =
        do_indent ind ^ do_have qs ^ " " ^
        do_label l ^ do_term t ^ do_facts ind facts ^ "\n"
      | do_step ind (Prove (qs, l, t, Case_Split (proofs, facts))) =
        implode (map (prefix (do_indent ind ^ "moreover\n") o do_block ind)
                     proofs) ^
        do_indent ind ^ do_have qs ^ " " ^ do_label l ^ do_term t ^
        do_facts ind facts ^ "\n"
    and do_steps prefix suffix ind steps =
      let val s = implode (map (do_step ind) steps) in
        replicate_string (ind * indent_size - size prefix) " " ^ prefix ^
        String.extract (s, ind * indent_size,
                        SOME (size s - ind * indent_size - 1)) ^
        suffix ^ "\n"
      end
    and do_block ind proof = do_steps "{ " " }" (ind + 1) proof
    (* One-step proofs are pointless; better use the Metis one-liner
       directly. *)
    and do_proof [Prove (_, _, _, By_Metis _)] = ""
      | do_proof proof =
        (if i <> 1 then "prefer " ^ string_of_int i ^ "\n" else "") ^
        do_indent 0 ^ "proof -\n" ^ do_steps "" "" 1 proof ^ do_indent 0 ^
        (if n <> 1 then "next" else "qed")
  in do_proof end

fun used_labels_of_step (Prove (_, _, _, by)) =
    (case by of
       By_Metis (ls, _) => ls
     | Case_Split (proofs, (ls, _)) =>
       fold (union (op =) o used_labels_of) proofs ls)
  | used_labels_of_step _ = []
and used_labels_of proof = fold (union (op =) o used_labels_of_step) proof []

fun kill_useless_labels_in_proof proof =
  let
    val used_ls = used_labels_of proof
    fun do_label l = if member (op =) used_ls l then l else no_label
    fun do_step (Assume (l, t)) = Assume (do_label l, t)
      | do_step (Prove (qs, l, t, by)) =
        Prove (qs, do_label l, t,
               case by of
                 Case_Split (proofs, facts) =>
                 Case_Split (map (map do_step) proofs, facts)
               | _ => by)
      | do_step step = step
  in map do_step proof end

fun prefix_for_depth n = replicate_string (n + 1)

val relabel_proof =
  let
    fun aux _ _ _ [] = []
      | aux subst depth (next_assum, next_have) (Assume (l, t) :: proof) =
        if l = no_label then
          Assume (l, t) :: aux subst depth (next_assum, next_have) proof
        else
          let val l' = (prefix_for_depth depth assume_prefix, next_assum) in
            Assume (l', t) ::
            aux ((l, l') :: subst) depth (next_assum + 1, next_have) proof
          end
      | aux subst depth (next_assum, next_have)
            (Prove (qs, l, t, by) :: proof) =
        let
          val (l', subst, next_have) =
            if l = no_label then
              (l, subst, next_have)
            else
              let val l' = (prefix_for_depth depth have_prefix, next_have) in
                (l', (l, l') :: subst, next_have + 1)
              end
          val relabel_facts =
            apfst (maps (the_list o AList.lookup (op =) subst))
          val by =
            case by of
              By_Metis facts => By_Metis (relabel_facts facts)
            | Case_Split (proofs, facts) =>
              Case_Split (map (aux subst (depth + 1) (1, 1)) proofs,
                          relabel_facts facts)
        in
          Prove (qs, l', t, by) :: aux subst depth (next_assum, next_have) proof
        end
      | aux subst depth nextp (step :: proof) =
        step :: aux subst depth nextp proof
  in aux [] 0 (1, 1) end

val merge_timeout_slack = 1.2

val label_ord = prod_ord int_ord fast_string_ord o pairself swap

structure Label_Table = Table(
  type key = label
  val ord = label_ord)

fun shrink_proof debug ctxt type_enc lam_trans preplay preplay_timeout
                 isar_shrink proof =
  let
    (* clean vector interface *)
    fun get i v = Vector.sub (v, i)
    fun replace x i v = Vector.update (v, i, x)
    fun update f i v = replace (get i v |> f) i v
    fun v_fold_index f v s =
      Vector.foldl (fn (x, (i, s)) => (i+1, f (i, x) s)) (0, s) v |> snd

    (* Queue interface to table *)
    fun pop tab key =
      let val v = hd (Inttab.lookup_list tab key) in
        (v, Inttab.remove_list (op =) (key, v) tab)
      end
    fun pop_max tab = pop tab (the (Inttab.max_key tab))
    val is_empty = Inttab.is_empty
    fun add_list tab xs = fold (Inttab.insert_list (op =)) xs tab

    (* proof vector *)
    val proof_vect = proof |> map SOME |> Vector.fromList
    val n = Vector.length proof_vect
    val n_target = Real.fromInt n / isar_shrink |> Real.round

    (* table for mapping from label to proof position *)
    fun update_table (i, Assume (label, _)) = 
        Label_Table.update_new (label, i)
      | update_table (i, Prove (_, label, _, _)) =
        Label_Table.update_new (label, i)
      | update_table _ = I
    val label_index_table = fold_index update_table proof Label_Table.empty

    (* proof references *)
    fun refs (Prove (_, _, _, By_Metis (refs, _))) =
        map (the o Label_Table.lookup label_index_table) refs
      | refs _ = []
    val refed_by_vect =
      Vector.tabulate (n, (fn _ => []))
      |> fold_index (fn (i, step) => fold (update (cons i)) (refs step)) proof
      |> Vector.map rev (* after rev, indices are sorted in ascending order *)

    (* candidates for elimination, use table as priority queue (greedy
       algorithm) *)
    fun add_if_cand proof_vect (i, [j]) =
        (case (the (get i proof_vect), the (get j proof_vect)) of
          (Prove (_, _, t, By_Metis _), Prove (_, _, _, By_Metis _)) =>
            cons (Term.size_of_term t, i)
        | _ => I)
      | add_if_cand _ _ = I
    val cand_tab = 
      v_fold_index (add_if_cand proof_vect) refed_by_vect []
      |> Inttab.make_list

    (* Enrich context with local facts *)
    val thy = Proof_Context.theory_of ctxt
    fun enrich_ctxt (Assume (label, t)) ctxt = 
        Proof_Context.put_thms false
            (string_for_label label, SOME [Skip_Proof.make_thm thy t]) ctxt
      | enrich_ctxt (Prove (_, label, t, _)) ctxt =
        Proof_Context.put_thms false
            (string_for_label label, SOME [Skip_Proof.make_thm thy t]) ctxt
      | enrich_ctxt _ ctxt = ctxt
    val rich_ctxt = fold enrich_ctxt proof ctxt

    (* Timing *)
    fun take_time timeout tac arg =
      let val timing = Timing.start () in
        (TimeLimit.timeLimit timeout tac arg;
         Timing.result timing |> #cpu |> SOME)
        handle _ => NONE
      end
    val sum_up_time =
      Vector.foldl
        ((fn (SOME t, (b, s)) => (b, Time.+ (t, s))
           | (NONE, (_, s)) => (true, Time.+ (preplay_timeout, s))) o apfst Lazy.force)
        (false, seconds 0.0)

    (* Metis preplaying *)
    fun try_metis timeout (Prove (_, _, t, By_Metis fact_names)) =
      if not preplay then (fn () => SOME (seconds 0.0)) else
        let
          val facts =
            fact_names
            |>> map string_for_label 
            |> op @
            |> map (the_single o thms_of_name rich_ctxt)
          val goal =
            Goal.prove (Config.put Metis_Tactic.verbose debug ctxt) [] [] t
          fun tac {context = ctxt, prems = _} =
            Metis_Tactic.metis_tac [type_enc] lam_trans ctxt facts 1
        in
          take_time timeout (fn () => goal tac)
        end
      (*| try_metis timeout (Prove (_, _, t, Case_Split _)) = *) (*FIXME: Yet to be implemented *)
      | try_metis _ _ = (fn () => SOME (seconds 0.0) )

    (* Lazy metis time vector, cache *)
    val metis_time =
      Vector.map (Lazy.lazy o try_metis preplay_timeout o the) proof_vect

    (* Merging *)
    fun merge (Prove (qs1, label1, _, By_Metis (lfs1, gfs1)))
              (Prove (qs2, label2 , t, By_Metis (lfs2, gfs2))) =
      let
        val qs = inter (op =) qs1 qs2 (* FIXME: Is this correct? *)
          |> member (op =) (union (op =) qs1 qs2) Ultimately ? cons Ultimately
          |> member (op =) qs2 Show ? cons Show
        val ls = remove (op =) label1 lfs2 |> union (op =) lfs1
        val ss = union (op =) gfs1 gfs2
      in Prove (qs, label2, t, By_Metis (ls, ss)) end
    fun try_merge metis_time (s1, i) (s2, j) =
      (case get i metis_time |> Lazy.force of
        NONE => (NONE, metis_time)
      | SOME t1 =>
        (case get j metis_time |> Lazy.force of
          NONE => (NONE, metis_time)
        | SOME t2 =>
          let
            val s12 = merge s1 s2
            val timeout =
              Time.+ (t1, t2) |> Time.toReal |> curry Real.* merge_timeout_slack
                      |> Time.fromReal
          in
            case try_metis timeout s12 () of
              NONE => (NONE, metis_time)
            | some_t12 =>
              (SOME s12, metis_time
                         |> replace (seconds 0.0 |> SOME |> Lazy.value) i
                         |> replace (Lazy.value some_t12) j)

          end))

    fun merge_steps metis_time proof_vect refed_by cand_tab n' =
      if is_empty cand_tab orelse n' <= n_target orelse n'<3 then
        (Vector.foldr
           (fn (NONE, proof) => proof | (SOME s, proof) => s :: proof)
           [] proof_vect,
         sum_up_time metis_time)
      else
        let
          val (i, cand_tab) = pop_max cand_tab
          val j = get i refed_by |> the_single
          val s1 = get i proof_vect |> the
          val s2 = get j proof_vect |> the
        in
          case try_merge metis_time (s1, i) (s2, j) of
            (NONE, metis_time) =>
            merge_steps metis_time proof_vect refed_by cand_tab n'
          | (s, metis_time) => let
            val refs = refs s1
            val refed_by = refed_by |> fold
              (update (Ord_List.remove int_ord i #> Ord_List.insert int_ord j)) refs
            val new_candidates =
              fold (add_if_cand proof_vect)
                (map (fn i => (i, get i refed_by)) refs) []
            val cand_tab = add_list cand_tab new_candidates
            val proof_vect = proof_vect |> replace NONE i |> replace s j
          in
            merge_steps metis_time proof_vect refed_by cand_tab (n' - 1)
          end
        end
  in
    merge_steps metis_time proof_vect refed_by_vect cand_tab n
  end

val chain_direct_proof =
  let
    fun succedent_of_step (Prove (_, label, _, _)) = SOME label
      | succedent_of_step (Assume (label, _)) = SOME label
      | succedent_of_step _ = NONE
    fun chain_inf (SOME label0)
                  (step as Prove (qs, label, t, By_Metis (lfs, gfs))) =
        if member (op =) lfs label0 then
          Prove (Then :: qs, label, t,
                 By_Metis (filter_out (curry (op =) label0) lfs, gfs))
        else
          step
      | chain_inf _ (Prove (qs, label, t, Case_Split (proofs, facts))) =
        Prove (qs, label, t, Case_Split ((map (chain_proof NONE) proofs), facts))
      | chain_inf _ step = step
    and chain_proof _ [] = []
      | chain_proof (prev as SOME _) (i :: is) =
        chain_inf prev i :: chain_proof (succedent_of_step i) is
      | chain_proof _ (i :: is) =
        i :: chain_proof (succedent_of_step i) is
  in chain_proof NONE end

type isar_params =
  bool * bool * Time.time * real * string Symtab.table
  * (string * stature) list vector * int Symtab.table * string proof * thm

fun isar_proof_text ctxt isar_proofs
    (debug, verbose, preplay_timeout, isar_shrink, pool, fact_names, sym_tab,
     atp_proof, goal)
    (one_line_params as (_, _, _, _, subgoal, subgoal_count)) =
  let
    val (params, hyp_ts, concl_t) = strip_subgoal ctxt goal subgoal
    val frees = fold Term.add_frees (concl_t :: hyp_ts) []
    val one_line_proof = one_line_proof_text 0 one_line_params
    val type_enc =
      if is_typed_helper_used_in_atp_proof atp_proof then full_typesN
      else partial_typesN
    val lam_trans = lam_trans_from_atp_proof atp_proof metis_default_lam_trans
    val preplay = preplay_timeout <> seconds 0.0

    fun isar_proof_of () =
      let
        val atp_proof =
          atp_proof
          |> clean_up_atp_proof_dependencies
          |> nasty_atp_proof pool
          |> map_term_names_in_atp_proof repair_name
          |> decode_lines ctxt sym_tab
          |> rpair [] |-> fold_rev (add_line fact_names)
          |> repair_waldmeister_endgame
          |> rpair [] |-> fold_rev add_nontrivial_line
          |> rpair (0, [])
          |-> fold_rev (add_desired_line fact_names frees)
          |> snd
        val conj_name = conjecture_prefix ^ string_of_int (length hyp_ts)
        val conjs =
          atp_proof |> map_filter
            (fn Inference_Step (name as (_, ss), _, _, _, []) =>
                if member (op =) ss conj_name then SOME name else NONE
              | _ => NONE)
        val assms =
          atp_proof |> map_filter
            (fn Inference_Step (name as (_, ss), _, _, _, []) =>
                (case resolve_conjecture ss of
                   [j] =>
                   if j = length hyp_ts then NONE
                   else SOME (Assume (raw_label_for_name name, nth hyp_ts j))
                 | _ => NONE)
              | _ => NONE)
        fun dep_of_step (Definition_Step _) = NONE
          | dep_of_step (Inference_Step (name, _, _, _, from)) =
            SOME (from, name)
        val ref_graph = atp_proof |> map_filter dep_of_step |> make_ref_graph
        val axioms = axioms_of_ref_graph ref_graph conjs
        val tainted = tainted_atoms_of_ref_graph ref_graph conjs
        val props =
          Symtab.empty
          |> fold (fn Definition_Step _ => I (* FIXME *)
                    | Inference_Step (name as (s, ss), role, t, _, _) =>
                      Symtab.update_new (s,
                        if member (op = o apsnd fst) tainted s then
                          t |> role <> Conjecture ? s_not
                            |> fold exists_of (map Var (Term.add_vars t []))
                        else
                          t))
                  atp_proof
        (* The assumptions and conjecture are props; the rest are bools. *)
        fun prop_of_clause [name as (s, ss)] =
            (case resolve_conjecture ss of
               [j] => if j = length hyp_ts then concl_t else nth hyp_ts j
             | _ => the_default @{term False} (Symtab.lookup props s)
                    |> HOLogic.mk_Trueprop |> close_form)
          | prop_of_clause names =
            let val lits = map_filter (Symtab.lookup props o fst) names in
              case List.partition (can HOLogic.dest_not) lits of
                (negs as _ :: _, pos as _ :: _) =>
                HOLogic.mk_imp
                  (Library.foldr1 s_conj (map HOLogic.dest_not negs),
                   Library.foldr1 s_disj pos)
              | _ => fold (curry s_disj) lits @{term False}
            end
            |> HOLogic.mk_Trueprop |> close_form
        fun maybe_show outer c =
          (outer andalso length c = 1 andalso subset (op =) (c, conjs))
          ? cons Show
        fun do_have outer qs (gamma, c) =
          Prove (maybe_show outer c qs, label_of_clause c, prop_of_clause c,
                 By_Metis (fold (add_fact_from_dependencies fact_names) gamma
                                ([], [])))
        fun do_inf outer (Have z) = do_have outer [] z
          | do_inf outer (Cases cases) =
            let val c = succedent_of_cases cases in
              Prove (maybe_show outer c [Ultimately], label_of_clause c,
                     prop_of_clause c,
                     Case_Split (map (do_case false) cases, ([], [])))
            end
        and do_case outer (c, infs) =
          Assume (label_of_clause c, prop_of_clause c) ::
          map (do_inf outer) infs
        val (isar_proof, ext_time) =
          ref_graph
          |> redirect_graph axioms tainted
          |> map (do_inf true)
          |> append assms
          |> relabel_proof (* FIXME: Is there a better way? *)
          |> (if not preplay andalso isar_shrink <= 1.0
              then pair (true, seconds 0.0) #> swap
              else shrink_proof debug ctxt type_enc lam_trans preplay
                preplay_timeout (if isar_proofs then isar_shrink else 1000.0))
       (* |>> reorder_proof_to_minimize_jumps (* ? *) *)
          |>> chain_direct_proof
          |>> kill_useless_labels_in_proof
          |>> relabel_proof
          |>> not (null params) ? cons (Fix params)
        val num_steps = length isar_proof
        val isar_text =
          string_for_proof ctxt type_enc lam_trans subgoal subgoal_count
                           isar_proof
      in
        case isar_text of
          "" =>
          if isar_proofs then
            "\nNo structured proof available (proof too short)."
          else
            ""
        | _ =>
          "\n\nStructured proof" ^
          (if verbose then
             " (" ^ string_of_int num_steps ^ " step" ^ plural_s num_steps ^
             (if preplay then ", " ^ string_from_ext_time ext_time
              else "") ^ ")"
           else if preplay then
             " (" ^ string_from_ext_time ext_time ^ ")"
           else
             "") ^ ":\n" ^ Sendback.markup isar_text
      end
    val isar_proof =
      if debug then
        isar_proof_of ()
      else case try isar_proof_of () of
        SOME s => s
      | NONE => if isar_proofs then
                  "\nWarning: The Isar proof construction failed."
                else
                  ""
  in one_line_proof ^ isar_proof end

fun proof_text ctxt isar_proofs isar_params num_chained
               (one_line_params as (preplay, _, _, _, _, _)) =
  (if case preplay of Failed_to_Play _ => true | _ => isar_proofs then
     isar_proof_text ctxt isar_proofs isar_params
   else
     one_line_proof_text num_chained) one_line_params

end;