src/Pure/Thy/export.scala
changeset 75393 87ebf5a50283
parent 74686 42f358ea8dee
child 75394 42267c650205
equal deleted inserted replaced
75388:b3ca4a6ed74b 75393:87ebf5a50283
     9 
     9 
    10 import scala.annotation.tailrec
    10 import scala.annotation.tailrec
    11 import scala.util.matching.Regex
    11 import scala.util.matching.Regex
    12 
    12 
    13 
    13 
    14 object Export
    14 object Export {
    15 {
       
    16   /* artefact names */
    15   /* artefact names */
    17 
    16 
    18   val DOCUMENT_ID = "PIDE/document_id"
    17   val DOCUMENT_ID = "PIDE/document_id"
    19   val FILES = "PIDE/files"
    18   val FILES = "PIDE/files"
    20   val MARKUP = "PIDE/markup"
    19   val MARKUP = "PIDE/markup"
    29   def implode_name(elems: Iterable[String]): String = elems.mkString("/")
    28   def implode_name(elems: Iterable[String]): String = elems.mkString("/")
    30 
    29 
    31 
    30 
    32   /* SQL data model */
    31   /* SQL data model */
    33 
    32 
    34   object Data
    33   object Data {
    35   {
       
    36     val session_name = SQL.Column.string("session_name").make_primary_key
    34     val session_name = SQL.Column.string("session_name").make_primary_key
    37     val theory_name = SQL.Column.string("theory_name").make_primary_key
    35     val theory_name = SQL.Column.string("theory_name").make_primary_key
    38     val name = SQL.Column.string("name").make_primary_key
    36     val name = SQL.Column.string("name").make_primary_key
    39     val executable = SQL.Column.bool("executable")
    37     val executable = SQL.Column.bool("executable")
    40     val compressed = SQL.Column.bool("compressed")
    38     val compressed = SQL.Column.bool("compressed")
    48       "WHERE " + Data.session_name.equal(session_name) +
    46       "WHERE " + Data.session_name.equal(session_name) +
    49         (if (theory_name == "") "" else " AND " + Data.theory_name.equal(theory_name)) +
    47         (if (theory_name == "") "" else " AND " + Data.theory_name.equal(theory_name)) +
    50         (if (name == "") "" else " AND " + Data.name.equal(name))
    48         (if (name == "") "" else " AND " + Data.name.equal(name))
    51   }
    49   }
    52 
    50 
    53   def read_name(db: SQL.Database, session_name: String, theory_name: String, name: String): Boolean =
    51   def read_name(
    54   {
    52     db: SQL.Database,
       
    53     session_name: String,
       
    54     theory_name: String,
       
    55     name: String
       
    56   ): Boolean = {
    55     val select =
    57     val select =
    56       Data.table.select(List(Data.name), Data.where_equal(session_name, theory_name, name))
    58       Data.table.select(List(Data.name), Data.where_equal(session_name, theory_name, name))
    57     db.using_statement(select)(stmt => stmt.execute_query().next())
    59     db.using_statement(select)(stmt => stmt.execute_query().next())
    58   }
    60   }
    59 
    61 
    60   def read_names(db: SQL.Database, session_name: String, theory_name: String): List[String] =
    62   def read_names(db: SQL.Database, session_name: String, theory_name: String): List[String] = {
    61   {
       
    62     val select = Data.table.select(List(Data.name), Data.where_equal(session_name, theory_name))
    63     val select = Data.table.select(List(Data.name), Data.where_equal(session_name, theory_name))
    63     db.using_statement(select)(stmt =>
    64     db.using_statement(select)(stmt =>
    64       stmt.execute_query().iterator(res => res.string(Data.name)).toList)
    65       stmt.execute_query().iterator(res => res.string(Data.name)).toList)
    65   }
    66   }
    66 
    67 
    67   def read_theory_names(db: SQL.Database, session_name: String): List[String] =
    68   def read_theory_names(db: SQL.Database, session_name: String): List[String] = {
    68   {
       
    69     val select =
    69     val select =
    70       Data.table.select(List(Data.theory_name), Data.where_equal(session_name), distinct = true)
    70       Data.table.select(List(Data.theory_name), Data.where_equal(session_name), distinct = true)
    71     db.using_statement(select)(stmt =>
    71     db.using_statement(select)(stmt =>
    72       stmt.execute_query().iterator(_.string(Data.theory_name)).toList)
    72       stmt.execute_query().iterator(_.string(Data.theory_name)).toList)
    73   }
    73   }
    74 
    74 
    75   def read_theory_exports(db: SQL.Database, session_name: String): List[(String, String)] =
    75   def read_theory_exports(db: SQL.Database, session_name: String): List[(String, String)] = {
    76   {
       
    77     val select = Data.table.select(List(Data.theory_name, Data.name), Data.where_equal(session_name))
    76     val select = Data.table.select(List(Data.theory_name, Data.name), Data.where_equal(session_name))
    78     db.using_statement(select)(stmt =>
    77     db.using_statement(select)(stmt =>
    79       stmt.execute_query().iterator(res =>
    78       stmt.execute_query().iterator(res =>
    80         (res.string(Data.theory_name), res.string(Data.name))).toList)
    79         (res.string(Data.theory_name), res.string(Data.name))).toList)
    81   }
    80   }
    93     session_name: String,
    92     session_name: String,
    94     theory_name: String,
    93     theory_name: String,
    95     name: String,
    94     name: String,
    96     executable: Boolean,
    95     executable: Boolean,
    97     body: Future[(Boolean, Bytes)],
    96     body: Future[(Boolean, Bytes)],
    98     cache: XML.Cache)
    97     cache: XML.Cache
    99   {
    98   ) {
   100     override def toString: String = name
    99     override def toString: String = name
   101 
   100 
   102     def compound_name: String = Export.compound_name(theory_name, name)
   101     def compound_name: String = Export.compound_name(theory_name, name)
   103 
   102 
   104     def name_has_prefix(s: String): Boolean = name.startsWith(s)
   103     def name_has_prefix(s: String): Boolean = name.startsWith(s)
   107     def name_extends(elems: List[String]): Boolean =
   106     def name_extends(elems: List[String]): Boolean =
   108       name_elems.startsWith(elems) && name_elems != elems
   107       name_elems.startsWith(elems) && name_elems != elems
   109 
   108 
   110     def text: String = uncompressed.text
   109     def text: String = uncompressed.text
   111 
   110 
   112     def uncompressed: Bytes =
   111     def uncompressed: Bytes = {
   113     {
       
   114       val (compressed, bytes) = body.join
   112       val (compressed, bytes) = body.join
   115       if (compressed) bytes.uncompress(cache = cache.xz) else bytes
   113       if (compressed) bytes.uncompress(cache = cache.xz) else bytes
   116     }
   114     }
   117 
   115 
   118     def uncompressed_yxml: XML.Body =
   116     def uncompressed_yxml: XML.Body =
   119       YXML.parse_body(UTF8.decode_permissive(uncompressed), cache = cache)
   117       YXML.parse_body(UTF8.decode_permissive(uncompressed), cache = cache)
   120 
   118 
   121     def write(db: SQL.Database): Unit =
   119     def write(db: SQL.Database): Unit = {
   122     {
       
   123       val (compressed, bytes) = body.join
   120       val (compressed, bytes) = body.join
   124       db.using_statement(Data.table.insert())(stmt =>
   121       db.using_statement(Data.table.insert())(stmt => {
   125       {
       
   126         stmt.string(1) = session_name
   122         stmt.string(1) = session_name
   127         stmt.string(2) = theory_name
   123         stmt.string(2) = theory_name
   128         stmt.string(3) = name
   124         stmt.string(3) = name
   129         stmt.bool(4) = executable
   125         stmt.bool(4) = executable
   130         stmt.bool(5) = compressed
   126         stmt.bool(5) = compressed
   132         stmt.execute()
   128         stmt.execute()
   133       })
   129       })
   134     }
   130     }
   135   }
   131   }
   136 
   132 
   137   def make_regex(pattern: String): Regex =
   133   def make_regex(pattern: String): Regex = {
   138   {
       
   139     @tailrec def make(result: List[String], depth: Int, chs: List[Char]): Regex =
   134     @tailrec def make(result: List[String], depth: Int, chs: List[Char]): Regex =
   140       chs match {
   135       chs match {
   141         case '*' :: '*' :: rest => make("[^:]*" :: result, depth, rest)
   136         case '*' :: '*' :: rest => make("[^:]*" :: result, depth, rest)
   142         case '*' :: rest => make("[^:/]*" :: result, depth, rest)
   137         case '*' :: rest => make("[^:/]*" :: result, depth, rest)
   143         case '?' :: rest => make("[^:/]" :: result, depth, rest)
   138         case '?' :: rest => make("[^:/]" :: result, depth, rest)
   150         case Nil => result.reverse.mkString.r
   145         case Nil => result.reverse.mkString.r
   151       }
   146       }
   152     make(Nil, 0, pattern.toList)
   147     make(Nil, 0, pattern.toList)
   153   }
   148   }
   154 
   149 
   155   def make_matcher(pattern: String): (String, String) => Boolean =
   150   def make_matcher(pattern: String): (String, String) => Boolean = {
   156   {
       
   157     val regex = make_regex(pattern)
   151     val regex = make_regex(pattern)
   158     (theory_name: String, name: String) =>
   152     (theory_name: String, name: String) =>
   159       regex.pattern.matcher(compound_name(theory_name, name)).matches
   153       regex.pattern.matcher(compound_name(theory_name, name)).matches
   160   }
   154   }
   161 
   155 
   162   def make_entry(
   156   def make_entry(
   163     session_name: String, args: Protocol.Export.Args, bytes: Bytes, cache: XML.Cache): Entry =
   157     session_name: String,
   164   {
   158     args: Protocol.Export.Args,
       
   159     bytes: Bytes,
       
   160     cache: XML.Cache
       
   161   ): Entry = {
   165     val body =
   162     val body =
   166       if (args.compress) Future.fork(bytes.maybe_compress(cache = cache.xz))
   163       if (args.compress) Future.fork(bytes.maybe_compress(cache = cache.xz))
   167       else Future.value((false, bytes))
   164       else Future.value((false, bytes))
   168     Entry(session_name, args.theory_name, args.name, args.executable, body, cache)
   165     Entry(session_name, args.theory_name, args.name, args.executable, body, cache)
   169   }
   166   }
   170 
   167 
   171   def read_entry(db: SQL.Database, cache: XML.Cache,
   168   def read_entry(
   172     session_name: String, theory_name: String, name: String): Option[Entry] =
   169     db: SQL.Database,
   173   {
   170     cache: XML.Cache,
       
   171     session_name: String,
       
   172     theory_name: String,
       
   173     name: String
       
   174   ): Option[Entry] = {
   174     val select =
   175     val select =
   175       Data.table.select(List(Data.executable, Data.compressed, Data.body),
   176       Data.table.select(List(Data.executable, Data.compressed, Data.body),
   176         Data.where_equal(session_name, theory_name, name))
   177         Data.where_equal(session_name, theory_name, name))
   177     db.using_statement(select)(stmt =>
   178     db.using_statement(select)(stmt => {
   178     {
       
   179       val res = stmt.execute_query()
   179       val res = stmt.execute_query()
   180       if (res.next()) {
   180       if (res.next()) {
   181         val executable = res.bool(Data.executable)
   181         val executable = res.bool(Data.executable)
   182         val compressed = res.bool(Data.compressed)
   182         val compressed = res.bool(Data.compressed)
   183         val bytes = res.bytes(Data.body)
   183         val bytes = res.bytes(Data.body)
   186       }
   186       }
   187       else None
   187       else None
   188     })
   188     })
   189   }
   189   }
   190 
   190 
   191   def read_entry(dir: Path, cache: XML.Cache,
   191   def read_entry(
   192     session_name: String, theory_name: String, name: String): Option[Entry] =
   192     dir: Path,
   193   {
   193     cache: XML.Cache,
       
   194     session_name: String,
       
   195     theory_name: String,
       
   196     name: String
       
   197   ): Option[Entry] = {
   194     val path = dir + Path.basic(theory_name) + Path.explode(name)
   198     val path = dir + Path.basic(theory_name) + Path.explode(name)
   195     if (path.is_file) {
   199     if (path.is_file) {
   196       val executable = File.is_executable(path)
   200       val executable = File.is_executable(path)
   197       val uncompressed = Bytes.read(path)
   201       val uncompressed = Bytes.read(path)
   198       val body = Future.value((false, uncompressed))
   202       val body = Future.value((false, uncompressed))
   205   /* database consumer thread */
   209   /* database consumer thread */
   206 
   210 
   207   def consumer(db: SQL.Database, cache: XML.Cache, progress: Progress = new Progress): Consumer =
   211   def consumer(db: SQL.Database, cache: XML.Cache, progress: Progress = new Progress): Consumer =
   208     new Consumer(db, cache, progress)
   212     new Consumer(db, cache, progress)
   209 
   213 
   210   class Consumer private[Export](db: SQL.Database, cache: XML.Cache, progress: Progress)
   214   class Consumer private[Export](db: SQL.Database, cache: XML.Cache, progress: Progress) {
   211   {
       
   212     private val errors = Synchronized[List[String]](Nil)
   215     private val errors = Synchronized[List[String]](Nil)
   213 
   216 
   214     private val consumer =
   217     private val consumer =
   215       Consumer_Thread.fork_bulk[(Entry, Boolean)](name = "export")(
   218       Consumer_Thread.fork_bulk[(Entry, Boolean)](name = "export")(
   216         bulk = { case (entry, _) => entry.body.is_finished },
   219         bulk = { case (entry, _) => entry.body.is_finished },
   217         consume =
   220         consume =
   218           (args: List[(Entry, Boolean)]) =>
   221           (args: List[(Entry, Boolean)]) => {
   219           {
       
   220             val results =
   222             val results =
   221               db.transaction {
   223               db.transaction {
   222                 for ((entry, strict) <- args)
   224                 for ((entry, strict) <- args)
   223                 yield {
   225                 yield {
   224                   if (progress.stopped) {
   226                   if (progress.stopped) {
   236                 }
   238                 }
   237               }
   239               }
   238             (results, true)
   240             (results, true)
   239           })
   241           })
   240 
   242 
   241     def apply(session_name: String, args: Protocol.Export.Args, body: Bytes): Unit =
   243     def apply(session_name: String, args: Protocol.Export.Args, body: Bytes): Unit = {
   242     {
       
   243       if (!progress.stopped) {
   244       if (!progress.stopped) {
   244         consumer.send(make_entry(session_name, args, body, cache) -> args.strict)
   245         consumer.send(make_entry(session_name, args, body, cache) -> args.strict)
   245       }
   246       }
   246     }
   247     }
   247 
   248 
   248     def shutdown(close: Boolean = false): List[String] =
   249     def shutdown(close: Boolean = false): List[String] = {
   249     {
       
   250       consumer.shutdown()
   250       consumer.shutdown()
   251       if (close) db.close()
   251       if (close) db.close()
   252       errors.value.reverse ::: (if (progress.stopped) List("Export stopped") else Nil)
   252       errors.value.reverse ::: (if (progress.stopped) List("Export stopped") else Nil)
   253     }
   253     }
   254   }
   254   }
   255 
   255 
   256 
   256 
   257   /* abstract provider */
   257   /* abstract provider */
   258 
   258 
   259   object Provider
   259   object Provider {
   260   {
       
   261     def none: Provider =
   260     def none: Provider =
   262       new Provider {
   261       new Provider {
   263         def apply(export_name: String): Option[Entry] = None
   262         def apply(export_name: String): Option[Entry] = None
   264         def focus(other_theory: String): Provider = this
   263         def focus(other_theory: String): Provider = this
   265 
   264 
   277         def focus(other_theory: String): Provider = this
   276         def focus(other_theory: String): Provider = this
   278 
   277 
   279         override def toString: String = context.toString
   278         override def toString: String = context.toString
   280       }
   279       }
   281 
   280 
   282     def database(db: SQL.Database, cache: XML.Cache, session_name: String, theory_name: String)
   281     def database(
   283       : Provider =
   282       db: SQL.Database,
   284     {
   283       cache: XML.Cache,
       
   284       session_name: String,
       
   285       theory_name: String
       
   286     ) : Provider = {
   285       new Provider {
   287       new Provider {
   286         def apply(export_name: String): Option[Entry] =
   288         def apply(export_name: String): Option[Entry] =
   287           read_entry(db, cache, session_name, theory_name, export_name)
   289           read_entry(db, cache, session_name, theory_name, export_name)
   288 
   290 
   289         def focus(other_theory: String): Provider =
   291         def focus(other_theory: String): Provider =
   309           }
   311           }
   310 
   312 
   311         override def toString: String = snapshot.toString
   313         override def toString: String = snapshot.toString
   312       }
   314       }
   313 
   315 
   314     def directory(dir: Path, cache: XML.Cache, session_name: String, theory_name: String)
   316     def directory(
   315       : Provider =
   317       dir: Path,
   316     {
   318       cache: XML.Cache,
       
   319       session_name: String,
       
   320       theory_name: String
       
   321     ) : Provider = {
   317       new Provider {
   322       new Provider {
   318         def apply(export_name: String): Option[Entry] =
   323         def apply(export_name: String): Option[Entry] =
   319           read_entry(dir, cache, session_name, theory_name, export_name)
   324           read_entry(dir, cache, session_name, theory_name, export_name)
   320 
   325 
   321         def focus(other_theory: String): Provider =
   326         def focus(other_theory: String): Provider =
   325         override def toString: String = dir.toString
   330         override def toString: String = dir.toString
   326       }
   331       }
   327     }
   332     }
   328   }
   333   }
   329 
   334 
   330   trait Provider
   335   trait Provider {
   331   {
       
   332     def apply(export_name: String): Option[Entry]
   336     def apply(export_name: String): Option[Entry]
   333 
   337 
   334     def uncompressed_yxml(export_name: String): XML.Body =
   338     def uncompressed_yxml(export_name: String): XML.Body =
   335       apply(export_name) match {
   339       apply(export_name) match {
   336         case Some(entry) => entry.uncompressed_yxml
   340         case Some(entry) => entry.uncompressed_yxml
   348     session_name: String,
   352     session_name: String,
   349     export_dir: Path,
   353     export_dir: Path,
   350     progress: Progress = new Progress,
   354     progress: Progress = new Progress,
   351     export_prune: Int = 0,
   355     export_prune: Int = 0,
   352     export_list: Boolean = false,
   356     export_list: Boolean = false,
   353     export_patterns: List[String] = Nil): Unit =
   357     export_patterns: List[String] = Nil
   354   {
   358   ): Unit = {
   355     using(store.open_database(session_name))(db =>
   359     using(store.open_database(session_name))(db => {
   356     {
       
   357       db.transaction {
   360       db.transaction {
   358         val export_names = read_theory_exports(db, session_name)
   361         val export_names = read_theory_exports(db, session_name)
   359 
   362 
   360         // list
   363         // list
   361         if (export_list) {
   364         if (export_list) {
   398   /* Isabelle tool wrapper */
   401   /* Isabelle tool wrapper */
   399 
   402 
   400   val default_export_dir: Path = Path.explode("export")
   403   val default_export_dir: Path = Path.explode("export")
   401 
   404 
   402   val isabelle_tool = Isabelle_Tool("export", "retrieve theory exports",
   405   val isabelle_tool = Isabelle_Tool("export", "retrieve theory exports",
   403     Scala_Project.here, args =>
   406     Scala_Project.here, args => {
   404   {
       
   405     /* arguments */
   407     /* arguments */
   406 
   408 
   407     var export_dir = default_export_dir
   409     var export_dir = default_export_dir
   408     var dirs: List[Path] = Nil
   410     var dirs: List[Path] = Nil
   409     var export_list = false
   411     var export_list = false