src/Pure/General/sql.scala
author wenzelm
Fri Apr 28 11:50:31 2017 +0200 (2017-04-28)
changeset 65602 d9533e9615ad
parent 65598 5deef985e38e
child 65610 e6e3fed86519
permissions -rw-r--r--
tuned signature;
wenzelm@63788
     1
/*  Title:      Pure/General/sql.scala
wenzelm@63778
     2
    Author:     Makarius
wenzelm@63778
     3
wenzelm@65006
     4
Support for SQL databases: SQLite and PostgreSQL.
wenzelm@63778
     5
*/
wenzelm@63778
     6
wenzelm@63778
     7
package isabelle
wenzelm@63778
     8
wenzelm@65021
     9
import java.time.OffsetDateTime
wenzelm@65021
    10
import java.sql.{DriverManager, Connection, PreparedStatement, ResultSet}
wenzelm@63779
    11
wenzelm@63779
    12
wenzelm@63778
    13
object SQL
wenzelm@63778
    14
{
wenzelm@65006
    15
  /** SQL language **/
wenzelm@65006
    16
wenzelm@63778
    17
  /* concrete syntax */
wenzelm@63778
    18
wenzelm@65321
    19
  def escape_char(c: Char): String =
wenzelm@63778
    20
    c match {
wenzelm@63778
    21
      case '\u0000' => "\\0"
wenzelm@63778
    22
      case '\'' => "\\'"
wenzelm@63778
    23
      case '\"' => "\\\""
wenzelm@63778
    24
      case '\b' => "\\b"
wenzelm@63778
    25
      case '\n' => "\\n"
wenzelm@63778
    26
      case '\r' => "\\r"
wenzelm@63778
    27
      case '\t' => "\\t"
wenzelm@63778
    28
      case '\u001a' => "\\Z"
wenzelm@63778
    29
      case '\\' => "\\\\"
wenzelm@63778
    30
      case _ => c.toString
wenzelm@63778
    31
    }
wenzelm@63778
    32
wenzelm@63778
    33
  def quote_string(s: String): String =
wenzelm@65321
    34
    "'" + s.map(escape_char(_)).mkString + "'"
wenzelm@63778
    35
wenzelm@63779
    36
  def quote_ident(s: String): String =
wenzelm@65003
    37
    quote(s.replace("\"", "\"\""))
wenzelm@63779
    38
wenzelm@63791
    39
  def enclosure(ss: Iterable[String]): String = ss.mkString("(", ", ", ")")
wenzelm@63791
    40
wenzelm@63779
    41
wenzelm@65008
    42
  /* types */
wenzelm@65008
    43
wenzelm@65008
    44
  object Type extends Enumeration
wenzelm@65008
    45
  {
wenzelm@65011
    46
    val Boolean = Value("BOOLEAN")
wenzelm@65008
    47
    val Int = Value("INTEGER")
wenzelm@65008
    48
    val Long = Value("BIGINT")
wenzelm@65008
    49
    val Double = Value("DOUBLE PRECISION")
wenzelm@65008
    50
    val String = Value("TEXT")
wenzelm@65008
    51
    val Bytes = Value("BLOB")
wenzelm@65014
    52
    val Date = Value("TIMESTAMP WITH TIME ZONE")
wenzelm@65008
    53
  }
wenzelm@65008
    54
wenzelm@65019
    55
  def sql_type_default(T: Type.Value): String = T.toString
wenzelm@65013
    56
wenzelm@65019
    57
  def sql_type_sqlite(T: Type.Value): String =
wenzelm@65019
    58
    if (T == Type.Boolean) "INTEGER"
wenzelm@65019
    59
    else if (T == Type.Date) "TEXT"
wenzelm@65019
    60
    else sql_type_default(T)
wenzelm@65013
    61
wenzelm@65019
    62
  def sql_type_postgresql(T: Type.Value): String =
wenzelm@65019
    63
    if (T == Type.Bytes) "BYTEA"
wenzelm@65019
    64
    else sql_type_default(T)
wenzelm@65008
    65
wenzelm@65008
    66
wenzelm@63779
    67
  /* columns */
wenzelm@63779
    68
wenzelm@63779
    69
  object Column
wenzelm@63779
    70
  {
wenzelm@65280
    71
    def bool(name: String, strict: Boolean = false, primary_key: Boolean = false): Column =
wenzelm@65018
    72
      Column(name, Type.Boolean, strict, primary_key)
wenzelm@65280
    73
    def int(name: String, strict: Boolean = false, primary_key: Boolean = false): Column =
wenzelm@65018
    74
      Column(name, Type.Int, strict, primary_key)
wenzelm@65280
    75
    def long(name: String, strict: Boolean = false, primary_key: Boolean = false): Column =
wenzelm@65018
    76
      Column(name, Type.Long, strict, primary_key)
wenzelm@65280
    77
    def double(name: String, strict: Boolean = false, primary_key: Boolean = false): Column =
wenzelm@65018
    78
      Column(name, Type.Double, strict, primary_key)
wenzelm@65280
    79
    def string(name: String, strict: Boolean = false, primary_key: Boolean = false): Column =
wenzelm@65018
    80
      Column(name, Type.String, strict, primary_key)
wenzelm@65280
    81
    def bytes(name: String, strict: Boolean = false, primary_key: Boolean = false): Column =
wenzelm@65018
    82
      Column(name, Type.Bytes, strict, primary_key)
wenzelm@65280
    83
    def date(name: String, strict: Boolean = false, primary_key: Boolean = false): Column =
wenzelm@65018
    84
      Column(name, Type.Date, strict, primary_key)
wenzelm@63779
    85
  }
wenzelm@63779
    86
wenzelm@65018
    87
  sealed case class Column(
wenzelm@65280
    88
    name: String, T: Type.Value, strict: Boolean = false, primary_key: Boolean = false)
wenzelm@63779
    89
  {
wenzelm@63779
    90
    def sql_name: String = quote_ident(name)
wenzelm@65019
    91
    def sql_decl(sql_type: Type.Value => String): String =
wenzelm@65325
    92
      sql_name + " " + sql_type(T) + (if (strict || primary_key) " NOT NULL" else "")
wenzelm@63781
    93
wenzelm@65593
    94
    def sql_where_eq: String = "WHERE " + sql_name + " = "
wenzelm@65602
    95
    def sql_where_equal(s: String): String = sql_where_eq + quote_string(s)
wenzelm@65593
    96
wenzelm@65019
    97
    override def toString: String = sql_decl(sql_type_default)
wenzelm@63779
    98
  }
wenzelm@63779
    99
wenzelm@63780
   100
wenzelm@63780
   101
  /* tables */
wenzelm@63780
   102
wenzelm@65018
   103
  sealed case class Table(name: String, columns: List[Column])
wenzelm@63780
   104
  {
wenzelm@63790
   105
    private val columns_index: Map[String, Int] =
wenzelm@63790
   106
      columns.iterator.map(_.name).zipWithIndex.toMap
wenzelm@63790
   107
wenzelm@63781
   108
    Library.duplicates(columns.map(_.name)) match {
wenzelm@63781
   109
      case Nil =>
wenzelm@63781
   110
      case bad => error("Duplicate column names " + commas_quote(bad) + " for table " + quote(name))
wenzelm@63781
   111
    }
wenzelm@63781
   112
wenzelm@65325
   113
    def sql_columns(sql_type: Type.Value => String): String =
wenzelm@65325
   114
    {
wenzelm@65325
   115
      val primary_key =
wenzelm@65325
   116
        columns.filter(_.primary_key).map(_.name) match {
wenzelm@65325
   117
          case Nil => Nil
wenzelm@65325
   118
          case keys => List("PRIMARY KEY " + enclosure(keys))
wenzelm@65325
   119
        }
wenzelm@65325
   120
      enclosure(columns.map(_.sql_decl(sql_type)) ::: primary_key)
wenzelm@63781
   121
    }
wenzelm@63781
   122
wenzelm@65327
   123
    def sql_create(strict: Boolean, sql_type: Type.Value => String): String =
wenzelm@63784
   124
      "CREATE TABLE " + (if (strict) "" else "IF NOT EXISTS ") +
wenzelm@65327
   125
        quote_ident(name) + " " + sql_columns(sql_type)
wenzelm@63780
   126
wenzelm@63780
   127
    def sql_drop(strict: Boolean): String =
wenzelm@63784
   128
      "DROP TABLE " + (if (strict) "" else "IF EXISTS ") + quote_ident(name)
wenzelm@63783
   129
wenzelm@63791
   130
    def sql_create_index(
wenzelm@65018
   131
        index_name: String, index_columns: List[Column],
wenzelm@63791
   132
        strict: Boolean, unique: Boolean): String =
wenzelm@63791
   133
      "CREATE " + (if (unique) "UNIQUE " else "") + "INDEX " +
wenzelm@63791
   134
        (if (strict) "" else "IF NOT EXISTS ") + quote_ident(index_name) + " ON " +
wenzelm@63791
   135
        quote_ident(name) + " " + enclosure(index_columns.map(_.name))
wenzelm@63791
   136
wenzelm@63791
   137
    def sql_drop_index(index_name: String, strict: Boolean): String =
wenzelm@63791
   138
      "DROP INDEX " + (if (strict) "" else "IF EXISTS ") + quote_ident(index_name)
wenzelm@63791
   139
wenzelm@63790
   140
    def sql_insert: String =
wenzelm@63791
   141
      "INSERT INTO " + quote_ident(name) + " VALUES " + enclosure(columns.map(_ => "?"))
wenzelm@63791
   142
wenzelm@65319
   143
    def sql_delete: String =
wenzelm@65319
   144
      "DELETE FROM " + quote_ident(name)
wenzelm@65319
   145
wenzelm@65018
   146
    def sql_select(select_columns: List[Column], distinct: Boolean): String =
wenzelm@63791
   147
      "SELECT " + (if (distinct) "DISTINCT " else "") +
wenzelm@63791
   148
      commas(select_columns.map(_.sql_name)) + " FROM " + quote_ident(name)
wenzelm@63790
   149
wenzelm@63783
   150
    override def toString: String =
wenzelm@65325
   151
      "TABLE " + quote_ident(name) + " " + sql_columns(sql_type_default)
wenzelm@63780
   152
  }
wenzelm@63790
   153
wenzelm@63790
   154
wenzelm@65012
   155
wenzelm@65012
   156
  /** SQL database operations **/
wenzelm@65012
   157
wenzelm@63790
   158
  /* results */
wenzelm@63790
   159
wenzelm@63790
   160
  def iterator[A](rs: ResultSet)(get: ResultSet => A): Iterator[A] = new Iterator[A]
wenzelm@63790
   161
  {
wenzelm@63790
   162
    private var _next: Boolean = rs.next()
wenzelm@63790
   163
    def hasNext: Boolean = _next
wenzelm@63790
   164
    def next: A = { val x = get(rs); _next = rs.next(); x }
wenzelm@63790
   165
  }
wenzelm@65006
   166
wenzelm@65006
   167
  trait Database
wenzelm@65006
   168
  {
wenzelm@65008
   169
    /* types */
wenzelm@65008
   170
wenzelm@65019
   171
    def sql_type(T: Type.Value): String
wenzelm@65008
   172
wenzelm@65008
   173
wenzelm@65006
   174
    /* connection */
wenzelm@65006
   175
wenzelm@65006
   176
    def connection: Connection
wenzelm@65006
   177
wenzelm@65006
   178
    def close() { connection.close }
wenzelm@65006
   179
wenzelm@65006
   180
    def transaction[A](body: => A): A =
wenzelm@65006
   181
    {
wenzelm@65006
   182
      val auto_commit = connection.getAutoCommit
wenzelm@65006
   183
      try {
wenzelm@65006
   184
        connection.setAutoCommit(false)
wenzelm@65022
   185
        val savepoint = connection.setSavepoint
wenzelm@65022
   186
        try {
wenzelm@65022
   187
          val result = body
wenzelm@65022
   188
          connection.commit
wenzelm@65022
   189
          result
wenzelm@65022
   190
        }
wenzelm@65022
   191
        catch { case exn: Throwable => connection.rollback(savepoint); throw exn }
wenzelm@65006
   192
      }
wenzelm@65006
   193
      finally { connection.setAutoCommit(auto_commit) }
wenzelm@65006
   194
    }
wenzelm@65006
   195
wenzelm@65006
   196
wenzelm@65006
   197
    /* statements */
wenzelm@65006
   198
wenzelm@65006
   199
    def statement(sql: String): PreparedStatement = connection.prepareStatement(sql)
wenzelm@65006
   200
wenzelm@65006
   201
    def insert_statement(table: Table): PreparedStatement = statement(table.sql_insert)
wenzelm@65006
   202
wenzelm@65319
   203
    def delete_statement(table: Table, sql: String = ""): PreparedStatement =
wenzelm@65319
   204
      statement(table.sql_delete + (if (sql == "") "" else " " + sql))
wenzelm@65319
   205
wenzelm@65018
   206
    def select_statement(table: Table, columns: List[Column],
wenzelm@65006
   207
        sql: String = "", distinct: Boolean = false): PreparedStatement =
wenzelm@65006
   208
      statement(table.sql_select(columns, distinct) + (if (sql == "") "" else " " + sql))
wenzelm@65006
   209
wenzelm@65006
   210
wenzelm@65020
   211
    /* input */
wenzelm@65020
   212
wenzelm@65020
   213
    def set_bool(stmt: PreparedStatement, i: Int, x: Boolean) { stmt.setBoolean(i, x) }
wenzelm@65020
   214
    def set_int(stmt: PreparedStatement, i: Int, x: Int) { stmt.setInt(i, x) }
wenzelm@65020
   215
    def set_long(stmt: PreparedStatement, i: Int, x: Long) { stmt.setLong(i, x) }
wenzelm@65020
   216
    def set_double(stmt: PreparedStatement, i: Int, x: Double) { stmt.setDouble(i, x) }
wenzelm@65020
   217
    def set_string(stmt: PreparedStatement, i: Int, x: String) { stmt.setString(i, x) }
wenzelm@65020
   218
    def set_bytes(stmt: PreparedStatement, i: Int, bytes: Bytes)
wenzelm@65020
   219
    { stmt.setBinaryStream(i, bytes.stream(), bytes.length) }
wenzelm@65020
   220
    def set_date(stmt: PreparedStatement, i: Int, date: Date)
wenzelm@65020
   221
wenzelm@65022
   222
wenzelm@65020
   223
    /* output */
wenzelm@65018
   224
wenzelm@65018
   225
    def bool(rs: ResultSet, name: String): Boolean = rs.getBoolean(name)
wenzelm@65018
   226
    def int(rs: ResultSet, name: String): Int = rs.getInt(name)
wenzelm@65018
   227
    def long(rs: ResultSet, name: String): Long = rs.getLong(name)
wenzelm@65018
   228
    def double(rs: ResultSet, name: String): Double = rs.getDouble(name)
wenzelm@65018
   229
    def string(rs: ResultSet, name: String): String =
wenzelm@65018
   230
    {
wenzelm@65018
   231
      val s = rs.getString(name)
wenzelm@65018
   232
      if (s == null) "" else s
wenzelm@65018
   233
    }
wenzelm@65018
   234
    def bytes(rs: ResultSet, name: String): Bytes =
wenzelm@65018
   235
    {
wenzelm@65018
   236
      val bs = rs.getBytes(name)
wenzelm@65018
   237
      if (bs == null) Bytes.empty else Bytes(bs)
wenzelm@65018
   238
    }
wenzelm@65021
   239
    def date(rs: ResultSet, name: String): Date
wenzelm@65018
   240
wenzelm@65323
   241
    def bool(rs: ResultSet, column: Column): Boolean = bool(rs, column.name)
wenzelm@65323
   242
    def int(rs: ResultSet, column: Column): Int = int(rs, column.name)
wenzelm@65323
   243
    def long(rs: ResultSet, column: Column): Long = long(rs, column.name)
wenzelm@65323
   244
    def double(rs: ResultSet, column: Column): Double = double(rs, column.name)
wenzelm@65323
   245
    def string(rs: ResultSet, column: Column): String = string(rs, column.name)
wenzelm@65323
   246
    def bytes(rs: ResultSet, column: Column): Bytes = bytes(rs, column.name)
wenzelm@65323
   247
    def date(rs: ResultSet, column: Column): Date = date(rs, column.name)
wenzelm@65323
   248
wenzelm@65323
   249
    def get[A, B](rs: ResultSet, a: A, f: (ResultSet, A) => B): Option[B] =
wenzelm@65018
   250
    {
wenzelm@65323
   251
      val x = f(rs, a)
wenzelm@65018
   252
      if (rs.wasNull) None else Some(x)
wenzelm@65018
   253
    }
wenzelm@65018
   254
wenzelm@65018
   255
wenzelm@65006
   256
    /* tables */
wenzelm@65006
   257
wenzelm@65006
   258
    def tables: List[String] =
wenzelm@65006
   259
      iterator(connection.getMetaData.getTables(null, null, "%", null))(_.getString(3)).toList
wenzelm@65006
   260
wenzelm@65327
   261
    def create_table(table: Table, strict: Boolean = false, sql: String = ""): Unit =
wenzelm@65327
   262
      using(statement(table.sql_create(strict, sql_type) + (if (sql == "") "" else " " + sql)))(
wenzelm@65327
   263
        _.execute())
wenzelm@65006
   264
wenzelm@65280
   265
    def drop_table(table: Table, strict: Boolean = false): Unit =
wenzelm@65006
   266
      using(statement(table.sql_drop(strict)))(_.execute())
wenzelm@65006
   267
wenzelm@65018
   268
    def create_index(table: Table, name: String, columns: List[Column],
wenzelm@65280
   269
        strict: Boolean = false, unique: Boolean = false): Unit =
wenzelm@65006
   270
      using(statement(table.sql_create_index(name, columns, strict, unique)))(_.execute())
wenzelm@65006
   271
wenzelm@65280
   272
    def drop_index(table: Table, name: String, strict: Boolean = false): Unit =
wenzelm@65006
   273
      using(statement(table.sql_drop_index(name, strict)))(_.execute())
wenzelm@65006
   274
  }
wenzelm@63778
   275
}
wenzelm@65006
   276
wenzelm@65006
   277
wenzelm@65006
   278
wenzelm@65006
   279
/** SQLite **/
wenzelm@65006
   280
wenzelm@65006
   281
object SQLite
wenzelm@65006
   282
{
wenzelm@65021
   283
  // see https://www.sqlite.org/lang_datefunc.html
wenzelm@65021
   284
  val date_format: Date.Format = Date.Format("uuuu-MM-dd HH:mm:ss.SSS x")
wenzelm@65021
   285
wenzelm@65292
   286
  lazy val init_jdbc: Unit = Class.forName("org.sqlite.JDBC")
wenzelm@65292
   287
wenzelm@65006
   288
  def open_database(path: Path): Database =
wenzelm@65006
   289
  {
wenzelm@65292
   290
    init_jdbc
wenzelm@65006
   291
    val path0 = path.expand
wenzelm@65006
   292
    val s0 = File.platform_path(path0)
wenzelm@65006
   293
    val s1 = if (Platform.is_windows) s0.replace('\\', '/') else s0
wenzelm@65006
   294
    val connection = DriverManager.getConnection("jdbc:sqlite:" + s1)
wenzelm@65007
   295
    new Database(path0.toString, connection)
wenzelm@65006
   296
  }
wenzelm@65006
   297
wenzelm@65007
   298
  class Database private[SQLite](name: String, val connection: Connection) extends SQL.Database
wenzelm@65006
   299
  {
wenzelm@65007
   300
    override def toString: String = name
wenzelm@65006
   301
wenzelm@65019
   302
    def sql_type(T: SQL.Type.Value): String = SQL.sql_type_sqlite(T)
wenzelm@65011
   303
wenzelm@65021
   304
    def set_date(stmt: PreparedStatement, i: Int, date: Date): Unit =
wenzelm@65598
   305
      if (date == null) set_string(stmt, i, null)
wenzelm@65598
   306
      else set_string(stmt, i, date_format(date))
wenzelm@65598
   307
wenzelm@65021
   308
    def date(rs: ResultSet, name: String): Date =
wenzelm@65021
   309
      date_format.parse(string(rs, name))
wenzelm@65021
   310
wenzelm@65006
   311
    def rebuild { using(statement("VACUUM"))(_.execute()) }
wenzelm@65006
   312
  }
wenzelm@65006
   313
}
wenzelm@65006
   314
wenzelm@65006
   315
wenzelm@65006
   316
wenzelm@65006
   317
/** PostgreSQL **/
wenzelm@65006
   318
wenzelm@65006
   319
object PostgreSQL
wenzelm@65006
   320
{
wenzelm@65006
   321
  val default_port = 5432
wenzelm@65006
   322
wenzelm@65292
   323
  lazy val init_jdbc: Unit = Class.forName("org.postgresql.Driver")
wenzelm@65292
   324
wenzelm@65006
   325
  def open_database(
wenzelm@65006
   326
    user: String,
wenzelm@65006
   327
    password: String,
wenzelm@65006
   328
    database: String = "",
wenzelm@65006
   329
    host: String = "",
wenzelm@65594
   330
    port: Int = 0,
wenzelm@65009
   331
    ssh: Option[SSH.Session] = None): Database =
wenzelm@65006
   332
  {
wenzelm@65292
   333
    init_jdbc
wenzelm@65292
   334
wenzelm@65006
   335
    require(user != "")
wenzelm@65009
   336
wenzelm@65009
   337
    val db_host = if (host != "") host else "localhost"
wenzelm@65594
   338
    val db_port = if (port > 0 && port != default_port) ":" + port else ""
wenzelm@65009
   339
    val db_name = "/" + (if (database != "") database else user)
wenzelm@65009
   340
wenzelm@65010
   341
    val (url, name, port_forwarding) =
wenzelm@65009
   342
      ssh match {
wenzelm@65010
   343
        case None =>
wenzelm@65010
   344
          val spec = db_host + db_port + db_name
wenzelm@65010
   345
          val url = "jdbc:postgresql://" + spec
wenzelm@65010
   346
          val name = user + "@" + spec
wenzelm@65010
   347
          (url, name, None)
wenzelm@65009
   348
        case Some(ssh) =>
wenzelm@65594
   349
          val fw =
wenzelm@65594
   350
            ssh.port_forwarding(remote_host = db_host,
wenzelm@65594
   351
              remote_port = if (port > 0) port else default_port)
wenzelm@65010
   352
          val url = "jdbc:postgresql://localhost:" + fw.local_port + db_name
wenzelm@65010
   353
          val name = user + "@" + fw + db_name + " via ssh " + ssh
wenzelm@65010
   354
          (url, name, Some(fw))
wenzelm@65009
   355
      }
wenzelm@65009
   356
    try {
wenzelm@65010
   357
      val connection = DriverManager.getConnection(url, user, password)
wenzelm@65010
   358
      new Database(name, connection, port_forwarding)
wenzelm@65009
   359
    }
wenzelm@65009
   360
    catch { case exn: Throwable => port_forwarding.foreach(_.close); throw exn }
wenzelm@65006
   361
  }
wenzelm@65006
   362
wenzelm@65009
   363
  class Database private[PostgreSQL](
wenzelm@65009
   364
      name: String, val connection: Connection, port_forwarding: Option[SSH.Port_Forwarding])
wenzelm@65009
   365
    extends SQL.Database
wenzelm@65006
   366
  {
wenzelm@65010
   367
    override def toString: String = name
wenzelm@65008
   368
wenzelm@65019
   369
    def sql_type(T: SQL.Type.Value): String = SQL.sql_type_postgresql(T)
wenzelm@65009
   370
wenzelm@65021
   371
    // see https://jdbc.postgresql.org/documentation/head/8-date-time.html
wenzelm@65021
   372
    def set_date(stmt: PreparedStatement, i: Int, date: Date): Unit =
wenzelm@65598
   373
      if (date == null) stmt.setObject(i, null)
wenzelm@65598
   374
      else stmt.setObject(i, OffsetDateTime.from(date.to_utc.rep))
wenzelm@65598
   375
wenzelm@65021
   376
    def date(rs: ResultSet, name: String): Date =
wenzelm@65021
   377
      Date.instant(rs.getObject(name, classOf[OffsetDateTime]).toInstant)
wenzelm@65021
   378
wenzelm@65009
   379
    override def close() { super.close; port_forwarding.foreach(_.close) }
wenzelm@65006
   380
  }
wenzelm@65006
   381
}