Added JDBC-integration (#451)

* Created a module

* Updated a module with the inital integration and test

* Added a new complex example for reading with Native SQL Query

* Added an implementation for a new complex example for reading with Native SQL Query

* Added an implementation for a new complex example for reading with Native SQL Query

* Added idea for test

* Added mariadb4j integration

* Attempt with test containers

* Added the H2 support for testing database capabilities

* Set up a draft Kotlin logging

* Started with ImportDataSchema changes

* Started with ImportDataSchema changes

* Added missed dependencies

* Added simple generation for one table

* Finished simple prototype

* Added some minor ideas

* Fixed bug in the KNB test

* Fixed bug in the KNB test

* Add JDBC support to dataframe gradle plugin

* Added API methods

* Finished API methods

* Added import data schema annotation support

* Added import data schema annotation support

* Added force-classloading for drivers

* Refactored jdbc

* Updated tests

* Support schema generation for SqlQuery

* Support schema generation for SqlQuery

* Added experimental methods

* Added experimental methods

* Added experimental methods

* Added H2 types support

* Added SQlite types support

* Added initial Postgre test data

* Fixed PostgreSQL mapping

* Made test green

* Added type mapping for mariadb and mysql

* Add test for mariadb

* Added initial support for Mariadb

* Refactored sealed hierarchy and fixed SQlite tests

* Updated test and moved coverage above 60 percentage

* Implemented readAllTables function for H2

* Implemented readAllTables for other databases

* Added documentation

* Fixed in plugins

* Fixed in plugins

* Refactored toml

* Fixed symbol processing plugin

* Added integration test

* Fixed inspections

* Added documentation

* Update SQL reading documentation

* Fixed Review Part 1

* Fixed Review Part 1

* Rename test files to match Kotlin conventions and refactor tests

* Refactor readAllTables method name to readAllSqlTables

* Enhance exception messages and add uniqueness check

* Added buildTableMetadata method to DbType and handle JSON type issues

* Ignore test cases due to configuration issues.

* Remove SQL reading features

Author: zaleslaw

build.gradle.kts

@@ -37,6 +37,7 @@ dependencies {
     api(project(":dataframe-arrow"))
     api(project(":dataframe-excel"))
     api(project(":dataframe-openapi"))
+    api(project(":dataframe-jdbc"))
 }
 
 allprojects {

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/annotations/ImportDataSchema.kt

@@ -8,21 +8,24 @@ import org.jetbrains.kotlinx.dataframe.io.JSON
 
 /**
  * Annotation preprocessing will generate a DataSchema interface from the data at `path`.
- * Data must be of supported format: CSV, JSON, Apache Arrow, Excel, OpenAPI (Swagger) in YAML/JSON.
+ * Data must be of supported format: CSV, JSON, Apache Arrow, Excel, OpenAPI (Swagger) in YAML/JSON, JDBC.
  * Generated data schema has properties inferred from data and a companion object with `read method`.
  * `read method` is either `readCSV` or `readJson` that returns `DataFrame<name>`
  *
  * @param name name of the generated interface
  * @param path URL or relative path to data.
- * if path starts with protocol (http, https, ftp), it's considered a URL. Otherwise, it's treated as relative path.
+ * If a path starts with protocol (http, https, ftp, jdbc), it's considered a URL.
+ * Otherwise, it's treated as a relative path.
  * By default, it will be resolved relatively to project dir, i.e. File(projectDir, path)
- * You can configure it by passing `dataframe.resolutionDir` option to preprocessor, see https://kotlinlang.org/docs/ksp-quickstart.html#pass-options-to-processors
+ * You can configure it by passing `dataframe.resolutionDir` option to preprocessor,
+ * see https://kotlinlang.org/docs/ksp-quickstart.html#pass-options-to-processors
  * @param visibility visibility of the generated interface.
  * @param normalizationDelimiters if not empty, split property names by delimiters,
  * lowercase parts and join to camel case. Set empty list to disable normalization
  * @param withDefaultPath if `true`, generate `defaultPath` property to the data schema's companion object and make it default argument for a `read method`
  * @param csvOptions options to parse CSV data. Not used when data is not Csv
  * @param jsonOptions options to parse JSON data. Not used when data is not Json
+ * @param jdbcOptions options to parse data from a database via JDBC. Not used when data is not stored in the database
  */
 @Retention(AnnotationRetention.SOURCE)
 @Target(AnnotationTarget.FILE)
@@ -35,6 +38,7 @@ public annotation class ImportDataSchema(
     val withDefaultPath: Boolean = true,
     val csvOptions: CsvOptions = CsvOptions(','),
     val jsonOptions: JsonOptions = JsonOptions(),
+    val jdbcOptions: JdbcOptions = JdbcOptions(),
 )
 
 public enum class DataSchemaVisibility {
@@ -45,6 +49,12 @@ public annotation class CsvOptions(
     public val delimiter: Char,
 )
 
+public annotation class JdbcOptions(
+    public val user: String = "", // TODO: I'm not sure about the default parameters
+    public val password: String = "", // TODO: I'm not sure about the default parameters)
+    public val sqlQuery: String = ""
+)
+
 public annotation class JsonOptions(
 
     /** Allows the choice of how to handle type clashes when reading a JSON file. */

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/DefaultReadDfMethods.kt

@@ -24,6 +24,7 @@ private const val verify = "verify" // cast(true) is obscure, i think it's bette
 private const val readCSV = "readCSV"
 private const val readTSV = "readTSV"
 private const val readJson = "readJson"
+private const val readJdbc = "readJdbc"
 
 public abstract class AbstractDefaultReadMethod(
     private val path: String?,

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/schema/DataFrameSchemaImpl.kt

@@ -5,7 +5,7 @@ import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema
 import org.jetbrains.kotlinx.dataframe.schema.CompareResult
 import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema
 
-internal class DataFrameSchemaImpl(override val columns: Map<String, ColumnSchema>) : DataFrameSchema {
+public class DataFrameSchemaImpl(override val columns: Map<String, ColumnSchema>) : DataFrameSchema {
 
     override fun compare(other: DataFrameSchema): CompareResult {
         require(other is DataFrameSchemaImpl)

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt

@@ -169,14 +169,15 @@ internal class Integration(
         if (version != null) {
             dependencies(
                 "org.jetbrains.kotlinx:dataframe-excel:$version",
+                "org.jetbrains.kotlinx:dataframe-jdbc:$version",
                 "org.jetbrains.kotlinx:dataframe-arrow:$version",
                 "org.jetbrains.kotlinx:dataframe-openapi:$version",
             )
         }
 
         try {
             setMinimalKernelVersion(MIN_KERNEL_VERSION)
-        } catch (_: NoSuchMethodError) { // will be thrown on version < 0.11.0.198
+        } catch (_: NoSuchMethodError) { // will be thrown when a version < 0.11.0.198
             throw IllegalStateException(
                 getKernelUpdateMessage(notebook.kernelVersion, MIN_KERNEL_VERSION, notebook.jupyterClientType)
             )

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/annotations/ImportDataSchema.kt

@@ -8,21 +8,24 @@ import org.jetbrains.kotlinx.dataframe.io.JSON
 
 /**
  * Annotation preprocessing will generate a DataSchema interface from the data at `path`.
- * Data must be of supported format: CSV, JSON, Apache Arrow, Excel, OpenAPI (Swagger) in YAML/JSON.
+ * Data must be of supported format: CSV, JSON, Apache Arrow, Excel, OpenAPI (Swagger) in YAML/JSON, JDBC.
  * Generated data schema has properties inferred from data and a companion object with `read method`.
  * `read method` is either `readCSV` or `readJson` that returns `DataFrame<name>`
  *
  * @param name name of the generated interface
  * @param path URL or relative path to data.
- * if path starts with protocol (http, https, ftp), it's considered a URL. Otherwise, it's treated as relative path.
+ * If a path starts with protocol (http, https, ftp, jdbc), it's considered a URL.
+ * Otherwise, it's treated as a relative path.
  * By default, it will be resolved relatively to project dir, i.e. File(projectDir, path)
- * You can configure it by passing `dataframe.resolutionDir` option to preprocessor, see https://kotlinlang.org/docs/ksp-quickstart.html#pass-options-to-processors
+ * You can configure it by passing `dataframe.resolutionDir` option to preprocessor,
+ * see https://kotlinlang.org/docs/ksp-quickstart.html#pass-options-to-processors
  * @param visibility visibility of the generated interface.
  * @param normalizationDelimiters if not empty, split property names by delimiters,
  * lowercase parts and join to camel case. Set empty list to disable normalization
  * @param withDefaultPath if `true`, generate `defaultPath` property to the data schema's companion object and make it default argument for a `read method`
  * @param csvOptions options to parse CSV data. Not used when data is not Csv
  * @param jsonOptions options to parse JSON data. Not used when data is not Json
+ * @param jdbcOptions options to parse data from a database via JDBC. Not used when data is not stored in the database
  */
 @Retention(AnnotationRetention.SOURCE)
 @Target(AnnotationTarget.FILE)
@@ -35,6 +38,7 @@ public annotation class ImportDataSchema(
     val withDefaultPath: Boolean = true,
     val csvOptions: CsvOptions = CsvOptions(','),
     val jsonOptions: JsonOptions = JsonOptions(),
+    val jdbcOptions: JdbcOptions = JdbcOptions(),
 )
 
 public enum class DataSchemaVisibility {
@@ -45,6 +49,12 @@ public annotation class CsvOptions(
     public val delimiter: Char,
 )
 
+public annotation class JdbcOptions(
+    public val user: String = "", // TODO: I'm not sure about the default parameters
+    public val password: String = "", // TODO: I'm not sure about the default parameters)
+    public val sqlQuery: String = ""
+)
+
 public annotation class JsonOptions(
 
     /** Allows the choice of how to handle type clashes when reading a JSON file. */

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/DefaultReadDfMethods.kt

@@ -24,6 +24,7 @@ private const val verify = "verify" // cast(true) is obscure, i think it's bette
 private const val readCSV = "readCSV"
 private const val readTSV = "readTSV"
 private const val readJson = "readJson"
+private const val readJdbc = "readJdbc"
 
 public abstract class AbstractDefaultReadMethod(
     private val path: String?,

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/schema/DataFrameSchemaImpl.kt

@@ -5,7 +5,7 @@ import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema
 import org.jetbrains.kotlinx.dataframe.schema.CompareResult
 import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema
 
-internal class DataFrameSchemaImpl(override val columns: Map<String, ColumnSchema>) : DataFrameSchema {
+public class DataFrameSchemaImpl(override val columns: Map<String, ColumnSchema>) : DataFrameSchema {
 
     override fun compare(other: DataFrameSchema): CompareResult {
         require(other is DataFrameSchemaImpl)

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt

@@ -169,14 +169,15 @@ internal class Integration(
         if (version != null) {
             dependencies(
                 "org.jetbrains.kotlinx:dataframe-excel:$version",
+                "org.jetbrains.kotlinx:dataframe-jdbc:$version",
                 "org.jetbrains.kotlinx:dataframe-arrow:$version",
                 "org.jetbrains.kotlinx:dataframe-openapi:$version",
             )
         }
 
         try {
             setMinimalKernelVersion(MIN_KERNEL_VERSION)
-        } catch (_: NoSuchMethodError) { // will be thrown on version < 0.11.0.198
+        } catch (_: NoSuchMethodError) { // will be thrown when a version < 0.11.0.198
             throw IllegalStateException(
                 getKernelUpdateMessage(notebook.kernelVersion, MIN_KERNEL_VERSION, notebook.jupyterClientType)
             )

dataframe-excel/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/xlsx.kt

@@ -57,7 +57,7 @@ internal class DefaultReadExcelMethod(path: String?) : AbstractDefaultReadMethod
 private const val readExcel = "readExcel"
 
 /**
- * @param sheetName sheet to read. By default, first sheet in the document
+ * @param sheetName sheet to read. By default, the first sheet in the document
  * @param columns comma separated list of Excel column letters and column ranges (e.g. “A:E” or “A,C,E:F”)
  * @param skipRows number of rows before header
  * @param rowsCount number of rows to read.
@@ -77,7 +77,7 @@ public fun DataFrame.Companion.readExcel(
 }
 
 /**
- * @param sheetName sheet to read. By default, first sheet in the document
+ * @param sheetName sheet to read. By default, the first sheet in the document
  * @param columns comma separated list of Excel column letters and column ranges (e.g. “A:E” or “A,C,E:F”)
  * @param skipRows number of rows before header
  * @param rowsCount number of rows to read.
@@ -97,7 +97,7 @@ public fun DataFrame.Companion.readExcel(
 }
 
 /**
- * @param sheetName sheet to read. By default, first sheet in the document
+ * @param sheetName sheet to read. By default, the first sheet in the document
  * @param columns comma separated list of Excel column letters and column ranges (e.g. “A:E” or “A,C,E:F”)
  * @param skipRows number of rows before header
  * @param rowsCount number of rows to read.
@@ -114,7 +114,7 @@ public fun DataFrame.Companion.readExcel(
 ): AnyFrame = readExcel(asURL(fileOrUrl), sheetName, skipRows, columns, rowsCount, nameRepairStrategy)
 
 /**
- * @param sheetName sheet to read. By default, first sheet in the document
+ * @param sheetName sheet to read. By default, the first sheet in the document
  * @param columns comma separated list of Excel column letters and column ranges (e.g. “A:E” or “A,C,E:F”)
  * @param skipRows number of rows before header
  * @param rowsCount number of rows to read.
@@ -134,7 +134,7 @@ public fun DataFrame.Companion.readExcel(
 }
 
 /**
- * @param sheetName sheet to read. By default, first sheet in the document
+ * @param sheetName sheet to read. By default, the first sheet in the document
  * @param columns comma separated list of Excel column letters and column ranges (e.g. “A:E” or “A,C,E:F”)
  * @param skipRows number of rows before header
  * @param rowsCount number of rows to read.
@@ -446,18 +446,18 @@ private fun Cell.setCellValueByGuessedType(any: Any) {
 
 /**
  * Set LocalDateTime value correctly also if date have zero value in Excel.
- * Zero date is usually used fore storing time component only,
- * is displayed as 00.01.1900 in Excel and as 30.12.1899 in LibreOffice Calc and also in POI.
+ * Zero dates are usually used for storing a time component only,
+ * are displayed as 00.01.1900 in Excel and as 30.12.1899 in LibreOffice Calc and also in POI.
  * POI can not set 1899 year directly.
  */
 private fun Cell.setTime(localDateTime: LocalDateTime) {
     this.setCellValue(DateUtil.getExcelDate(localDateTime.plusDays(1)) - 1.0)
 }
 
 /**
- * Set Date value correctly also if date have zero value in Excel.
- * Zero date is usually used fore storing time component only,
- * is displayed as 00.01.1900 in Excel and as 30.12.1899 in LibreOffice Calc and also in POI.
+ * Set Date value correctly also if date has zero value in Excel.
+ * Zero dates are usually used for storing a time component only,
+ * are displayed as 00.01.1900 in Excel and as 30.12.1899 in LibreOffice Calc and also in POI.
  * POI can not set 1899 year directly.
  */
 private fun Cell.setDate(date: Date) {