Skip to content

Commit fdfa60c

Browse files
kinationkination
authored
Merge pull request #7 from kination/apply-trino-sdk
Apply trino sdk
2 parents 74ef708 + cff1c57 commit fdfa60c

17 files changed

Lines changed: 1047 additions & 0 deletions

‎vine-trino/.gitignore‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
.gradle/
2+
build/
3+
!gradle/wrapper/gradle-wrapper.jar
4+
*.class
5+
*.jar
6+
!gradle/wrapper/*.jar
7+
.idea/
8+
*.iml
9+
out/

‎vine-trino/README.md‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
# Vine Trino Connector
2+
3+
A read-only [Trino](https://trino.io/) connector for querying Vine tables stored in [Vortex](https://github.com/spiraldb/vortex) columnar format (`.vtx`) via standard SQL.
4+
5+
## Requirements
6+
7+
- Java 11+
8+
- Trino 439
9+
- vine-core native library (requires Rust build)
10+
11+
## Build
12+
13+
```bash
14+
# Build vine-core native library (Rust)
15+
cd vine-core && cargo build --release && cd ..
16+
17+
# Build vine-trino
18+
cd vine-trino
19+
./gradlew clean build # compile + test
20+
./gradlew shadowJar # produce deployable fat JAR
21+
```
22+
23+
Build artifacts:
24+
- `build/libs/vine-trino-0.1.0-all.jar` — shadow JAR (bundles Arrow, Jackson with relocated packages)
25+
- `build/libs/vine-trino-0.1.0.jar` — thin JAR
26+
27+
## Deployment
28+
29+
### 1. Install the Plugin
30+
31+
```bash
32+
mkdir -p <trino-home>/plugin/vine/
33+
cp build/libs/vine-trino-0.1.0-all.jar <trino-home>/plugin/vine/
34+
35+
# Copy the native library for your platform:
36+
# macOS
37+
cp ../vine-core/target/release/libvine_core.dylib <trino-home>/plugin/vine/
38+
# Linux
39+
cp ../vine-core/target/release/libvine_core.so <trino-home>/plugin/vine/
40+
```
41+
42+
### 2. Configure the Catalog
43+
44+
Create `<trino-home>/etc/catalog/vine.properties`:
45+
46+
```properties
47+
connector.name=vine
48+
vine.data-dir=/path/to/vine/tables
49+
```
50+
51+
### 3. Data Directory Layout
52+
53+
The path specified by `vine.data-dir` must follow this structure. Each subdirectory with a `vine_meta.json` file is treated as a table.
54+
55+
```
56+
/path/to/vine/tables/
57+
├── events/
58+
│ ├── vine_meta.json
59+
│ ├── 2024-12-26/
60+
│ │ ├── data_143025_123456000.vtx
61+
│ │ └── data_150130_789012000.vtx
62+
│ └── 2024-12-27/
63+
│ └── data_091500_345678000.vtx
64+
└── users/
65+
├── vine_meta.json
66+
└── 2024-12-26/
67+
└── data_100000_000000000.vtx
68+
```
69+
70+
### 4. Query
71+
72+
```sql
73+
-- List tables
74+
SHOW TABLES FROM vine.default;
75+
76+
-- Inspect schema
77+
DESCRIBE vine.default.events;
78+
79+
-- Query data
80+
SELECT * FROM vine.default.events;
81+
82+
SELECT user_id, COUNT(*) AS event_count
83+
FROM vine.default.events
84+
GROUP BY user_id;
85+
```
86+
87+
## Architecture
88+
89+
### Data Flow
90+
91+
```
92+
Trino SQL Query
93+
│
94+
▼
95+
VinePlugin ← discovered via ServiceLoader
96+
│
97+
▼
98+
VineConnectorFactory ← reads vine.data-dir config
99+
│
100+
▼
101+
VineConnector
102+
├─ VineConnectorMetadata vine_meta.json → schema / table / column info
103+
├─ VineSplitManager one split per table
104+
└─ VineRecordSetProvider
105+
│
106+
▼
107+
VineModule.readDataArrow(path) [JNI → Rust vine-core]
108+
│
109+
▼
110+
Arrow IPC bytes
111+
│
112+
▼
113+
VineArrowConverter Arrow IPC → Object[][]
114+
│
115+
▼
116+
VineRecordCursor row-by-row delivery to Trino
117+
```
118+
119+
### Module Structure
120+
121+
```
122+
io.kination.vine/
123+
├── VinePlugin.java Trino Plugin entry point
124+
├── VineConnectorFactory.java Creates Connector from catalog properties
125+
├── VineConnector.java Read-only connector (metadata + splits + record sets)
126+
├── VineTransactionHandle.java Singleton transaction handle
127+
│
128+
├── VineConnectorMetadata.java Schema discovery (listSchemas, listTables, getColumnHandles)
129+
├── VineMetadata.java vine_meta.json POJO
130+
├── VineMetadataReader.java vine_meta.json parser (Jackson)
131+
├── VineTypeMapping.java Vine type → Trino type mapping
132+
│
133+
├── VineTableHandle.java Table reference (schema, name, path)
134+
├── VineColumnHandle.java Column reference (name, type, ordinal)
135+
│
136+
├── VineSplitManager.java Split generation (1 per table)
137+
├── VineSplit.java Split payload (table path)
138+
│
139+
├── VineRecordSetProvider.java JNI invocation → RecordSet creation
140+
├── VineRecordSet.java Holds Arrow data, produces cursor
141+
├── VineRecordCursor.java Row-by-row cursor consumed by Trino
142+
├── VineArrowConverter.java Arrow IPC → Object[][] conversion
143+
│
144+
└── VineModule.java JNI bridge (readDataArrow)
145+
```
146+
147+
### Type Mapping
148+
149+
| Vine Type (`vine_meta.json`) | Alias | Trino Type | Arrow Vector | Cursor Method |
150+
|---|---|---|---|---|
151+
| `integer` | `int` | `INTEGER` | IntVector | `getLong()` |
152+
| `long` | `bigint` | `BIGINT` | BigIntVector | `getLong()` |
153+
| `short` | `smallint` | `SMALLINT` | SmallIntVector | `getLong()` |
154+
| `byte` | `tinyint` | `TINYINT` | TinyIntVector | `getLong()` |
155+
| `float` | — | `REAL` | Float4Vector | `getLong()` (float bits) |
156+
| `double` | — | `DOUBLE` | Float8Vector | `getDouble()` |
157+
| `boolean` | `bool` | `BOOLEAN` | BitVector | `getBoolean()` |
158+
| `string` | — | `VARCHAR` | VarCharVector | `getSlice()` |
159+
| `binary` | — | `VARBINARY` | VarBinaryVector | `getSlice()` |
160+
| `date` | — | `DATE` | DateDayVector | `getLong()` |
161+
| `timestamp` | — | `TIMESTAMP(3)` | TimeStampMilliVector | `getLong()` |
162+
| `decimal` | — | `VARCHAR` | VarCharVector | `getSlice()` |
163+
164+
### vine_meta.json Schema
165+
166+
```json
167+
{
168+
"table_name": "events",
169+
"fields": [
170+
{"id": 1, "name": "user_id", "data_type": "integer", "is_required": true},
171+
{"id": 2, "name": "event_type", "data_type": "string", "is_required": false},
172+
{"id": 3, "name": "timestamp", "data_type": "long", "is_required": true}
173+
]
174+
}
175+
```
176+
177+
## Limitations
178+
179+
- **Read-only** — only `SELECT` queries are supported; `INSERT`, `UPDATE`, and `DELETE` are not implemented.
180+
- **Single split per table** — the entire table is read in one pass, which may cause high memory usage for large datasets.
181+
- **Single schema** — all tables reside under the `default` schema.
182+
- **No partition pruning** — all date partitions are read regardless of query predicates.
183+
184+
## Roadmap
185+
186+
- Per-partition splits for parallel reads
187+
- Partition pruning based on `WHERE` clause predicates
188+
- Predicate pushdown
189+
- Column pruning (project only required columns)
190+
- Hive Metastore (HMS) integration

‎vine-trino/build.gradle‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
plugins {
2+
id 'java'
3+
id 'com.github.johnrengelman.shadow' version '8.1.1'
4+
}
5+
6+
group = 'io.kination.vine'
7+
version = '0.1.0'
8+
9+
java {
10+
sourceCompatibility = JavaVersion.VERSION_11
11+
targetCompatibility = JavaVersion.VERSION_11
12+
}
13+
14+
repositories {
15+
mavenCentral()
16+
}
17+
18+
ext {
19+
trinoVersion = '439'
20+
arrowVersion = '14.0.2'
21+
}
22+
23+
dependencies {
24+
// Trino SPI (provided at runtime by Trino)
25+
compileOnly "io.trino:trino-spi:${trinoVersion}"
26+
compileOnly "io.airlift:slice:2.1"
27+
28+
// Jackson for vine_meta.json parsing
29+
implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.3'
30+
31+
// Apache Arrow for reading JNI data
32+
implementation "org.apache.arrow:arrow-vector:${arrowVersion}"
33+
implementation "org.apache.arrow:arrow-memory-netty:${arrowVersion}"
34+
35+
// Logging
36+
implementation 'org.slf4j:slf4j-api:2.0.9'
37+
38+
// Test dependencies
39+
testImplementation "io.trino:trino-spi:${trinoVersion}"
40+
testImplementation "io.airlift:slice:2.1"
41+
testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0'
42+
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
43+
}
44+
45+
test {
46+
useJUnitPlatform()
47+
48+
// Set native library path for tests
49+
systemProperty 'java.library.path', "${projectDir}/../vine-core/target/release"
50+
51+
testLogging {
52+
events "passed", "skipped", "failed"
53+
showStandardStreams = true
54+
}
55+
}
56+
57+
shadowJar {
58+
archiveBaseName.set('vine-trino')
59+
archiveClassifier.set('all')
60+
61+
// Relocate to avoid conflicts with Trino's bundled libraries
62+
relocate 'org.apache.arrow', 'io.kination.vine.shaded.arrow'
63+
relocate 'com.fasterxml.jackson', 'io.kination.vine.shaded.jackson'
64+
65+
// Exclude Trino dependencies (provided at runtime)
66+
dependencies {
67+
exclude(dependency('io.trino:.*'))
68+
exclude(dependency('io.airlift:.*'))
69+
}
70+
71+
mergeServiceFiles()
72+
}
73+
74+
tasks.named('jar') {
75+
manifest {
76+
attributes(
77+
'Implementation-Title': 'Vine Trino Connector',
78+
'Implementation-Version': version
79+
)
80+
}
81+
}
82+
83+
// Task to copy native library for local testing
84+
tasks.register('copyNativeLib', Copy) {
85+
from "${projectDir}/../vine-core/target/release"
86+
into "${buildDir}/native"
87+
include "libvine_core.*"
88+
include "vine_core.dll"
89+
}
90+
91+
test.dependsOn copyNativeLib

‎vine-trino/settings.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
rootProject.name = 'vine-trino'

0 commit comments

Comments
 (0)