Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# NEM12 Parser

A Kotlin-based parser for NEM12 format energy meter reading files, generating SQL INSERT statements for the `meter_readings` database table.
A Kotlin-based parser for NEM12 format energy meter reading files, generating SQL INSERT statements for the `meter_reading` database table.

## Features

Expand Down Expand Up @@ -114,13 +114,13 @@ class BatchInsertGenerator(outputPath: Path, batchSize: Int) : SQLGenerator
## Database Schema

```sql
create table meter_readings (
create table meter_reading (
id uuid default gen_random_uuid() not null,
nmi varchar(10) not null,
timestamp timestamp not null,
consumption numeric not null,
constraint meter_readings_pk primary key (id),
constraint meter_readings_unique_consumption unique (nmi, timestamp)
constraint meter_reading_pk primary key (id),
constraint meter_reading_unique_consumption unique (nmi, timestamp)
);
```

Expand All @@ -144,7 +144,7 @@ create table meter_readings (

**Output (PostgreSQL COPY):**
```sql
COPY meter_readings (nmi, timestamp, consumption) FROM STDIN WITH (FORMAT CSV);
COPY meter_reading (nmi, timestamp, consumption) FROM STDIN WITH (FORMAT CSV);
NEM1201009,2005-03-01 00:00:00,0.461
NEM1201009,2005-03-01 00:30:00,0.810
\.
Expand Down
12 changes: 6 additions & 6 deletions src/main/kotlin/com/flo/nem12/config/DatabaseConfig.kt
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ object DatabaseConfig {
const val DEFAULT_BATCH_SIZE = 50

const val CREATE_TABLE_SQL = """
CREATE TABLE IF NOT EXISTS meter_readings (
CREATE TABLE IF NOT EXISTS meter_reading (
id TEXT PRIMARY KEY,
nmi VARCHAR(10) NOT NULL,
timestamp TIMESTAMP NOT NULL,
Expand All @@ -20,17 +20,17 @@ object DatabaseConfig {
"""

const val INSERT_SQL = """
INSERT OR IGNORE INTO meter_readings (id, nmi, timestamp, consumption)
INSERT OR IGNORE INTO meter_reading (id, nmi, timestamp, consumption)
VALUES (?, ?, ?, ?)
"""

const val TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss"

const val CREATE_FAILED_READINGS_TABLE_SQL = """
CREATE TABLE IF NOT EXISTS failed_readings (
const val CREATE_FAILURE_READINGS_TABLE_SQL = """
CREATE TABLE IF NOT EXISTS failure_reading (
id TEXT PRIMARY KEY,
line_number INTEGER NOT NULL,
nmi TEXT,
nmi TEXT,
interval_index INTEGER,
raw_value TEXT NOT NULL,
failure_reason TEXT NOT NULL,
Expand All @@ -39,7 +39,7 @@ object DatabaseConfig {
"""

const val INSERT_FAILED_READING_SQL = """
INSERT INTO failed_readings (id, line_number, nmi, interval_index, raw_value, failure_reason, timestamp)
INSERT INTO failure_reading (id, line_number, nmi, interval_index, raw_value, failure_reason, timestamp)
VALUES (?, ?, ?, ?, ?, ?, ?)
"""
}
2 changes: 1 addition & 1 deletion src/main/kotlin/com/flo/nem12/model/MeterReading.kt
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import java.time.LocalDateTime

/**
* Represents a single meter reading record
* Corresponds to the meter_readings table in the database
* Corresponds to the meter_reading table in the database
*/
data class MeterReading(
val nmi: String,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ class FailureReadingsRepositoryImpl(
) : BaseSQLiteRepository<FailureRecord>(connection, batchSize), FailureReadingsRepository {
private val statistics = mutableMapOf<FailureReason, Int>()

override fun getCreateTableSql(): String = DatabaseConfig.CREATE_FAILED_READINGS_TABLE_SQL
override fun getCreateTableSql(): String = DatabaseConfig.CREATE_FAILURE_READINGS_TABLE_SQL

override fun getInsertSql(): String = DatabaseConfig.INSERT_FAILED_READING_SQL

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ class FailureReadingsRepositoryTest {
// Then
val resultSet =
connection.createStatement()
.executeQuery("SELECT COUNT(*) as cnt FROM failed_readings")
.executeQuery("SELECT COUNT(*) as cnt FROM failure_reading")
assertEquals(1, resultSet.getInt("cnt"))
connection.close()
}
Expand All @@ -73,7 +73,7 @@ class FailureReadingsRepositoryTest {
// Then
val resultSet =
connection.createStatement()
.executeQuery("SELECT COUNT(*) as cnt FROM failed_readings")
.executeQuery("SELECT COUNT(*) as cnt FROM failure_reading")
assertEquals(3, resultSet.getInt("cnt"))
connection.close()
}
Expand Down Expand Up @@ -104,7 +104,7 @@ class FailureReadingsRepositoryTest {
// Then
val resultSet =
connection.createStatement()
.executeQuery("SELECT COUNT(*) as cnt FROM failed_readings")
.executeQuery("SELECT COUNT(*) as cnt FROM failure_reading")
assertEquals(10, resultSet.getInt("cnt"))
connection.close()
}
Expand Down Expand Up @@ -159,7 +159,7 @@ class FailureReadingsRepositoryTest {
// Then
val resultSet =
connection.createStatement()
.executeQuery("SELECT DISTINCT failure_reason FROM failed_readings ORDER BY failure_reason")
.executeQuery("SELECT DISTINCT failure_reason FROM failure_reading ORDER BY failure_reason")
val reasons = mutableListOf<String>()
while (resultSet.next()) {
reasons.add(resultSet.getString("failure_reason"))
Expand Down Expand Up @@ -199,7 +199,7 @@ class FailureReadingsRepositoryTest {
// Then
val resultSet =
connection.createStatement()
.executeQuery("SELECT interval_index FROM failed_readings")
.executeQuery("SELECT interval_index FROM failure_reading")
assertTrue(resultSet.next())
assertEquals(0, resultSet.getInt("interval_index"))
assertTrue(resultSet.wasNull()) // Check that it's actually NULL
Expand Down Expand Up @@ -229,7 +229,7 @@ class FailureReadingsRepositoryTest {
// Then
val resultSet =
connection.createStatement()
.executeQuery("SELECT timestamp FROM failed_readings")
.executeQuery("SELECT timestamp FROM failure_reading")
assertTrue(resultSet.next())
assertEquals(null, resultSet.getString("timestamp"))
connection.close()
Expand Down Expand Up @@ -260,7 +260,7 @@ class FailureReadingsRepositoryTest {
// Then
val resultSet =
connection.createStatement()
.executeQuery("SELECT COUNT(*) as cnt FROM failed_readings")
.executeQuery("SELECT COUNT(*) as cnt FROM failure_reading")
assertEquals(5, resultSet.getInt("cnt"))
connection.close()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class MeterReadingRepositoryTest {
// Then
val resultSet =
connection.createStatement()
.executeQuery("SELECT COUNT(*) as cnt FROM meter_readings")
.executeQuery("SELECT COUNT(*) as cnt FROM meter_reading")
assertEquals(1, resultSet.getInt("cnt"))
connection.close()
}
Expand All @@ -70,7 +70,7 @@ class MeterReadingRepositoryTest {

val resultSet =
connection.createStatement()
.executeQuery("SELECT COUNT(*) as cnt FROM meter_readings")
.executeQuery("SELECT COUNT(*) as cnt FROM meter_reading")
assertEquals(3, resultSet.getInt("cnt"))
connection.close()
}
Expand Down Expand Up @@ -98,7 +98,7 @@ class MeterReadingRepositoryTest {
// Then
val resultSet =
connection.createStatement()
.executeQuery("SELECT COUNT(*) as cnt FROM meter_readings")
.executeQuery("SELECT COUNT(*) as cnt FROM meter_reading")
assertEquals(10, resultSet.getInt("cnt"))
connection.close()
}
Expand Down Expand Up @@ -130,12 +130,12 @@ class MeterReadingRepositoryTest {
// Then
val resultSet =
connection.createStatement()
.executeQuery("SELECT COUNT(*) as cnt FROM meter_readings")
.executeQuery("SELECT COUNT(*) as cnt FROM meter_reading")
assertEquals(1, resultSet.getInt("cnt"))

val dataResult =
connection.createStatement()
.executeQuery("SELECT consumption FROM meter_readings")
.executeQuery("SELECT consumption FROM meter_reading")
assertEquals("10.5", dataResult.getBigDecimal("consumption").toPlainString())
connection.close()
}
Expand All @@ -160,7 +160,7 @@ class MeterReadingRepositoryTest {
// Then
val resultSet =
connection.createStatement()
.executeQuery("SELECT DISTINCT nmi FROM meter_readings ORDER BY nmi")
.executeQuery("SELECT DISTINCT nmi FROM meter_reading ORDER BY nmi")
val nmis = mutableListOf<String>()
while (resultSet.next()) {
nmis.add(resultSet.getString("nmi"))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ class NEM12ParserServiceImplTest {
FailureReadingsRepository {
val savedFailures = mutableListOf<FailureRecord>()

override fun getCreateTableSql(): String = DatabaseConfig.CREATE_FAILED_READINGS_TABLE_SQL
override fun getCreateTableSql(): String = DatabaseConfig.CREATE_FAILURE_READINGS_TABLE_SQL

override fun getInsertSql(): String = DatabaseConfig.INSERT_FAILED_READING_SQL

Expand Down