Merge pull request #1005 from tgnottingham/sqlite-postgres-converters

Add database conversion scripts

Author: Mark-Simulacrum

.github/workflows/ci.yml

@@ -131,8 +131,8 @@ jobs:
       - name: Check benchmarks
         run: sh -x -c "ci/check-profiling.sh"
 
-  database-check-postgres:
-    name: Database Check (postgres)
+  database-check:
+    name: Database Check
     runs-on: ubuntu-latest
     services:
       postgres:
@@ -156,27 +156,48 @@ jobs:
         with:
           toolchain: stable
 
+      - name: Build site
+        run: cargo build --bin site
+
       - name: Connect to PostgreSQL
         run: |
-          cargo build --bin site
           timeout 5s ./target/debug/site 'postgresql://postgres:postgres@localhost:5432/postgres' 2>&1 | tee -a log || true
           grep -Fxq "Loading complete but no data identified; exiting." log
 
-  database-check-sqlite:
-    name: Database Check (sqlite)
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Check out repository code
-        uses: actions/checkout@v2
-
-      - name: Install stable
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-
       - name: Connect to SQLite
         run: |
-          cargo build --bin site
           timeout 5s ./target/debug/site 2>&1 | tee -a log || true
           grep -Fxq "Loading complete but no data identified; exiting." log
+
+      - name: Build Postgres to SQLite exporter
+        run: cargo build --bin postgres-to-sqlite
+
+      - name: Export empty Postgres DB to SQLite
+        run: ./target/debug/postgres-to-sqlite 'postgresql://postgres:postgres@localhost:5432/postgres' "$(mktemp)"
+
+      - name: Export empty Postgres DB to SQLite with date filter
+        run: ./target/debug/postgres-to-sqlite --since-weeks-ago=1 'postgresql://postgres:postgres@localhost:5432/postgres' "$(mktemp)"
+
+      - name: Build SQLite to Postgres exporter
+        run: cargo build --bin sqlite-to-postgres
+
+      - name: Export empty SQLite DB to Postgres
+        run: ./target/debug/sqlite-to-postgres "$(mktemp)" 'postgresql://postgres:postgres@localhost:5432/postgres'
+
+      - name: Roundtrip export
+        run: |
+            sudo apt-get update
+            sudo apt-get install -y wget python3-snappy sqlite3
+
+            wget -q 'http://perf-data.rust-lang.org/export.db.sz'
+            python3 -m snappy -d export.db.sz > before.db
+
+            ./target/debug/sqlite-to-postgres before.db 'postgresql://postgres:postgres@localhost:5432/postgres'
+            sqlite3 before.db .dump > before.dump
+
+            rm before.db
+
+            ./target/debug/postgres-to-sqlite 'postgresql://postgres:postgres@localhost:5432/postgres' after.db
+            sqlite3 after.db .dump > after.dump
+
+            diff -w before.dump after.dump

Cargo.lock

@@ -24,15 +24,15 @@ COPY ./database ./database
 COPY ./intern ./intern
 
 RUN bash -c 'source $HOME/.cargo/env && cargo build --release -p site'
-RUN bash -c 'source $HOME/.cargo/env && cargo build --release --bin export-to-sqlite'
+RUN bash -c 'source $HOME/.cargo/env && cargo build --release --bin postgres-to-sqlite'
 
 FROM ubuntu:20.04 as binary
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     ca-certificates \
     git
 
-COPY --from=build /target/release/export-to-sqlite /usr/local/bin/rustc-perf-export-to-sqlite
+COPY --from=build /target/release/postgres-to-sqlite /usr/local/bin/rustc-perf-postgres-to-sqlite
 COPY --from=build /target/release/site /usr/local/bin/rustc-perf-site
 COPY --from=build site/static /site/static
 

Dockerfile

@@ -15,11 +15,14 @@ async-trait = "0.1"
 tokio = { version = "1.6", features = ["sync", "macros"] }
 snap = "1"
 intern = { path = "../intern" }
-chrono = "0.4"
+chrono = { version = "0.4", features = ["serde"] }
 reqwest = { version = "0.11", features = ["blocking"] }
 postgres-native-tls = "0.5"
 native-tls = "0.2"
 lazy_static = "1"
 env_logger = "0.8"
 futures = "0.3.5"
 log = "0.4"
+bytes = "1"
+csv = "1"
+clap = "2.25"