Import from sqlite into postgres DB

This provides a pathway for us to import results from local machines into the
official perf server so they can be incorporated into blog posts and similar
material relatively easily.

Author: Mark-Simulacrum

collector/src/main.rs

@@ -236,7 +236,7 @@ fn bench(
         let mut tx = rt.block_on(conn.transaction());
         rt.block_on(
             tx.conn()
-                .record_benchmark(benchmark.name.0.as_str(), benchmark.supports_stable()),
+                .record_benchmark(benchmark.name.0.as_str(), Some(benchmark.supports_stable())),
         );
         eprintln!(
             "{}",

database/src/bin/import-sqlite.rs

@@ -0,0 +1,77 @@
+use database::{Lookup, Pool};
+use std::collections::HashSet;
+
+#[tokio::main]
+async fn main() {
+    env_logger::init();
+
+    let sqlite = std::env::args()
+        .nth(1)
+        .unwrap_or_else(|| panic!("needs sqlite database as 1st argument"));
+    let postgres = std::env::args()
+        .nth(2)
+        .unwrap_or_else(|| panic!("needs postgres database as 2nd argument"));
+
+    let sqlite = Pool::open(&sqlite);
+    let mut sqlite_conn = sqlite.connection().await;
+    let postgres = Pool::open(&postgres);
+    let postgres_conn = postgres.connection().await;
+
+    let sqlite_idx = sqlite_conn.load_index().await;
+
+    let cid_name = format!("imported-{}", chrono::Utc::now().timestamp());
+    println!("Collection ID for import is {}", cid_name);
+    let cid = postgres_conn.collection_id(&cid_name).await;
+
+    let mut benchmarks = HashSet::new();
+
+    // Starting after the sqlite and postgres db args, the rest are artifact
+    // names to import.
+    for artifact in std::env::args().skip(3) {
+        let aid = sqlite_conn
+            .artifact_by_name(&artifact)
+            .await
+            .unwrap_or_else(|| {
+                panic!("{} not found in sqlite db", artifact);
+            });
+
+        let sqlite_aid = sqlite_conn.artifact_id(&aid).await;
+        let postgres_aid = postgres_conn.artifact_id(&aid).await;
+
+        for &(krate, profile, cache, stat) in sqlite_idx.all_pstat_series() {
+            if benchmarks.insert(krate) {
+                postgres_conn.record_benchmark(krate.as_str(), None).await;
+            }
+
+            let id = database::DbLabel::ProcessStat {
+                krate,
+                profile,
+                cache,
+                stat,
+            }
+            .lookup(&sqlite_idx)
+            .unwrap();
+
+            let value = sqlite_conn
+                .get_pstats(&[id], &[Some(sqlite_aid)])
+                .await
+                .pop()
+                .unwrap()
+                .pop()
+                .unwrap();
+            if let Some(value) = value {
+                postgres_conn
+                    .record_statistic(
+                        cid,
+                        postgres_aid,
+                        &krate.to_string(),
+                        profile,
+                        cache,
+                        stat.as_str(),
+                        value,
+                    )
+                    .await;
+            }
+        }
+    }
+}

database/src/pool.rs

@@ -16,13 +16,17 @@ pub trait Connection: Send + Sync {
 
     async fn load_index(&mut self) -> Index;
 
+    async fn artifact_by_name(&self, artifact: &str) -> Option<ArtifactId>;
+
     /// This records the duration of a collection run, i.e., collecting all of
     /// the statistics for a particular artifact.
     async fn record_duration(&self, artifact: ArtifactIdNumber, duration: Duration);
 
     async fn collection_id(&self, version: &str) -> CollectionId;
     async fn artifact_id(&self, artifact: &ArtifactId) -> ArtifactIdNumber;
-    async fn record_benchmark(&self, krate: &str, supports_stable: bool);
+    /// None means that the caller doesn't know; it should be left alone if
+    /// known or set to false if unknown.
+    async fn record_benchmark(&self, krate: &str, supports_stable: Option<bool>);
     async fn record_statistic(
         &self,
         collection: CollectionId,

database/src/pool/postgres.rs

@@ -941,7 +941,7 @@ where
             .await
             .unwrap();
     }
-    async fn record_benchmark(&self, krate: &str, supports_stable: bool) {
+    async fn record_benchmark(&self, krate: &str, supports_stable: Option<bool>) {
         if let Some(r) = self
             .conn()
             .query_opt(
@@ -951,18 +951,29 @@ where
             .await
             .unwrap()
         {
-            if r.get::<_, bool>(0) == supports_stable {
+            if Some(r.get::<_, bool>(0)) == supports_stable {
                 return;
             }
         }
-        self.conn()
-            .execute(
-                "insert into benchmark (name, stabilized) VALUES ($1, $2)
+        if let Some(stable) = supports_stable {
+            self.conn()
+                .execute(
+                    "insert into benchmark (name, stabilized) VALUES ($1, $2)
                 ON CONFLICT (name) DO UPDATE SET stabilized = EXCLUDED.stabilized",
-                &[&krate, &supports_stable],
-            )
-            .await
-            .unwrap();
+                    &[&krate, &stable],
+                )
+                .await
+                .unwrap();
+        } else {
+            self.conn()
+                .execute(
+                    "insert into benchmark (name, stabilized) VALUES ($1, $2)
+                ON CONFLICT DO NOTHING",
+                    &[&krate, &false],
+                )
+                .await
+                .unwrap();
+        }
     }
 
     async fn collector_start(&self, aid: ArtifactIdNumber, steps: &[String]) {
@@ -1197,4 +1208,30 @@ where
 
         result
     }
+
+    async fn artifact_by_name(&self, artifact: &str) -> Option<ArtifactId> {
+        let row = self
+            .conn()
+            .query_opt(
+                "select date, type from artifact where name = $1",
+                &[&artifact],
+            )
+            .await
+            .unwrap()?;
+        let date = row.get::<_, Option<DateTime<Utc>>>(0);
+        let ty = row.get::<_, String>(1);
+
+        match ty.as_str() {
+            "master" => Some(ArtifactId::Commit(Commit {
+                sha: artifact.to_owned(),
+                date: Date(date.expect("date present for master commits")),
+            })),
+            "try" => Some(ArtifactId::Commit(Commit {
+                sha: artifact.to_owned(),
+                date: Date::ymd_hms(2000, 1, 1, 0, 0, 0),
+            })),
+            "release" => Some(ArtifactId::Artifact(artifact.to_owned())),
+            _ => panic!("unknown artifact type: {:?}", ty),
+        }
+    }
 }

database/src/pool/sqlite.rs

@@ -255,7 +255,7 @@ impl Connection for SqliteConnection {
             .unwrap()
             .query_map(params![], |row| {
                 Ok((
-                    row.get::<_, i16>(0)? as u32,
+                    row.get::<_, i32>(0)? as u32,
                     Commit {
                         sha: row.get::<_, String>(1)?.as_str().into(),
                         date: {
@@ -277,7 +277,7 @@ impl Connection for SqliteConnection {
             .unwrap()
             .query_map(params![], |row| {
                 Ok((
-                    row.get::<_, i16>(0)? as u32,
+                    row.get::<_, i32>(0)? as u32,
                     row.get::<_, String>(1)?.into_boxed_str(),
                 ))
             })
@@ -635,7 +635,7 @@ impl Connection for SqliteConnection {
                 .query_row(
                     "select id from artifact where name = $1",
                     params![&name],
-                    |r| r.get::<_, i16>(0),
+                    |r| r.get::<_, i32>(0),
                 )
                 .unwrap() as u32,
         )
@@ -713,14 +713,24 @@ impl Connection for SqliteConnection {
             )
             .unwrap();
     }
-    async fn record_benchmark(&self, krate: &str, supports_stable: bool) {
-        self.raw_ref()
-            .execute(
-                "insert into benchmark (name, stabilized) VALUES (?, ?)
+    async fn record_benchmark(&self, krate: &str, supports_stable: Option<bool>) {
+        if let Some(stable) = supports_stable {
+            self.raw_ref()
+                .execute(
+                    "insert into benchmark (name, stabilized) VALUES (?, ?)
                 ON CONFLICT (name) do update set stabilized = excluded.stabilized",
-                params![krate, supports_stable],
-            )
-            .unwrap();
+                    params![krate, stable],
+                )
+                .unwrap();
+        } else {
+            self.raw_ref()
+                .execute(
+                    "insert into benchmark (name, stabilized) VALUES (?, ?)
+                ON CONFLICT DO NOTHING",
+                    params![krate, false],
+                )
+                .unwrap();
+        }
     }
     async fn collector_start(&self, aid: ArtifactIdNumber, steps: &[String]) {
         // Clean out any leftover unterminated steps.
@@ -771,7 +781,7 @@ impl Connection for SqliteConnection {
             )
             .unwrap();
 
-        let aids = aids.query_map(params![], |r| r.get::<_, i16>(0)).unwrap();
+        let aids = aids.query_map(params![], |r| r.get::<_, i32>(0)).unwrap();
 
         let mut artifacts = Vec::new();
         for aid in aids {
@@ -932,4 +942,31 @@ impl Connection for SqliteConnection {
 
         results
     }
+
+    async fn artifact_by_name(&self, artifact: &str) -> Option<ArtifactId> {
+        let (date, ty) = self
+            .raw_ref()
+            .prepare("select date, type from artifact where name = ?")
+            .unwrap()
+            .query_row(params![&artifact], |r| {
+                let date = r.get::<_, Option<i64>>(0)?;
+                let ty = r.get::<_, String>(1)?;
+                Ok((date, ty))
+            })
+            .optional()
+            .unwrap()?;
+
+        match ty.as_str() {
+            "master" => Some(ArtifactId::Commit(Commit {
+                sha: artifact.to_owned(),
+                date: Date(Utc.timestamp(date.expect("master has date"), 0)),
+            })),
+            "try" => Some(ArtifactId::Commit(Commit {
+                sha: artifact.to_owned(),
+                date: Date::ymd_hms(2000, 1, 1, 0, 0, 0),
+            })),
+            "release" => Some(ArtifactId::Artifact(artifact.to_owned())),
+            _ => panic!("unknown artifact type: {:?}", ty),
+        }
+    }
 }

site/src/server.rs

@@ -101,7 +101,10 @@ pub async fn handle_dashboard(data: Arc<InputData>) -> ServerResult<dashboard::R
         return Ok(dashboard::Response::default());
     }
 
-    let mut versions = index.artifacts().collect::<Vec<_>>();
+    let mut versions = index
+        .artifacts()
+        .filter(|a| a.starts_with("1.") || a.starts_with("beta"))
+        .collect::<Vec<_>>();
     versions.sort_by(|a, b| {
         match (
             a.parse::<semver::Version>().ok(),