Ingest Debian Triage

.gitignore

@@ -20,3 +20,7 @@ bin/
 
 # Internally used sqlite DB
 data/internal.sqlite
+
+# test code coverage
+coverage.out
+coverage.html

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "submodules/cvelistV5"]
 	path = submodules/cvelistV5
 	url = https://github.com/CVEProject/cvelistV5.git
+[submodule "submodules/debian-security-tracker"]
+	path = submodules/debian-security-tracker
+	url = https://salsa.debian.org/security-tracker-team/security-tracker.git

.golangci.yaml

@@ -7,7 +7,10 @@ issues:
 
 linters:
   default: all
+  enable:
+  - gomodguard_v2
   disable:
+  - gomodguard
   - gocognit
   - gocyclo
   - cyclop
@@ -113,7 +116,7 @@ linters:
     tagliatelle:
       case:
         rules:
-          json: camel
+          json: snake
           yaml: camel
           xml: camel
           toml: camel
@@ -132,7 +135,7 @@ linters:
           - CVSSV2_0
     exhaustruct:
       include:
-        - "github.com/github.com/gardenlinux/glvd2"          
+        - "github.com/github.com/gardenlinux/glvd2"
 
 formatters:
   enable:

Makefile

@@ -8,7 +8,7 @@ build:
 
 .PHONY: format
 format:
-	gofmt -l -s -w .
+	golangci-lint fmt
 
 .PHONY: lint
 lint:
@@ -18,6 +18,11 @@ lint:
 test: clean_test
 	go test -v ./...
 
+.PHONY: test-coverage-html
+test-coverage-html: clean_test
+	go test --covermode=atomic --coverprofile=coverage.out -v ./...
+	go tool cover -html=coverage.out
+
 .PHONY: db_code_generate
 db_code_generate:
 	sqlc generate
@@ -41,4 +46,4 @@ clean_test:
 
 .PHONY: clean
 clean: clean_test
-	$(RM) $(BINARY)
+	$(RM) $(BINARY)

cmd/glvd2/main.go

@@ -12,6 +12,8 @@ import (
 	"github.com/gardenlinux/glvd2/internal/gardenlinux/packages"
 	"github.com/gardenlinux/glvd2/internal/git"
 	"github.com/gardenlinux/glvd2/internal/ingestion"
+	"github.com/gardenlinux/glvd2/internal/ingestion/debsectracker"
+	"github.com/gardenlinux/glvd2/internal/repository"
 	"github.com/spf13/cobra"
 	_ "modernc.org/sqlite"
 )
@@ -33,10 +35,8 @@ func setLogLevel(logLevelStr string) error {
 	return nil
 }
 
-func ingestCve(cfg *config.AppConfig) error {
-	var err error
-
-	db, err := database.Open()
+func ingestCVEs(cfg *config.AppConfig) error {
+	db, err := database.Regenerate(cfg.InternalSqliteDBPath)
 	if err != nil {
 		slog.Error("could not open database", slog.Any("error", err))
 		return err
@@ -53,9 +53,11 @@ func ingestCve(cfg *config.AppConfig) error {
 		return err
 	}
 
-	// TODO: clean DB and apply migrations
+	queries := repository.New(db)
+
 	ctx := context.Background()
 
+	// CVEListV5 and the Debian Security Tracker are currently added as git submodules
 	submoduleService, err := git.NewSubmoduleService(cfg)
 	if err != nil {
 		slog.Error("Could not initialize the submodule service", slog.Any("error", err))
@@ -67,6 +69,13 @@ func ingestCve(cfg *config.AppConfig) error {
 		return err
 	}
 
+	debSecTrackerIngestion := debsectracker.NewService(db, queries, cfg)
+	err = debSecTrackerIngestion.IngestTriage(ctx)
+	if err != nil {
+		slog.Error("Ingestion from Debian Security Tracker: %w", slog.Any("error", err))
+		return err
+	}
+
 	cveV5Ingestion := ingestion.NewCVEV5IngestionService(cfg)
 	resCh, errCh := cveV5Ingestion.ReceiveCVEs()
 	tmp := ""
@@ -116,7 +125,7 @@ func cmd(cfg *config.AppConfig) *cobra.Command {
 				return err
 			}
 
-			return ingestCve(cfg)
+			return ingestCVEs(cfg)
 		},
 	}
 	rootCmd.PersistentFlags().String("log-level", "error", "specify log-level")

config/default.yaml

@@ -1 +1,3 @@
+internal_sqlite_db_path: "./data/internal.sqlite"
 cve_list_v5_sub_repo_path: "./submodules/cvelistV5"
+deb_sec_tracker_sub_repo_path: "./submodules/debian-security-tracker"

go.mod

@@ -11,16 +11,17 @@ require (
 )
 
 require (
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/golang-migrate/migrate/v4 v4.19.1 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/ncruces/go-strftime v1.0.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	github.com/sagikazarmark/locafero v0.11.0 // indirect

go.sum

@@ -1,6 +1,8 @@
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@@ -9,6 +11,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA=
+github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
@@ -34,6 +38,8 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0
 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=

internal/config/config.go

@@ -8,7 +8,9 @@ import (
 
 // AppConfig contains different configuration options for the application.
 type AppConfig struct {
-	CVEListV5SubRepoPath string `mapstructure:"cve_list_v5_sub_repo_path"`
+	CVEListV5SubRepoPath     string `mapstructure:"cve_list_v5_sub_repo_path"`
+	DebSecTrackerSubRepoPath string `mapstructure:"deb_sec_tracker_sub_repo_path"`
+	InternalSqliteDBPath     string `mapstructure:"internal_sqlite_db_path"`
 }
 
 // LoadAppConfig loads the configuration from disk.

internal/db/database.go

@@ -2,12 +2,47 @@ package db
 
 import (
 	"database/sql"
+	"os"
+
+	"github.com/golang-migrate/migrate/v4"
+	_ "github.com/golang-migrate/migrate/v4/database/sqlite" // blank import like lib proposes
+	_ "github.com/golang-migrate/migrate/v4/source/file"
 )
 
+const sqliteConnectionStringSuffix = "?journal_mode=WAL&busy_timeout=3000&secure_delete=true" +
+	"&foreign_keys=true&cache=shared&x-no-tx-wrap=true"
+
+// Regenerate clears the DB file, recreates the structure via migration, and returns a DB connection.
+func Regenerate(fp string) (*sql.DB, error) {
+	// ensure that the file exists
+	f, err := os.OpenFile(fp, os.O_CREATE, 0o644) //nolint:gosec,mnd // no user input and fil
+	if err != nil {
+		return nil, err
+	}
+	if err = f.Close(); err != nil {
+		return nil, err
+	}
+
+	// clear the file content
+	if err = os.Truncate(fp, 0); err != nil {
+		return nil, err
+	}
+
+	if err = Migrate(fp); err != nil {
+		return nil, err
+	}
+
+	db, err := Open(fp)
+	if err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}
+
 // Open a sqlite db from file.
-func Open() (*sql.DB, error) {
-	db, err := sql.Open("sqlite",
-		"data/internal.sqlite?journal_mode=WAL&busy_timeout=3000&secure_delete=true&foreign_keys=true&cache=shared")
+func Open(fp string) (*sql.DB, error) {
+	db, err := sql.Open("sqlite", fp+sqliteConnectionStringSuffix)
 	if err != nil {
 		return nil, err
 	}
@@ -17,3 +52,19 @@ func Open() (*sql.DB, error) {
 
 	return db, nil
 }
+
+func Migrate(fp string) error {
+	m, err := migrate.New(
+		"file://internal/db/migrations/",
+		"sqlite://"+fp+sqliteConnectionStringSuffix)
+	if err != nil {
+		return err
+	}
+
+	err = m.Up()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

internal/db/migrations/000001_init.down.sql

@@ -5,4 +5,10 @@ DROP TABLE IF EXISTS product;
 DROP TABLE IF EXISTS affect;
 DROP TABLE IF EXISTS cvss_metric;
 
+DROP TABLE IF EXISTS debian_triage;
+DROP TABLE IF EXISTS debian_package;
+DROP TABLE IF EXISTS debian_triage_affected_package;
+DROP TABLE IF EXISTS debian_release;
+DROP TABLE IF EXISTS debian_triage_affected_release;
+
 END;

internal/db/migrations/000001_init.up.sql

@@ -4,8 +4,8 @@ CREATE TABLE IF NOT EXISTS cve(
     id TEXT PRIMARY KEY,
     title TEXT,
     description TEXT,
-    serial INT NOT NULL DEFAULT 1,
-    is_rejected INT NOT NULL DEFAULT 0,
+    serial INTEGER NOT NULL DEFAULT 1,
+    is_rejected INTEGER NOT NULL DEFAULT 0,
     assigner_org_id TEXT,
     assigner_short_name TEXT,
     date_reserved TEXT,
@@ -16,20 +16,21 @@ CREATE TABLE IF NOT EXISTS cve(
 );
 
 CREATE TABLE IF NOT EXISTS product(
-    id INT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
     title TEXT,
     cpe TEXT
 );
 
 CREATE TABLE IF NOT EXISTS affect(
-    id INT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
     cve_id TEXT,
+    product_id INTEGER,
     FOREIGN KEY(cve_id) REFERENCES cve(id),
     FOREIGN KEY(product_id) REFERENCES product(id)
 );
 
 CREATE TABLE IF NOT EXISTS cvss_metric(
-    id INT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
     cve_id TEXT,
     issuer TEXT,
     version TEXT,
@@ -39,4 +40,41 @@ CREATE TABLE IF NOT EXISTS cvss_metric(
     FOREIGN KEY(cve_id) REFERENCES cve(id)
 );
 
+CREATE TABLE IF NOT EXISTS debian_triage(
+    cve_id TEXT PRIMARY KEY,
+    status TEXT CHECK( status IN ('UNKNOWN', 'TODO', 'REJECTED', 'NOT-FOR-US', 'PROCESSED', 'RESERVED') ) NOT NULL,
+    not_for_us TEXT NOT NULL,
+    notes TEXT NOT NULL,
+    to_dos TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS debian_package(
+    name TEXT PRIMARY KEY
+);
+
+CREATE TABLE IF NOT EXISTS debian_triage_affected_package(
+    id INTEGER PRIMARY KEY,
+    cve_id TEXT NOT NULL,
+    package_name TEXT NOT NULL,
+    version TEXT NOT NULL,
+    info TEXT,
+    FOREIGN KEY(cve_id) REFERENCES debian_triage(cve_id),
+    FOREIGN KEY(package_name) REFERENCES debian_package(name)
+);
+
+CREATE TABLE IF NOT EXISTS debian_release(
+    name TEXT PRIMARY KEY
+);
+
+CREATE TABLE IF NOT EXISTS debian_triage_affected_release(
+    id INTEGER PRIMARY KEY,
+    cve_id TEXT NOT NULL,
+    release_name TEXT NOT NULL,
+    package_name TEXT NOT NULL,
+    action TEXT NOT NULL,
+    info TEXT,
+    FOREIGN KEY(cve_id) REFERENCES debian_triage(cve_id),
+    FOREIGN KEY(release_name) REFERENCES debian_release(name)
+);
+
 END;

internal/db/queries/cve.sql

@@ -1,2 +1,2 @@
--- name: ListCves :many
-SELECT sqlc.embed(c) FROM cve As c ORDER BY c.name DESC;
+-- name: ListCVEs :many
+SELECT * FROM cve ORDER BY name DESC;

internal/db/queries/debiantriage.sql

@@ -0,0 +1,26 @@
+-- name: GetDebianTriage :one
+SELECT * FROM debian_triage WHERE cve_id = ? LIMIT 1;
+
+-- name: ListDebianTriages :many
+SELECT * FROM debian_triage ORDER BY cve_id DESC;
+
+-- name: ListAffectedPackagesForDebianTriage :many
+SELECT * FROM debian_triage_affected_package WHERE cve_id = ? ORDER BY id ASC;
+
+-- name: ListAffectedReleasesForDebianTriage :many
+SELECT * FROM debian_triage_affected_release WHERE cve_id = ? ORDER BY id ASC;
+
+-- name: InsertDebianPackageOrIgnore :one
+INSERT OR IGNORE INTO debian_package(name) VALUES (?) RETURNING *;
+
+-- name: InsertDebianReleaseOrIgnore :one
+INSERT OR IGNORE INTO debian_release(name) VALUES (?) RETURNING *;
+
+-- name: InsertDebianTriage :one
+INSERT INTO debian_triage(cve_id, status, not_for_us, notes, to_dos) VALUES (?, ?, ?, ?, ?) RETURNING *;
+
+-- name: InsertDebianTriageAffectedPackage :one
+INSERT INTO debian_triage_affected_package(cve_id, package_name, version, info) VALUES (?, ?, ?, ?) RETURNING *;
+
+-- name: InsertDebianTriageAffectedRelease :one
+INSERT INTO debian_triage_affected_release(cve_id, release_name, package_name, action, info) VALUES (?, ?, ?, ?, ?) RETURNING *;