feat: scrape UI

.gitignore

@@ -58,4 +58,3 @@ screenshots/
 *.har
 *.png
 .playwright-mcp/
-*.json

Makefile

@@ -45,6 +45,10 @@ tidy:
 	go mod tidy
 	git add go.mod go.sum
 
+.PHONY: scrapeui-build
+scrapeui-build:
+	cd cmd/scrapeui/frontend && npm ci && npm run build
+
 # Generate OpenAPI schema
 .PHONY: gen-schemas
 gen-schemas:
@@ -77,8 +81,16 @@ manifests: generate gen-schemas ## Generate WebhookConfiguration, ClusterRole an
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./api/..."
 
+.PHONY: scrapeui-fallback-assets
+scrapeui-fallback-assets:
+	@mkdir -p cmd/scrapeui/frontend/dist
+	@test -f cmd/scrapeui/frontend/dist/scrapeui.js || \
+		echo "console.warn('scrapeui fallback bundle: run make scrapeui-build for full UI');" > cmd/scrapeui/frontend/dist/scrapeui.js
+	@test -f cmd/scrapeui/frontend/dist/scrapeui.css || \
+		echo "/* scrapeui fallback styles: run make scrapeui-build for full UI */" > cmd/scrapeui/frontend/dist/scrapeui.css
+
 .PHONY: resources
-resources: fmt manifests
+resources: scrapeui-fallback-assets fmt manifests
 
 test: manifests generate fmt vet envtest  ## Run tests.
 	$(MAKE) gotest
@@ -116,7 +128,6 @@ test-fast: ginkgo
 		ginkgo --tags slim --nodes=4   --label-filter "!slow" -r -v --skip-package=tests/e2e  ./...
 
 
-
 .PHONY: gotest-prod
 gotest-prod:
 	$(validate-envtest-assets) \
@@ -215,7 +226,7 @@ uninstall-crd: manifests
 
 # produce a build that's debuggable
 .PHONY: dev
-dev:
+dev: scrapeui-build
 	go build -o ./.bin/$(NAME) -v -gcflags="all=-N -l" main.go
 
 .PHONY: watch
@@ -302,10 +313,10 @@ rust-generate-header:
 
 .PHONY: bench
 bench:
-	go test ./scrapers/kubernetes/ -bench='^Benchmark(EventProcessing|CacheMemory|Deserialization)' \
+	go test ./... -bench='^Benchmark(EventProcessing|CacheMemory|Deserialization)' \
 		-benchmem -run='^$$' \
 		-count=3 \
-		-benchtime=2s -v
+		-benchtime=2s -v $(BENCH_ARGS)
 
 .PHONY: modernize
 modernize:

build/Dockerfile

@@ -8,6 +8,13 @@ COPY Makefile /app
 COPY external/diffgen /app/external/diffgen
 RUN make rust-diffgen
 
+FROM node:20-bookworm AS scrapeui-builder
+WORKDIR /app/cmd/scrapeui/frontend
+COPY cmd/scrapeui/frontend/package.json cmd/scrapeui/frontend/package-lock.json ./
+RUN npm ci
+COPY cmd/scrapeui/frontend ./
+RUN npm run build
+
 FROM golang:1.26-bookworm AS builder-base
 WORKDIR /app
 
@@ -22,6 +29,7 @@ FROM builder-base AS builder
 COPY ./ ./
 
 COPY --from=rust-builder /app/external/diffgen/target ./external/diffgen/target
+COPY --from=scrapeui-builder /app/cmd/scrapeui/frontend/dist ./cmd/scrapeui/frontend/dist
 
 RUN make build-prod
 

cmd/root.go

@@ -151,5 +151,5 @@ func init() {
 		},
 	})
 
-	Root.AddCommand(Run, Analyze, Serve, GoOffline, Operator)
+	Root.AddCommand(Run, Analyze, Serve, GoOffline, Operator, UI)
 }

cmd/run.go

@@ -1,17 +1,21 @@
 package cmd
 
 import (
-	"bytes"
 	gocontext "context"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"os"
+	"os/exec"
+	"os/signal"
 	"path"
+	"runtime"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/flanksource/clicky"
@@ -22,6 +26,7 @@ import (
 	"github.com/flanksource/commons/timer"
 	"github.com/flanksource/config-db/api"
 	v1 "github.com/flanksource/config-db/api/v1"
+	"github.com/flanksource/config-db/cmd/scrapeui"
 	"github.com/flanksource/config-db/db"
 	"github.com/flanksource/config-db/scrapers"
 	"github.com/flanksource/duty"
@@ -44,13 +49,15 @@ var outputDir string
 var debugPort int
 var export bool
 var save bool
+var uiEnabled bool
+var uiPort int
 
 // Run ...
 var Run = &cobra.Command{
 	Use:   "run <scraper.yaml>",
 	Short: "Run scrapers and return",
 	Run: func(cmd *cobra.Command, configFiles []string) {
-		var logBuf bytes.Buffer
+		var logBuf scrapeui.SafeBuffer
 		var harCollector *har.Collector
 
 		if logger.IsTraceEnabled() {
@@ -130,11 +137,30 @@ var Run = &cobra.Command{
 			}
 		}
 
+		var uiServer *scrapeui.Server
+		if uiEnabled {
+			names := make([]string, len(scraperConfigs))
+			specs := make([]any, len(scraperConfigs))
+			for i, sc := range scraperConfigs {
+				names[i] = sc.Name
+				specs[i] = sc.Spec
+			}
+			var scrapeSpec any = specs
+			if len(specs) == 1 {
+				scrapeSpec = specs[0]
+			}
+			uiServer = startScrapeUI(names, scrapeSpec, &logBuf)
+		}
+
 		var hasErrors bool
 		var allResults v1.ScrapeResults
 		var lastSummary *v1.ScrapeSummary
 		var lastSnapshotPair *v1.ScrapeSnapshotPair
 		for i := range scraperConfigs {
+			if uiServer != nil {
+				uiServer.UpdateScraper(scraperConfigs[i].Name, scrapeui.ScraperRunning, nil, nil, nil)
+			}
+
 			scrapeCtx, cancel, cancelTimeout := api.NewScrapeContext(dutyCtx).WithScrapeConfig(&scraperConfigs[i]).
 				WithTimeout(dutyCtx.Properties().Duration("scraper.timeout", 4*time.Hour))
 			defer cancelTimeout()
@@ -153,6 +179,14 @@ var Run = &cobra.Command{
 			if err != nil {
 				hasErrors = true
 				logger.Errorf("error scraping config: (name=%s) %+v", scraperConfigs[i].Name, err)
+				if uiServer != nil {
+					uiServer.UpdateScraper(scraperConfigs[i].Name, scrapeui.ScraperError, results, summary, err)
+				}
+			} else if uiServer != nil {
+				uiServer.UpdateScraper(scraperConfigs[i].Name, scrapeui.ScraperComplete, results, summary, nil)
+			}
+			if uiServer != nil && snapshotPair != nil {
+				uiServer.SetSnapshots(scraperConfigs[i].Name, snapshotPair)
 			}
 
 			scraperUID := string(scraperConfigs[i].GetUID())
@@ -180,6 +214,10 @@ var Run = &cobra.Command{
 				}
 			}
 
+			if uiServer != nil && summary != nil {
+				uiServer.SetLastScrapeSummary(*summary)
+			}
+
 			allResults = append(allResults, results...)
 			if summary != nil {
 				lastSummary = summary
@@ -192,7 +230,50 @@ var Run = &cobra.Command{
 		// Restore stderr-only logging before rendering
 		logger.Use(os.Stderr)
 
-		printOutput(allResults, lastSummary, lastSnapshotPair, harCollector, logBuf.String())
+		if uiServer != nil {
+			if harCollector != nil {
+				uiServer.SetHAR(harCollector.Entries())
+			}
+
+			props := make(map[string]scrapeui.PropertyInfo)
+			for k, v := range dutyCtx.Properties().SupportedProperties() {
+				props[k] = scrapeui.PropertyInfo{
+					Value:   v.Value,
+					Default: v.Default,
+					Type:    v.Type,
+				}
+			}
+			scraperLogLevel := ""
+			for _, sc := range scraperConfigs {
+				if sc.Spec.LogLevel != "" {
+					scraperLogLevel = sc.Spec.LogLevel
+					break
+				}
+			}
+			globalLevel := "info"
+			if logger.IsTraceEnabled() {
+				globalLevel = "trace"
+			}
+			uiServer.SetProperties(props, scrapeui.LogLevelInfo{
+				Scraper: scraperLogLevel,
+				Global:  globalLevel,
+			})
+
+			uiServer.SetDone()
+		}
+
+		if uiServer == nil {
+			printOutput(allResults, lastSummary, lastSnapshotPair, harCollector, logBuf.String())
+		} else {
+			sig := make(chan os.Signal, 1)
+			signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
+			<-sig
+			shutdown.Shutdown()
+			if hasErrors {
+				os.Exit(1)
+			}
+			return
+		}
 
 		if hasErrors {
 			os.Exit(1)
@@ -554,11 +635,61 @@ func ensureScraper(ctx context.Context, sc *v1.ScrapeConfig) error {
 	return nil
 }
 
+func startScrapeUI(scraperNames []string, scrapeSpec any, logBuf *scrapeui.SafeBuffer) *scrapeui.Server {
+	srv := scrapeui.NewServer(scraperNames, scrapeSpec, logBuf)
+	bi := GetBuildInfo()
+	srv.SetBuildInfo(scrapeui.BuildInfo{Version: bi.Version, Commit: bi.Commit, Date: bi.Date})
+	addr := fmt.Sprintf("localhost:%d", uiPort)
+	listener, err := net.Listen("tcp", addr)
+	if err != nil && uiPort != 0 {
+		logger.Warnf("Port %d in use, picking a free port", uiPort)
+		listener, err = net.Listen("tcp", "localhost:0")
+	}
+	if err != nil {
+		logger.Errorf("Failed to start scrape UI server: %v", err)
+		return nil
+	}
+	port := listener.Addr().(*net.TCPAddr).Port
+	url := fmt.Sprintf("http://localhost:%d", port)
+
+	httpServer := &http.Server{Handler: srv.Handler()}
+	shutdown.AddHook(func() {
+		ctx, cancel := gocontext.WithTimeout(gocontext.Background(), 5*time.Second)
+		defer cancel()
+		_ = httpServer.Shutdown(ctx)
+		_ = listener.Close()
+	})
+
+	go httpServer.Serve(listener) //nolint:errcheck
+
+	time.Sleep(100 * time.Millisecond)
+	logger.Infof("Scrape UI at %s", url)
+	openBrowser(url)
+	return srv
+}
+
+func openBrowser(url string) {
+	var cmd string
+	var args []string
+	switch runtime.GOOS {
+	case "windows":
+		cmd = "cmd"
+		args = []string{"/c", "start"}
+	case "darwin":
+		cmd = "open"
+	default:
+		cmd = "xdg-open"
+	}
+	args = append(args, url)
+	_ = exec.Command(cmd, args...).Start()
+}
+
 func init() {
 	Run.Flags().BoolVar(&save, "save", false, "Save scraped configurations to the database")
 	Run.Flags().BoolVar(&export, "export", true, "Export scraped configurations to files in the output directory and/or pretty print them")
 	Run.Flags().StringVarP(&outputDir, "output-dir", "o", "", "The output folder for configurations")
 	Run.Flags().IntVar(&debugPort, "debug-port", -1, "Start an HTTP server to use the /debug routes, Use -1 to disable and 0 to pick a free port")
+	Run.Flags().BoolVar(&uiEnabled, "ui", false, "Open a browser dashboard showing real-time scrape progress")
+	Run.Flags().IntVar(&uiPort, "ui-port", 9001, "Port for the UI server (0 to pick a free port)")
 	clicky.BindAllFlags(Run.Flags())
-
 }

cmd/scrapeui/assets.go

@@ -0,0 +1,9 @@
+package scrapeui
+
+import _ "embed"
+
+//go:embed frontend/dist/scrapeui.js
+var bundleJS string
+
+//go:embed frontend/dist/scrapeui.css
+var bundleCSS string