Release 🚀

.gitignore

@@ -2,3 +2,4 @@ diode/
 orb-configs/
 build/
 .coverage/
+.vscode/

agent/backend/devicediscovery/device_discovery.go

@@ -39,11 +39,13 @@ type deviceDiscoveryBackend struct {
 	apiPort     string
 	apiProtocol string
 
-	diodeTarget        string
-	diodeClientID      string
-	diodeClientSecret  string
-	diodeAppNamePrefix string
-	diodeOtelEndpoint  string
+	diodeTarget          string
+	diodeClientID        string
+	diodeClientSecret    string
+	diodeAppNamePrefix   string
+	diodeOtelEndpoint    string
+	diodeDryRun          bool
+	diodeDryRunOutputDir string
 
 	startTime  time.Time
 	proc       backend.Commander
@@ -84,6 +86,27 @@ func (d *deviceDiscoveryBackend) Configure(logger *slog.Logger, repo policies.Po
 	d.diodeClientID = common.Diode.ClientID
 	d.diodeClientSecret = common.Diode.ClientSecret
 	d.diodeAppNamePrefix = common.Diode.AgentName
+	d.diodeDryRun = common.Diode.DryRun
+	d.diodeDryRunOutputDir = common.Diode.DryRunOutputDir
+
+	if target, prs := config["target"].(string); prs {
+		d.diodeTarget = target
+	}
+	if clientID, prs := config["client_id"].(string); prs {
+		d.diodeClientID = clientID
+	}
+	if clientSecret, prs := config["client_secret"].(string); prs {
+		d.diodeClientSecret = clientSecret
+	}
+	if agentName, prs := config["agent_name"].(string); prs {
+		d.diodeAppNamePrefix = agentName
+	}
+	if dryRun, prs := config["dry_run"].(bool); prs {
+		d.diodeDryRun = dryRun
+	}
+	if dryRunOutputDir, prs := config["dry_run_output_dir"].(string); prs {
+		d.diodeDryRunOutputDir = dryRunOutputDir
+	}
 
 	if common.Otel.Grpc != "" {
 		d.diodeOtelEndpoint = common.Otel.Grpc
@@ -110,13 +133,22 @@ func (d *deviceDiscoveryBackend) Start(ctx context.Context, cancelFunc context.C
 	d.cancelFunc = cancelFunc
 	d.ctx = ctx
 
-	pvOptions := []string{
-		"--host", d.apiHost,
-		"--port", d.apiPort,
-		"--diode-target", d.diodeTarget,
-		"--diode-client-id", d.diodeClientID,
-		"--diode-client-secret", "********",
-		"--diode-app-name-prefix", d.diodeAppNamePrefix,
+	var pvOptions []string
+	if d.diodeDryRun {
+		pvOptions = []string{
+			"--dry-run",
+			"--dry-run-output-dir", d.diodeDryRunOutputDir,
+			"--diode-app-name-prefix", d.diodeAppNamePrefix,
+		}
+	} else {
+		pvOptions = []string{
+			"--host", d.apiHost,
+			"--port", d.apiPort,
+			"--diode-target", d.diodeTarget,
+			"--diode-client-id", d.diodeClientID,
+			"--diode-client-secret", "********",
+			"--diode-app-name-prefix", d.diodeAppNamePrefix,
+		}
 	}
 
 	if d.diodeOtelEndpoint != "" {
@@ -125,7 +157,9 @@ func (d *deviceDiscoveryBackend) Start(ctx context.Context, cancelFunc context.C
 
 	d.logger.Info("device-discovery startup", slog.Any("arguments", pvOptions))
 
-	pvOptions[9] = d.diodeClientSecret
+	if !d.diodeDryRun && len(pvOptions) > 9 {
+		pvOptions[9] = d.diodeClientSecret
+	}
 
 	d.proc = backend.NewCmdOptions(backend.CmdOptions{
 		Buffered:  false,

agent/backend/devicediscovery/device_discovery_test.go

@@ -201,3 +201,85 @@ func TestDeviceDiscoveryBackendCompleted(t *testing.T) {
 
 	assert.Error(t, err)
 }
+
+func TestDeviceDiscoveryBackendDryRun(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		switch {
+		case r.URL.Path == "/api/v1/status":
+			response := StatusResponse{Version: "1.3.5", StartTime: "2023-10-01T12:00:00Z", UpTime: 123.456}
+			_ = json.NewEncoder(w).Encode(response)
+		case r.URL.Path == "/api/v1/capabilities":
+			capabilities := map[string]any{"capability": true}
+			_ = json.NewEncoder(w).Encode(capabilities)
+		case strings.HasPrefix(r.URL.Path, "/api/v1/policies"):
+			_ = json.NewEncoder(w).Encode(map[string]any{"status": "ok"})
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer server.Close()
+
+	serverURL, err := url.Parse(server.URL)
+	assert.NoError(t, err)
+
+	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
+	repo, err := policies.NewMemRepo()
+	assert.NoError(t, err)
+
+	mockCmd := &mocks.MockCmd{}
+	mocks.SetupSuccessfulProcess(mockCmd, 12345)
+
+	originalNewCmdOptions := backend.NewCmdOptions
+	defer func() {
+		backend.NewCmdOptions = originalNewCmdOptions
+	}()
+
+	backend.NewCmdOptions = func(options backend.CmdOptions, name string, args ...string) backend.Commander {
+		assert.Equal(t, "device-discovery", name, "Expected command name to be device-discovery")
+		assert.Contains(t, args, "--dry-run")
+		assert.Contains(t, args, "--dry-run-output-dir")
+		assert.NotContains(t, args, "--host")
+		assert.NotContains(t, args, "--port")
+		assert.False(t, options.Buffered, "Expected buffered to be false")
+		assert.True(t, options.Streaming, "Expected streaming to be true")
+		return mockCmd
+	}
+
+	assert.True(t, devicediscovery.Register())
+	be := backend.GetBackend("device_discovery")
+
+	beCommons := config.BackendCommons{
+		Diode: struct {
+			Target          string `yaml:"target"`
+			ClientID        string `yaml:"client_id"`
+			ClientSecret    string `yaml:"client_secret"`
+			AgentName       string `yaml:"agent_name"`
+			DryRun          bool   `yaml:"dry_run"`
+			DryRunOutputDir string `yaml:"dry_run_output_dir"`
+		}{
+			DryRun:          true,
+			DryRunOutputDir: "/tmp/dry-run-output",
+		},
+	}
+
+	err = be.Configure(logger, repo, map[string]any{
+		"host": serverURL.Hostname(),
+		"port": serverURL.Port(),
+	}, beCommons)
+	assert.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	err = be.Start(ctx, cancel)
+	assert.NoError(t, err)
+
+	assert.False(t, be.GetStartTime().IsZero())
+
+	err = be.RemovePolicy(policies.PolicyData{ID: "1", Name: "policy", Data: map[string]any{"k": "v"}})
+	assert.NoError(t, err)
+
+	err = be.Stop(context.WithValue(context.Background(), config.ContextKey("routine"), "test"))
+	assert.NoError(t, err)
+
+	mockCmd.AssertExpectations(t)
+}

agent/backend/worker/worker.go

@@ -39,11 +39,13 @@ type workerBackend struct {
 	apiPort     string
 	apiProtocol string
 
-	diodeTarget        string
-	diodeClientID      string
-	diodeClientSecret  string
-	diodeAppNamePrefix string
-	diodeOtelEndpoint  string
+	diodeTarget          string
+	diodeClientID        string
+	diodeClientSecret    string
+	diodeAppNamePrefix   string
+	diodeOtelEndpoint    string
+	diodeDryRun          bool
+	diodeDryRunOutputDir string
 
 	startTime  time.Time
 	proc       backend.Commander
@@ -84,6 +86,27 @@ func (d *workerBackend) Configure(logger *slog.Logger, repo policies.PolicyRepo,
 	d.diodeClientID = common.Diode.ClientID
 	d.diodeClientSecret = common.Diode.ClientSecret
 	d.diodeAppNamePrefix = common.Diode.AgentName
+	d.diodeDryRun = common.Diode.DryRun
+	d.diodeDryRunOutputDir = common.Diode.DryRunOutputDir
+
+	if target, prs := config["target"].(string); prs {
+		d.diodeTarget = target
+	}
+	if clientID, prs := config["client_id"].(string); prs {
+		d.diodeClientID = clientID
+	}
+	if clientSecret, prs := config["client_secret"].(string); prs {
+		d.diodeClientSecret = clientSecret
+	}
+	if agentName, prs := config["agent_name"].(string); prs {
+		d.diodeAppNamePrefix = agentName
+	}
+	if dryRun, prs := config["dry_run"].(bool); prs {
+		d.diodeDryRun = dryRun
+	}
+	if dryRunOutputDir, prs := config["dry_run_output_dir"].(string); prs {
+		d.diodeDryRunOutputDir = dryRunOutputDir
+	}
 
 	if common.Otel.Grpc != "" {
 		d.diodeOtelEndpoint = common.Otel.Grpc
@@ -110,13 +133,22 @@ func (d *workerBackend) Start(ctx context.Context, cancelFunc context.CancelFunc
 	d.cancelFunc = cancelFunc
 	d.ctx = ctx
 
-	pvOptions := []string{
-		"--host", d.apiHost,
-		"--port", d.apiPort,
-		"--diode-target", d.diodeTarget,
-		"--diode-client-id", d.diodeClientID,
-		"--diode-client-secret", "********",
-		"--diode-app-name-prefix", d.diodeAppNamePrefix,
+	var pvOptions []string
+	if d.diodeDryRun {
+		pvOptions = []string{
+			"--dry-run",
+			"--dry-run-output-dir", d.diodeDryRunOutputDir,
+			"--diode-app-name-prefix", d.diodeAppNamePrefix,
+		}
+	} else {
+		pvOptions = []string{
+			"--host", d.apiHost,
+			"--port", d.apiPort,
+			"--diode-target", d.diodeTarget,
+			"--diode-client-id", d.diodeClientID,
+			"--diode-client-secret", "********",
+			"--diode-app-name-prefix", d.diodeAppNamePrefix,
+		}
 	}
 
 	if d.diodeOtelEndpoint != "" {
@@ -125,7 +157,9 @@ func (d *workerBackend) Start(ctx context.Context, cancelFunc context.CancelFunc
 
 	d.logger.Info("worker startup", slog.Any("arguments", pvOptions))
 
-	pvOptions[9] = d.diodeClientSecret
+	if !d.diodeDryRun && len(pvOptions) > 9 {
+		pvOptions[9] = d.diodeClientSecret
+	}
 
 	d.proc = backend.NewCmdOptions(backend.CmdOptions{
 		Buffered:  false,
@@ -145,7 +179,7 @@ func (d *workerBackend) Start(ctx context.Context, cancelFunc context.CancelFunc
 		stderr := d.proc.GetStderr()
 		for stdout != nil || stderr != nil {
 			select {
-			case line, open := <-stderr:
+			case line, open := <-stdout:
 				if !open {
 					stdout = nil
 					continue

agent/backend/worker/worker_test.go

@@ -201,3 +201,85 @@ func TestWorkerBackendCompleted(t *testing.T) {
 
 	assert.Error(t, err)
 }
+
+func TestWorkerBackendDryRun(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		switch {
+		case r.URL.Path == "/api/v1/status":
+			response := StatusResponse{Version: "1.3.4", StartTime: "2023-10-01T12:00:00Z", UpTime: 123.456}
+			_ = json.NewEncoder(w).Encode(response)
+		case r.URL.Path == "/api/v1/capabilities":
+			capabilities := map[string]any{"capability": true}
+			_ = json.NewEncoder(w).Encode(capabilities)
+		case strings.HasPrefix(r.URL.Path, "/api/v1/policies"):
+			_ = json.NewEncoder(w).Encode(map[string]any{"status": "ok"})
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer server.Close()
+
+	serverURL, err := url.Parse(server.URL)
+	assert.NoError(t, err)
+
+	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
+	repo, err := policies.NewMemRepo()
+	assert.NoError(t, err)
+
+	mockCmd := &mocks.MockCmd{}
+	mocks.SetupSuccessfulProcess(mockCmd, 12345)
+
+	originalNewCmdOptions := backend.NewCmdOptions
+	defer func() {
+		backend.NewCmdOptions = originalNewCmdOptions
+	}()
+
+	backend.NewCmdOptions = func(options backend.CmdOptions, name string, args ...string) backend.Commander {
+		assert.Equal(t, "orb-worker", name, "Expected command name to be orb-worker")
+		assert.Contains(t, args, "--dry-run")
+		assert.Contains(t, args, "--dry-run-output-dir")
+		assert.NotContains(t, args, "--host")
+		assert.NotContains(t, args, "--port")
+		assert.False(t, options.Buffered, "Expected buffered to be false")
+		assert.True(t, options.Streaming, "Expected streaming to be true")
+		return mockCmd
+	}
+
+	assert.True(t, worker.Register())
+	be := backend.GetBackend("worker")
+
+	beCommons := config.BackendCommons{
+		Diode: struct {
+			Target          string `yaml:"target"`
+			ClientID        string `yaml:"client_id"`
+			ClientSecret    string `yaml:"client_secret"`
+			AgentName       string `yaml:"agent_name"`
+			DryRun          bool   `yaml:"dry_run"`
+			DryRunOutputDir string `yaml:"dry_run_output_dir"`
+		}{
+			DryRun:          true,
+			DryRunOutputDir: "/tmp/dry-run-output",
+		},
+	}
+
+	err = be.Configure(logger, repo, map[string]any{
+		"host": serverURL.Hostname(),
+		"port": serverURL.Port(),
+	}, beCommons)
+	assert.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	err = be.Start(ctx, cancel)
+	assert.NoError(t, err)
+
+	assert.False(t, be.GetStartTime().IsZero())
+
+	err = be.RemovePolicy(policies.PolicyData{ID: "1", Name: "policy", Data: map[string]any{"k": "v"}})
+	assert.NoError(t, err)
+
+	err = be.Stop(context.WithValue(context.Background(), config.ContextKey("routine"), "test"))
+	assert.NoError(t, err)
+
+	mockCmd.AssertExpectations(t)
+}

agent/config/types.go

@@ -74,10 +74,12 @@ type BackendCommons struct {
 		AgentLabels map[string]string `yaml:"agent_labels"`
 	} `yaml:"otel"`
 	Diode struct {
-		Target       string `yaml:"target"`
-		ClientID     string `yaml:"client_id"`
-		ClientSecret string `yaml:"client_secret"`
-		AgentName    string `yaml:"agent_name"`
+		Target          string `yaml:"target"`
+		ClientID        string `yaml:"client_id"`
+		ClientSecret    string `yaml:"client_secret"`
+		AgentName       string `yaml:"agent_name"`
+		DryRun          bool   `yaml:"dry_run"`
+		DryRunOutputDir string `yaml:"dry_run_output_dir"`
 	}
 }
 

agent/docker/Dockerfile

@@ -50,9 +50,7 @@ COPY --from=otlpinf /exe /usr/local/bin/otlpinf
 
 COPY --from=network-discovery /usr/local/bin/network-discovery /usr/local/bin/network-discovery
 
-RUN mkdir -p /etc/snmp-discovery/lookup-extensions
 COPY --from=snmp-discovery /usr/local/bin/snmp-discovery /usr/local/bin/snmp-discovery
-COPY --from=snmp-discovery /etc/snmp-discovery/lookup-extensions /etc/snmp-discovery/lookup-extensions
 
 RUN pip3 install netboxlabs-device-discovery netboxlabs-orb-worker