Added integrated monitoring. (#5)

Signed-off-by: Bartlomiej Plotka <bwplotka@gmail.com>

Author: bwplotka

README.md

@@ -25,24 +25,25 @@ Let's go through an example leveraging `go test` flow:
 
 1. Implement the workload by embedding`e2e.Runnable` or `*e2e.InstrumentedRunnable`. Or you can use existing ones in [e2edb](db/) package. For example implementing Thanos Querier with our desired configuration could look like this:
 
-   ```go mdox-exec="sed -n '47,64p' examples/thanos/standalone.go"
+   ```go mdox-exec="sed -n '49,67p' examples/thanos/standalone.go"
    func newThanosSidecar(env e2e.Environment, name string, prom e2e.Linkable) *e2e.InstrumentedRunnable {
    	ports := map[string]int{
    		"http": 9090,
    		"grpc": 9091,
    	}
-   	return e2e.NewInstrumentedRunnable(env, name, ports, "http", e2e.StartOptions{
-   		Image: "quay.io/thanos/thanos:v0.21.1",
-   		Command: e2e.NewCommand("sidecar", e2e.BuildArgs(map[string]string{
-   			"--debug.name":     name,
-   			"--grpc-address":   fmt.Sprintf(":%d", ports["grpc"]),
-   			"--http-address":   fmt.Sprintf(":%d", ports["http"]),
-   			"--prometheus.url": "http://" + prom.InternalEndpoint(e2edb.AccessPortName),
-   			"--log.level":      "info",
-   		})...),
-   		Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200),
-   		User:      strconv.Itoa(os.Getuid()),
-   	})
+   	return e2e.NewInstrumentedRunnable(env, name, ports, "http").Init(
+   		e2e.StartOptions{
+   			Image: "quay.io/thanos/thanos:v0.21.1",
+   			Command: e2e.NewCommand("sidecar", e2e.BuildArgs(map[string]string{
+   				"--debug.name":     name,
+   				"--grpc-address":   fmt.Sprintf(":%d", ports["grpc"]),
+   				"--http-address":   fmt.Sprintf(":%d", ports["http"]),
+   				"--prometheus.url": "http://" + prom.InternalEndpoint(e2edb.AccessPortName),
+   				"--log.level":      "info",
+   			})...),
+   			Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200),
+   			User:      strconv.Itoa(os.Getuid()),
+   		})
    }
    ```
 
@@ -119,6 +120,30 @@ Let's go through an example leveraging `go test` flow:
    }
    ```
 
+### Monitoring
+
+Each instrumented workload have programmatic access to latest metrics with `WaitSumMetricsWithOptions` methods family. Yet, especially for standalone mode it's often useful to query yourself and visualisate metrics provided by your workloads and environment. In order to do so just start monitoring from `e2emontioring` package:
+
+```go
+mon, err := e2emonitoring.Start(e)
+if err != nil {
+	return err
+}
+```
+
+This will start Prometheus with automatic discovery for every new and old instrumented runnables being scraped. It also runs cadvisor that monitors docker itself if `env.DockerEnvironment` is started. Run `OpenUserInterfaceInBrowser()` to open Prometheus UI in browser.
+
+```go
+	// Open monitoring page with all metrics.
+	if err := mon.OpenUserInterfaceInBrowser(); err != nil {
+		return err
+	}
+```
+
+To see how it works in practice run our example code in [standalone.go](examples/thanos/standalone.go) by running `make run-example`. At the end two UI should show in your browser. Thanos one and monitoring one. You can then e.g query docker container metrics using `sum(container_memory_working_set_bytes{name!=""}) by (name)` metric e.g:
+
+![mem metric](monitoring.png)
+
 ## Credits
 
 * Initial Authors: [@pracucci](https://github.com/pracucci), [@bwplotka](https://github.com/bwplotka), [@pstibrany](https://github.com/pstibrany)

db/db.go

@@ -61,7 +61,7 @@ func NewMinio(env e2e.Environment, name, bktName string, opts ...Option) *e2e.In
 		"MINIO_KMS_KES_CERT_FILE=" + "root.cert",
 		"MINIO_KMS_KES_KEY_NAME=" + "my-minio-key",
 	}
-	f := e2e.NewFutureInstrumentedRunnable(env, name, ports, AccessPortName)
+	f := e2e.NewInstrumentedRunnable(env, name, ports, AccessPortName)
 	return f.Init(
 		e2e.StartOptions{
 			Image: o.image,
@@ -87,11 +87,7 @@ func NewConsul(env e2e.Environment, name string, opts ...Option) *e2e.Instrument
 	}
 
 	e2e.MergeFlags()
-	return e2e.NewInstrumentedRunnable(
-		env,
-		name,
-		map[string]int{AccessPortName: 8500},
-		AccessPortName,
+	return e2e.NewInstrumentedRunnable(env, name, map[string]int{AccessPortName: 8500}, AccessPortName).Init(
 		e2e.StartOptions{
 			Image: o.image,
 			// Run consul in "dev" mode so that the initial leader election is immediate.
@@ -107,11 +103,7 @@ func NewDynamoDB(env e2e.Environment, name string, opts ...Option) *e2e.Instrume
 		opt(&o)
 	}
 
-	return e2e.NewInstrumentedRunnable(
-		env,
-		name,
-		map[string]int{AccessPortName: 8000},
-		AccessPortName,
+	return e2e.NewInstrumentedRunnable(env, name, map[string]int{AccessPortName: 8000}, AccessPortName).Init(
 		e2e.StartOptions{
 			Image:   o.image,
 			Command: e2e.NewCommand("-jar", "DynamoDBLocal.jar", "-inMemory", "-sharedDb"),
@@ -127,11 +119,7 @@ func NewBigtable(env e2e.Environment, name string, opts ...Option) *e2e.Instrume
 		opt(&o)
 	}
 
-	return e2e.NewInstrumentedRunnable(
-		env,
-		name,
-		nil,
-		AccessPortName,
+	return e2e.NewInstrumentedRunnable(env, name, nil, AccessPortName).Init(
 		e2e.StartOptions{
 			Image: o.image,
 		},
@@ -144,11 +132,7 @@ func NewCassandra(env e2e.Environment, name string, opts ...Option) *e2e.Instrum
 		opt(&o)
 	}
 
-	return e2e.NewInstrumentedRunnable(
-		env,
-		name,
-		map[string]int{AccessPortName: 9042},
-		AccessPortName,
+	return e2e.NewInstrumentedRunnable(env, name, map[string]int{AccessPortName: 9042}, AccessPortName).Init(
 		e2e.StartOptions{
 			Image: o.image,
 			// Readiness probe inspired from https://github.com/kubernetes/examples/blob/b86c9d50be45eaf5ce74dee7159ce38b0e149d38/cassandra/image/files/ready-probe.sh
@@ -163,11 +147,7 @@ func NewSwiftStorage(env e2e.Environment, name string, opts ...Option) *e2e.Inst
 		opt(&o)
 	}
 
-	return e2e.NewInstrumentedRunnable(
-		env,
-		name,
-		map[string]int{AccessPortName: 8080},
-		AccessPortName,
+	return e2e.NewInstrumentedRunnable(env, name, map[string]int{AccessPortName: 8080}, AccessPortName).Init(
 		e2e.StartOptions{
 			Image:     o.image,
 			Readiness: e2e.NewHTTPReadinessProbe(AccessPortName, "/", 404, 404),
@@ -181,9 +161,7 @@ func NewMemcached(env e2e.Environment, name string, opts ...Option) e2e.Runnable
 		opt(&o)
 	}
 
-	return env.Runnable(
-		name,
-		map[string]int{AccessPortName: 11211},
+	return env.Runnable(name).WithPorts(map[string]int{AccessPortName: 11211}).Init(
 		e2e.StartOptions{
 			Image:     o.image,
 			Readiness: e2e.NewTCPReadinessProbe(AccessPortName),
@@ -197,14 +175,7 @@ func NewETCD(env e2e.Environment, name string, opts ...Option) *e2e.Instrumented
 		opt(&o)
 	}
 
-	return e2e.NewInstrumentedRunnable(
-		env,
-		name,
-		map[string]int{
-			AccessPortName: 2379,
-			"metrics":      9000,
-		},
-		"metrics",
+	return e2e.NewInstrumentedRunnable(env, name, map[string]int{AccessPortName: 2379, "metrics": 9000}, "metrics").Init(
 		e2e.StartOptions{
 			Image:     o.image,
 			Command:   e2e.NewCommand("/usr/local/bin/etcd", "--listen-client-urls=http://0.0.0.0:2379", "--advertise-client-urls=http://0.0.0.0:2379", "--listen-metrics-urls=http://0.0.0.0:9000", "--log-level=error"),

db/prometheus.go

@@ -26,7 +26,7 @@ func NewPrometheus(env e2e.Environment, name string, opts ...Option) *Prometheus
 
 	ports := map[string]int{"http": 9090}
 
-	f := e2e.NewFutureInstrumentedRunnable(env, name, ports, "http")
+	f := e2e.NewInstrumentedRunnable(env, name, ports, "http")
 	config := fmt.Sprintf(`
 global:
   external_labels:

db/thanos.go

@@ -38,7 +38,7 @@ func NewThanosQuerier(env e2e.Environment, name string, endpointsAddresses []str
 		args = e2e.MergeFlagsWithoutRemovingEmpty(args, o.flagOverride)
 	}
 
-	return e2e.NewInstrumentedRunnable(env, name, ports, "http", e2e.StartOptions{
+	return e2e.NewInstrumentedRunnable(env, name, ports, "http").Init(e2e.StartOptions{
 		Image:     o.image,
 		Command:   e2e.NewCommand("query", e2e.BuildKingpinArgs(args)...),
 		Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200),
@@ -68,7 +68,7 @@ func NewThanosSidecar(env e2e.Environment, name string, prom e2e.Linkable, opts
 		args = e2e.MergeFlagsWithoutRemovingEmpty(args, o.flagOverride)
 	}
 
-	return e2e.NewInstrumentedRunnable(env, name, ports, "http", e2e.StartOptions{
+	return e2e.NewInstrumentedRunnable(env, name, ports, "http").Init(e2e.StartOptions{
 		Image:     o.image,
 		Command:   e2e.NewCommand("sidecar", e2e.BuildKingpinArgs(args)...),
 		Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200),
@@ -87,7 +87,7 @@ func NewThanosStore(env e2e.Environment, name string, bktConfigYaml []byte, opts
 		"grpc": 9091,
 	}
 
-	f := e2e.NewFutureInstrumentedRunnable(env, name, ports, "http")
+	f := e2e.NewInstrumentedRunnable(env, name, ports, "http")
 	args := map[string]string{
 		"--debug.name":      name,
 		"--grpc-address":    fmt.Sprintf(":%d", ports["grpc"]),

env.go

@@ -42,23 +42,31 @@ func WithVerbose() EnvironmentOption {
 
 // Environment defines how to run Runnable in isolated area e.g via docker in isolated docker network.
 type Environment interface {
+	// SharedDir returns host directory that will be shared with all runnables.
 	SharedDir() string
-	// Runnable returns instance of runnable which can be started and stopped within this environment.
-	Runnable(name string, Ports map[string]int, opts StartOptions) Runnable
-	// FutureRunnable returns instance of runnable which can be started and stopped within this environment.
-	FutureRunnable(name string, Ports map[string]int) FutureRunnable
-	// Close shutdowns isolated environment and cleans it's resources.
+	// Runnable returns runnable builder which can build runnables that can be started and stopped within this environment.
+	Runnable(name string) RunnableBuilder
+	// AddListener registers given listener to be notified on environment runnable changes.
+	AddListener(listener EnvironmentListener)
+	// Close shutdowns isolated environment and cleans its resources.
 	Close()
 }
 
+type EnvironmentListener interface {
+	OnRunnableChange(started []Runnable) error
+}
+
 type StartOptions struct {
 	Image     string
 	EnvVars   map[string]string
 	User      string
 	Command   Command
 	Readiness ReadinessProbe
-	// WaitReadyBackoff represents backoff used for WaitReady.
+	// WaitReadyBackofff represents backoff used for WaitReady.
 	WaitReadyBackoff *backoff.Config
+	Volumes          []string
+	UserNs           string
+	Privileged       bool
 }
 
 // Linkable is the entity that one can use to link runnable to other runnables before started.
@@ -84,8 +92,24 @@ type FutureRunnable interface {
 	Init(opts StartOptions) Runnable
 }
 
+type RunnableBuilder interface {
+	WithPorts(map[string]int) RunnableBuilder
+	WithConcreteType(r Runnable) RunnableBuilder
+
+	// Future returns future runnable
+	Future() FutureRunnable
+	// Init returns runnable.
+	Init(opts StartOptions) Runnable
+}
+
+type identificable interface {
+	id() uintptr
+}
+
 // Runnable is the entity that environment returns to manage single instance.
 type Runnable interface {
+	identificable
+
 	Linkable
 
 	// IsRunning returns if runnable was started.