From a86f5326a8ea709f652a269a676f2784dc6c581e Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Fri, 10 Jul 2015 14:14:26 -0700 Subject: [PATCH 1/5] Add initial Makefile. --- Makefile | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..95d1206 --- /dev/null +++ b/Makefile @@ -0,0 +1,45 @@ +default: clean bin/go-fuzz_executor bin/go-fuzz_scheduler bin/go-fuzz + +run: clean bin/go-fuzz_executor bin/go-fuzz run-scheduler + +clean: + -rm bin/fuzzlr_* + +bin: + -mkdir bin + +bin/fuzzlr-scheduler: bin + go build -o bin/fuzzler-scheduler cmd/fuzzlr-scheduler/app.go + +bin/fuzzlr_executor: bin + go build -o bin/fuzzlr-executor cmd/fuzzlr-executor/app.go + +bin/go-fuzz: bin + git submodule init + git submodule update + cd _vendor/go-fuzz/go-fuzz; go build; mv go-fuzz ../../../bin/ + +bin/go-fuzz-build: bin + git submodule init + git submodule update + cd _vendor/go-fuzz/go-fuzz-build; \ + for arch in {darwin,linux,windows}; do \ + GOOS=$$arch go build; \ + mv go-fuzz-build ../../../bin/; \ + done + +run-scheduler: + go run -race cmd/fuzzlr-scheduler/app.go -logtostderr=true + +install: + go install ./cmd/... + +cover: + for i in `dirname **/*_test.go | grep -v "_vendor" | sort | uniq`; do \ + echo $$i; \ + go test -v -race ./$$i/... -coverprofile=em-coverage.out; \ + go tool cover -func=em-coverage.out; rm em-coverage.out; \ + done + +test: + go test -race ./... From 7a4dc33d44febc7547e2c7670b99dcbbbc38123f Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Fri, 10 Jul 2015 15:46:31 -0600 Subject: [PATCH 2/5] Add go-fuzz Makefile target that builds go-fuzz and image/png test for linux and darwin. --- Makefile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 95d1206..43a1220 100644 --- a/Makefile +++ b/Makefile @@ -17,16 +17,22 @@ bin/fuzzlr_executor: bin bin/go-fuzz: bin git submodule init git submodule update - cd _vendor/go-fuzz/go-fuzz; go build; mv go-fuzz ../../../bin/ + cd _vendor/go-fuzz/go-fuzz; \ + for arch in {darwin,linux}; do \ + GOOS=$$arch go build -o ../../../bin/go-fuzz-$$arch; \ + done bin/go-fuzz-build: bin git submodule init git submodule update - cd _vendor/go-fuzz/go-fuzz-build; \ - for arch in {darwin,linux,windows}; do \ - GOOS=$$arch go build; \ - mv go-fuzz-build ../../../bin/; \ - done + cd _vendor/go-fuzz/go-fuzz-build; go build -o ../../../bin/go-fuzz-build + +bin/test: bin/go-fuzz-build + for arch in {darwin,linux}; do \ + GOOS=$$arch bin/go-fuzz-build -o bin/fmt-fuzz-$$arch.zip github.com/dvyukov/go-fuzz/examples/png; \ + done + +go-fuzz: bin/go-fuzz bin/test run-scheduler: go run -race cmd/fuzzlr-scheduler/app.go -logtostderr=true From 51a314f4b1c964934fbda50ed200aefe61faa125 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Fri, 10 Jul 2015 15:53:24 -0600 Subject: [PATCH 3/5] Add Makefile target that creates zip archive with corpus. --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 43a1220..6e9d77c 100644 --- a/Makefile +++ b/Makefile @@ -32,8 +32,11 @@ bin/test: bin/go-fuzz-build GOOS=$$arch bin/go-fuzz-build -o bin/fmt-fuzz-$$arch.zip github.com/dvyukov/go-fuzz/examples/png; \ done -go-fuzz: bin/go-fuzz bin/test - +bin/corpus.zip: bin + zip bin/corpus.zip $(GOPATH)/src/github.com/dvyukov/go-fuzz/examples/png/corpus/* + +go-fuzz: bin/go-fuzz bin/test bin/corpus.zip + run-scheduler: go run -race cmd/fuzzlr-scheduler/app.go -logtostderr=true From dafc839f1de9f420b11b61e9b2601b2dea064b73 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Fri, 10 Jul 2015 15:19:50 -0700 Subject: [PATCH 4/5] Add serving of assets over HTTP --- cmds/fuzzlr-scheduler/main.go | 34 ++++++++++++++++++++++++++++++++-- scheduler/driver.go | 4 +++- scheduler/scheduler.go | 2 +- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/cmds/fuzzlr-scheduler/main.go b/cmds/fuzzlr-scheduler/main.go index 57b26ab..394e93a 100644 --- a/cmds/fuzzlr-scheduler/main.go +++ b/cmds/fuzzlr-scheduler/main.go @@ -2,22 +2,52 @@ package main import ( "flag" + "fmt" "log" + "net/http" "os" "os/signal" + "path/filepath" "time" + "github.com/gogo/protobuf/proto" + mesos "github.com/mesos/mesos-go/mesosproto" + "github.com/mesosphere/fuzzlr/scheduler" + "github.com/mesosphere/fuzzlr/serving" ) func main() { fs := flag.NewFlagSet("fuzzlr", flag.ExitOnError) - master := fs.String("master", "localhost:5050", "Location of leading Mesos master") + master := fs.String("master", "localhost:5050", "Location of leading Mesos master.") + servingPath := fs.String("binpath", "./bin/", "Directory where go-fuzz binaries are located.") + bin := fs.String("bin", "./fuzz.zip", "Archive generated by go-fuzz-build.") + corpus := fs.String("corpus", "./corpus.zip", "Corpus zip archive to seed the fuzzing with.") + artifactPort := fs.Int("artifactPort", 12300, "Binding port for artifact server") + address := fs.String("address", "127.0.0.1", "Binding address for artifact server") shutdownTimeout := fs.Duration("shutdown.timeout", 10*time.Second, "Shutdown timeout") fs.Parse(os.Args[1:]) - sched := scheduler.New() + binpaths, err := filepath.Glob(*servingPath) + if err != nil { + log.Printf("Unable to read files from provided serving path %s: %v", *servingPath, err) + return + } + commandInfos := []*mesos.CommandInfo_URI{} + uris := []string{} + for _, path := range binpaths { + uri := serving.ServeExecutorArtifact(path, *address, *artifactPort) + commandInfos = append(commandInfos, &mesos.CommandInfo_URI{ + Value: uri, + Executable: proto.Bool(true), + }) + uris = append(uris, *uri) + } + go http.ListenAndServe(fmt.Sprintf("%s:%d", *address, *artifactPort), nil) + log.Printf("Serving executor artifacts...") + + sched := scheduler.New(*bin, *corpus, uris...) driver, err := scheduler.NewDriver(*master, sched) if err != nil { log.Printf("Unable to create scheduler driver: %s", err) diff --git a/scheduler/driver.go b/scheduler/driver.go index 0806583..ddfaa9a 100644 --- a/scheduler/driver.go +++ b/scheduler/driver.go @@ -15,7 +15,9 @@ func NewDriver(master string, s sched.Scheduler) (sched.SchedulerDriver, error) return sched.NewMesosSchedulerDriver(sched.DriverConfig{ Master: master, Framework: &pb.FrameworkInfo{ - Name: proto.String("Fuzzlr"), + Name: proto.String("Fuzzlr"), + Checkpoint: proto.Bool(true), + FailoverTimeout: proto.Float64(60 * 60 * 24 * 7), // User: proto.String(""), }, Scheduler: s, diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index 5c1d8b9..7228312 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -18,7 +18,7 @@ type Scheduler struct { done chan struct{} } -func New(artifactURIs ...string) *Scheduler { +func New(bin, corpus string, artifactURIs ...string) *Scheduler { return &Scheduler{ ExecutorInfo: mesos.ExecutorInfo{ ExecutorId: &mesos.ExecutorID{Value: proto.String("fuzzlr-executor")}, From 39f3d18c54831a077f6060a364144849acf47067 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Fri, 10 Jul 2015 15:45:54 -0700 Subject: [PATCH 5/5] Add some functions for creating ExecutorInfo --- scheduler/scheduler.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index f876696..4acdb16 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -33,6 +33,32 @@ func New(bin, corpus string, artifactURIs ...string) *Scheduler { } } +func NewFuzzMasterExecutorInfo(os string, host string, port int, corpus string) *mesos.ExecutorInfo { + cmd := fmt.Sprintf("mkdir -p work/corpus; unzip %s-%s -d ./work/corpus; "+ + "./go-fuzz-%s -workdir=work -master=%s:%d > master.log", corpus, os, os, host, port) + return &mesos.ExecutorInfo{ + ExecutorId: &mesos.ExecutorID{Value: proto.String("fuzzlr-executor")}, + Command: &mesos.CommandInfo{ + Value: proto.String(cmd), + Uris: commandURIs(artifactURIs...), + }, + Name: proto.String("Fuzzer"), + } +} + +func NewSlaveExecutorInfo(masterHost string, masterPort int, bin string) *mesos.ExecutorInfo { + cmd := fmt.Sprintf("./go-fuzz -bin=%s -slave=%s:%d -v=1 > slave.log", + bin, masterHost, masterPort) + return &mesos.ExecutorInfo{ + ExecutorId: &mesos.ExecutorID{Value: proto.String("fuzzlr-executor")}, + Command: &mesos.CommandInfo{ + Value: proto.String(cmd), + Uris: commandURIs(artifactURIs...), + }, + Name: proto.String("Fuzzer"), + } +} + // Shutdown shuts down the scheduler or times out after a given duration. func (s *Scheduler) Shutdown(timeout time.Duration) error { close(s.shutdown)