From 75f2cf9a682dcbcdb7dfe191261fd1800723b507 Mon Sep 17 00:00:00 2001 From: Wojtek Bednarzak Date: Tue, 22 May 2018 19:29:49 +0100 Subject: [PATCH] feat(build): Add bazel build system Currently only added to Go targets. --- .gitignore | 7 +- BUILD.bazel | 7 ++ Makefile | 19 ++++- WORKSPACE | 158 +++++++++++++++++++++++++++++++++++ hrctl/BUILD.bazel | 29 +++++++ manager-fallback/BUILD.bazel | 111 ++++++++++++++++++++++++ proto/BUILD.bazel | 29 +++++++ proto/Makefile | 8 +- proto/mapreduce.proto | 2 +- worker/BUILD.bazel | 96 +++++++++++++++++++++ 10 files changed, 457 insertions(+), 9 deletions(-) create mode 100644 BUILD.bazel create mode 100644 WORKSPACE create mode 100644 hrctl/BUILD.bazel create mode 100644 manager-fallback/BUILD.bazel create mode 100644 proto/BUILD.bazel create mode 100644 worker/BUILD.bazel diff --git a/.gitignore b/.gitignore index 3702091..096a02d 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,9 @@ legacy-proto/src/* *.log # Go dependencies -vendor/ \ No newline at end of file +vendor/ + +# Bazel files +bazel-* +!WORKSPACE +!BUILD.bazel \ No newline at end of file diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 0000000..2322bd7 --- /dev/null +++ b/BUILD.bazel @@ -0,0 +1,7 @@ +load("@io_bazel_rules_docker//container:image.bzl", "container_image") + +container_image( + name = "worker_alpine", + files = ["//worker:worker"], + entrypoint = ["/worker"], +) \ No newline at end of file diff --git a/Makefile b/Makefile index 6fa1537..d50c061 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ release: clean: cargo clean - + ############################################################# # Runs all the tests @@ -47,8 +47,21 @@ clean-all: clean-docker clean ############################################################# -go-proto: - cd proto && make +go-proto-deps: + go get -v github.com/golang/protobuf/proto + go get -v github.com/golang/protobuf/protoc-gen-go + +go-proto: go-proto-deps + @mkdir -p proto/datatypes proto/mapreduce + protoc proto/datatypes.proto --go_out=proto/datatypes + + protoc proto/mapreduce.proto --go_out=plugins=grpc,Mproto/datatypes.proto=github.com/cpssd/heracles/proto/datatypes:proto/mapreduce + + # This is a weird issue where the files are generated not where they are + # supposed to be. + mv proto/datatypes/proto/datatypes.pb.go proto/datatypes + mv proto/mapreduce/proto/mapreduce.pb.go proto/mapreduce + rmdir proto/datatypes/proto proto/mapreduce/proto go-build: go-proto go build -o ./target/debug/manager-fallback manager-fallback/manager.go diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000..be47f2b --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,158 @@ +http_archive( + name = "io_bazel_rules_go", + url = "https://github.com/bazelbuild/rules_go/releases/download/0.12.0/rules_go-0.12.0.tar.gz", + sha256 = "c1f52b8789218bb1542ed362c4f7de7052abcf254d865d96fb7ba6d44bc15ee3", +) +load("@io_bazel_rules_go//go:def.bzl", "go_rules_dependencies", "go_register_toolchains",) +go_rules_dependencies() +go_register_toolchains() +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") +gazelle_dependencies() + +http_archive( + name = "io_bazel_rules_docker", + sha256 = "6dede2c65ce86289969b907f343a1382d33c14fbce5e30dd17bb59bb55bb6593", + strip_prefix = "rules_docker-0.4.0", + urls = ["https://github.com/bazelbuild/rules_docker/archive/v0.4.0.tar.gz"], +) +load( + "@io_bazel_rules_docker//container:container.bzl", + "container_pull", + container_repositories = "repositories", +) +container_repositories() + +# External Dependencies + +load("@bazel_gazelle//:deps.bzl", "go_repository") + +go_repository( + name = "com_github_golang_glog", + importpath = "github.com/golang/glog", + commit = "23def4e6c14b4da8ac2ed8007337bc5eb5007998", +) + +go_repository( + name = "com_github_golang_protobuf", + importpath = "github.com/golang/protobuf", + tag = "v1.1.0", +) + +go_repository( + name = "com_github_pkg_errors", + importpath = "github.com/pkg/errors", + tag = "v0.8.0", +) + +go_repository( + name = "com_github_spf13_pflag", + importpath = "github.com/spf13/pflag", + tag = "v1.0.1", +) + +go_repository( + name = "com_github_spf13_viper", + importpath = "github.com/spf13/viper", + tag = "v1.0.2", +) + +go_repository( + name = "com_github_streadway_amqp", + importpath = "github.com/streadway/amqp", + commit = "8e4aba63da9fc5571e01c6a45dc809a58cbc5a68", +) + +go_repository( + name = "org_golang_x_net", + importpath = "golang.org/x/net", + remote = "git@github.com:golang/net", + vcs = "git", + commit = "f73e4c9ed3b7ebdd5f699a16a880c2b1994e50dd", +) + +go_repository( + name = "org_golang_google_grpc", + importpath = "google.golang.org/grpc", + tag = "v1.12.0", +) + +go_repository( + name = "com_github_google_uuid", + importpath = "github.com/google/uuid", + tag = "0.2", +) + +go_repository( + name = "com_github_urfave_cli", + importpath = "github.com/urfave/cli", + commit = "8e01ec4cd3e2d84ab2fe90d8210528ffbb06d8ff", +) + +go_repository( + name = "com_github_olekukonko_tablewriter", + importpath = "github.com/olekukonko/tablewriter", + commit = "d4647c9c7a84d847478d890b816b7d8b62b0b279", +) + +# Dependencies of dependencies +go_repository( + name = "com_github_fsnotify_fsnotify", + importpath = "github.com/fsnotify/fsnotify", + tag = "v1.4.7", +) + +go_repository( + name = "com_github_hashicorp_hcl", + importpath = "github.com/hashicorp/hcl", + commit = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168", +) + +go_repository( + name = "com_github_spf13_jwalterweatherman", + importpath = "github.com/spf13/jwalterweatherman", + commit = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394", +) + +go_repository( + name = "com_github_magiconair_properties", + importpath = "github.com/magiconair/properties", + tag = "v1.8.0", +) + +go_repository( + name = "in_gopkg_yaml_v2", + importpath = "gopkg.in/yaml.v2", + tag = "v2.2.1", +) + +go_repository( + name = "com_github_mitchellh_mapstructure", + importpath = "github.com/mitchellh/mapstructure", + commit = "bb74f1db0675b241733089d5a1faa5dd8b0ef57b", +) + +go_repository( + name = "com_github_spf13_afero", + importpath = "github.com/spf13/afero", + tag = "v1.1.0", +) + +go_repository( + name = "com_github_spf13_cast", + importpath = "github.com/spf13/cast", + tag = "v1.2.0", +) + +go_repository( + name = "org_golang_x_sys", + importpath = "golang.org/x/sys", + remote = "git@github.com:golang/sys", + vcs = "git", + commit = "7c87d13f8e835d2fb3a70a2912c811ed0c1d241b", +) + +go_repository( + name = "com_github_mattn_go_runewidth", + importpath = "github.com/mattn/go-runewidth", + tag = "v0.0.2", +) \ No newline at end of file diff --git a/hrctl/BUILD.bazel b/hrctl/BUILD.bazel new file mode 100644 index 0000000..8e3a5be --- /dev/null +++ b/hrctl/BUILD.bazel @@ -0,0 +1,29 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_binary( + name = "hrctl", + srcs = ["hrctl.go"], + deps = [":app"], + visibility = ["//visibility:public"], + pure = "on", + static = "on", +) + +go_library( + name = "app", + srcs = [ + "app/app.go", + "app/parser.go", + "app/runner.go", + ], + deps = [ + "//proto:datatypes_go_proto", + "//proto:mapreduce_go_proto", + "@com_github_golang_protobuf//proto:go_default_library", + "@com_github_pkg_errors//:go_default_library", + "@org_golang_google_grpc//:go_default_library", + "@com_github_urfave_cli//:go_default_library", + "@com_github_olekukonko_tablewriter//:go_default_library", + ], + importpath = "github.com/cpssd/heracles/hrctl/app" +) \ No newline at end of file diff --git a/manager-fallback/BUILD.bazel b/manager-fallback/BUILD.bazel new file mode 100644 index 0000000..5bfe303 --- /dev/null +++ b/manager-fallback/BUILD.bazel @@ -0,0 +1,111 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_binary( + name = "manager-fallback", + srcs = ["manager.go"], + deps = [ + ":broker", + ":settings", + ":scheduler", + ":state", + ":server", + "@com_github_golang_glog//:go_default_library", + "@com_github_pkg_errors//:go_default_library", + ], + static = "on", + pure = "on", +) + +go_library( + name = "broker", + srcs = [ + "broker/amqp.go", + "broker/broker.go", + ], + deps = [ + ":settings", + "//proto:datatypes_go_proto", + "@com_github_golang_glog//:go_default_library", + "@com_github_golang_protobuf//proto:go_default_library", + "@com_github_streadway_amqp//:go_default_library", + "@com_github_pkg_errors//:go_default_library", + ], + importpath = "github.com/cpssd/heracles/manager-fallback/broker", +) + +go_library( + name = "settings", + srcs = ["settings/settings.go"], + deps = [ + "@com_github_golang_glog//:go_default_library", + "@com_github_spf13_pflag//:go_default_library", + "@com_github_spf13_viper//:go_default_library", + ], + importpath = "github.com/cpssd/heracles/manager-fallback/settings", +) + +go_library( + name = "scheduler", + srcs = ["scheduler/scheduler.go"], + deps = [ + ":broker", + ":splitting", + ":state", + "//proto:datatypes_go_proto", + "@com_github_google_uuid//:go_default_library", + "@com_github_golang_glog//:go_default_library", + "@com_github_pkg_errors//:go_default_library", + ], + importpath = "github.com/cpssd/heracles/manager-fallback/scheduler", +) + +go_library( + name = "splitting", + srcs = [ + "splitting/map.go", + "splitting/reduce.go", + "splitting/splitting.go", + ], + deps = [ + ":settings", + "//proto:datatypes_go_proto", + "@com_github_google_uuid//:go_default_library", + "@com_github_golang_glog//:go_default_library", + "@com_github_pkg_errors//:go_default_library", + ], + importpath = "github.com/cpssd/heracles/manager-fallback/splitting", +) + +go_library( + name = "state", + srcs = [ + "state/file.go", + "state/state.go", + ], + deps = [ + ":settings", + "//proto:datatypes_go_proto", + "@com_github_golang_glog//:go_default_library", + "@com_github_fsnotify_fsnotify//:go_default_library", + "@com_github_golang_protobuf//proto:go_default_library", + "@com_github_pkg_errors//:go_default_library", + ], + importpath = "github.com/cpssd/heracles/manager-fallback/state", +) + +go_library( + name = "server", + srcs = [ + "server/server.go", + "server/jobschedule.go", + ], + deps = [ + ":scheduler", + ":settings", + "//proto:mapreduce_go_proto", + "@com_github_pkg_errors//:go_default_library", + "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//codes:go_default_library", + ], + importpath = "github.com/cpssd/heracles/manager-fallback/server", +) \ No newline at end of file diff --git a/proto/BUILD.bazel b/proto/BUILD.bazel new file mode 100644 index 0000000..ca39a92 --- /dev/null +++ b/proto/BUILD.bazel @@ -0,0 +1,29 @@ +package(default_visibility = ["//visibility:public"]) + +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") + +proto_library( + name = "datatypes_proto", + srcs = ["datatypes.proto"], +) + +proto_library( + name = "mapreduce_proto", + srcs = ["mapreduce.proto"], + deps = [":datatypes_proto"], +) + +go_proto_library( + name = "datatypes_go_proto", + importpath = "github.com/cpssd/heracles/proto/datatypes", + proto = ":datatypes_proto", +) + +go_proto_library( + name = "mapreduce_go_proto", + importpath = "github.com/cpssd/heracles/proto/mapreduce", + proto = ":mapreduce_proto", + deps = [":datatypes_go_proto"], + compilers = ["@io_bazel_rules_go//proto:go_grpc"], +) + diff --git a/proto/Makefile b/proto/Makefile index 3146b1d..5515fa7 100644 --- a/proto/Makefile +++ b/proto/Makefile @@ -4,7 +4,7 @@ deps: go get -v github.com/golang/protobuf/proto go get -v github.com/golang/protobuf/protoc-gen-go -build: deps - mkdir -p datatypes mapreduce - protoc datatypes.proto --go_out=datatypes - protoc mapreduce.proto --go_out=plugins=grpc,Mdatatypes.proto=github.com/cpssd/heracles/proto/datatypes:mapreduce \ No newline at end of file +build: + @echo -e "\n\n\tThis target has been deprecated. Use root's go-proto\n\n" + cd .. && make go-proto + # protoc mapreduce.proto --go_out=plugins=grpc \ No newline at end of file diff --git a/proto/mapreduce.proto b/proto/mapreduce.proto index 36f74c1..31471fc 100644 --- a/proto/mapreduce.proto +++ b/proto/mapreduce.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package mapreduce; -import "datatypes.proto"; +import "proto/datatypes.proto"; // JobScheduleService adheres to API v2 service JobScheduleService { diff --git a/worker/BUILD.bazel b/worker/BUILD.bazel new file mode 100644 index 0000000..9851d56 --- /dev/null +++ b/worker/BUILD.bazel @@ -0,0 +1,96 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_binary( + name = "worker", + srcs = ["worker.go"], + deps = [ + ":broker", + ":runner", + ":settings", + ":state", + "@com_github_pkg_errors//:go_default_library", + ], + static = "on", + pure = "on", + visibility = ["//visibility:public"], +) + +go_library( + name = "broker", + srcs = [ + "broker/amqp.go", + "broker/broker.go", + ], + deps = [ + ":settings", + "//proto:datatypes_go_proto", + "@com_github_golang_glog//:go_default_library", + "@com_github_golang_protobuf//proto:go_default_library", + "@com_github_streadway_amqp//:go_default_library", + "@com_github_pkg_errors//:go_default_library", + ], + importpath = "github.com/cpssd/heracles/worker/broker", +) + +go_test( + name = "broker_test", + srcs = ["broker/amqp_test.go"], + embed = [":broker"], +) + +go_library( + name = "settings", + srcs = ["settings/settings.go"], + deps = [ + "@com_github_golang_glog//:go_default_library", + "@com_github_spf13_pflag//:go_default_library", + "@com_github_spf13_viper//:go_default_library", + ], + importpath = "github.com/cpssd/heracles/worker/settings", +) + +go_library( + name = "runner", + srcs = [ + "runner/runner.go", + "runner/util.go", + ], + deps = [ + ":broker", + ":state", + "//proto:datatypes_go_proto", + "@com_github_golang_glog//:go_default_library", + "@com_github_pkg_errors//:go_default_library", + ], + importpath = "github.com/cpssd/heracles/worker/runner", +) + +go_test( + name = "runner_test", + srcs = ["runner/runner_test.go"], + embed = [":runner"], +) + +go_library( + name = "state", + srcs = [ + "state/state.go", + "state/file.go", + ], + deps = [ + ":settings", + "//proto:datatypes_go_proto", + "@com_github_pkg_errors//:go_default_library", + "@com_github_golang_glog//:go_default_library", + "@com_github_golang_protobuf//proto:go_default_library", + ], + importpath = "github.com/cpssd/heracles/worker/state", +) + +go_test( + name = "state_test", + srcs = [ + "state/file_test.go", + ], + embed = [":state"], +) \ No newline at end of file