diff --git a/.github/scripts/bazel_build.sh b/.github/scripts/bazel_build.sh index cbfcc4e9..6353f923 100755 --- a/.github/scripts/bazel_build.sh +++ b/.github/scripts/bazel_build.sh @@ -58,15 +58,7 @@ BAZELISK_BUILD_CMD="${BAZELISK_CMD} build --noshow_progress --strategy=CppCompil BAZELISK_RUN_CMD="${BAZELISK_CMD} run" BAZEL_BINARY_TARGETS=( - "//:word_query" - "//:word_query_evolution" - "//:attention_broker_service" - "//:attention_broker_client" - "//:das" - "//:db_loader" - "//:busnode" - "//:busclient" - "//:database_adapter" + "//:ci_binaries" ) BAZEL_BINARY_OUTPUTS=( diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 965e583c..99a22f75 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -54,22 +54,41 @@ jobs: - name: Loading Load Knowledge Base run: das-cli metta load /tmp/animals.metta - - name: Setup bazel - run: .github/scripts/setup_bazel.sh - - - name: Compile Binaries - run: .github/scripts/bazel_build.sh - - name: Starting Attention Broker run: das-cli attention-broker restart - - name: Starting MORK Server and Load Knowledge Base + - name: Starting MORK Servers run: |- - make run-mork-server > /dev/null 2>&1 & - - until curl --silent http://localhost:40022/status/-; do - echo "Waiting MORK…" - sleep 1 + set -euo pipefail + + make run-mork-server & + pid_40022=$! + DAS_MORK_PORT=40032 make run-mork-server & + pid_40032=$! + + launch_failed=0 + wait "${pid_40022}" || launch_failed=1 + wait "${pid_40032}" || launch_failed=1 + if [[ "${launch_failed}" -ne 0 ]]; then + echo "MORK server launch failed" >&2 + docker ps -a --filter "name=das-mork-server-" >&2 || true + exit 1 + fi + + for port in 40022 40032; do + for attempt in $(seq 1 60); do + if curl --fail --silent --show-error "http://localhost:${port}/status/-"; then + echo "MORK ready on port ${port}" + break + fi + if [[ "${attempt}" -eq 60 ]]; then + echo "MORK failed to start on port ${port}" >&2 + docker ps -a --filter "name=das-mork-server-${port}" >&2 || true + exit 1 + fi + echo "Waiting for MORK on port ${port}…" + sleep 1 + done done - name: Starting Postgres Server and load database @@ -87,5 +106,13 @@ jobs: docker exec -i pg_test psql -U postgres -d postgres_wrapper_test < src/scripts/postgres_setup.sql + - name: Setup bazel + run: .github/scripts/setup_bazel.sh + - name: Execute Unit Test Suite - run: make run-tests-only + working-directory: ./src + run: ./scripts/bazel_exec.sh test --show_progress --cache_test_results=no //tests/cpp/... + + - name: Build production binaries + working-directory: ./src + run: ./scripts/bazel_exec.sh build --noshow_progress //:ci_binaries diff --git a/Makefile b/Makefile index a8c825a6..2c2dfa2e 100644 --- a/Makefile +++ b/Makefile @@ -112,6 +112,14 @@ test-agents-integration: run-tests-only: @$(MAKE) bazel 'test --show_progress --cache_test_results=no //tests/...' +build-ci-binaries: + @cd src && ./scripts/bazel_exec.sh build --noshow_progress //:ci_binaries + +run-tests-native: + @cd src && ./scripts/bazel_exec.sh test --show_progress --cache_test_results=no //tests/cpp/... + +ci-unit-tests: run-tests-native build-ci-binaries + lint-all: @$(MAKE) bazel lint \ "//... --fix --report --diff" \ diff --git a/src/.bazelrc b/src/.bazelrc index c290f259..259760da 100644 --- a/src/.bazelrc +++ b/src/.bazelrc @@ -9,8 +9,8 @@ common --color=yes common --curses=yes # common --show_timestamps -# Use half of all available CPU cores -common --jobs=HOST_CPUS*0.5 +# Use 85% all available CPU cores +common --jobs=HOST_CPUS*0.85 # Enable bazel bzlmod (for Bazel 8+) common --enable_bzlmod @@ -45,8 +45,8 @@ build --host_cxxopt=-w build --host_cxxopt=-Wno-all #################################### TEST ##################################### -# Use half of all available CPU cores -# test --jobs=HOST_CPUS*0.5 +# Use 85% of all available CPU cores +# test --jobs=HOST_CPUS*0.85 # Other options test --test_output=errors diff --git a/src/BUILD b/src/BUILD index acb33c62..7ab1ad72 100644 --- a/src/BUILD +++ b/src/BUILD @@ -215,3 +215,23 @@ cc_binary( "//main:bus_client_lib", ], ) + +# Meta-target for CI: building this forces all production binaries to compile/link. +# Used by .github/workflows/run-unit-tests.yml and bazel_build.sh. +genrule( + name = "ci_binaries", + srcs = [ + ":word_query", + ":word_query_evolution", + ":attention_broker_service", + ":attention_broker_client", + ":das", + ":db_loader", + ":busnode", + ":busclient", + ":database_adapter", + ], + outs = ["ci_binaries.stamp"], + cmd = "touch $@", + visibility = ["//visibility:public"], +) diff --git a/src/atomdb/morkdb/MorkDB.cc b/src/atomdb/morkdb/MorkDB.cc index 23f5e529..e2d9faa2 100644 --- a/src/atomdb/morkdb/MorkDB.cc +++ b/src/atomdb/morkdb/MorkDB.cc @@ -79,20 +79,12 @@ httplib::Result MorkClient::send_request(const string& method, const string& pat res = this->cli.Post(path, data, "text/plain"); } - if (!res) { - RAISE_ERROR("Connection error at http://" + this->base_url_ + path + ": " + - httplib::to_string(res.error())); - } - - if (res->status == httplib::StatusCode::OK_200) { - return res; - } + if (!res || is_transient_mork_http_status(res->status)) { + string status_str = res ? std::to_string(res->status) : "unknown"; - if (is_transient_mork_http_status(res->status)) { if (attempt + 1 >= max_attempts) { RAISE_ERROR("Http error at http://" + this->base_url_ + path + ": exhausted " + - std::to_string(max_attempts) + " retries, last status " + - std::to_string(res->status)); + std::to_string(max_attempts) + " retries, last status " + status_str); } unsigned int delay_ms = initial_delay_ms; @@ -100,12 +92,16 @@ httplib::Result MorkClient::send_request(const string& method, const string& pat delay_ms = static_cast(delay_ms * backoff); } - LOG_ERROR("MORK transient HTTP " << res->status << " at " << path << ", retrying (attempt " + LOG_ERROR("MORK transient HTTP " << status_str << " at " << path << ", retrying (attempt " << attempt + 1 << "/" << max_attempts << ")"); Utils::sleep(delay_ms); continue; } + if (res->status == httplib::StatusCode::OK_200) { + return res; + } + RAISE_ERROR("Http error at http://" + this->base_url_ + path + ": status " + std::to_string(res->status)); } diff --git a/src/scripts/mork_server.sh b/src/scripts/mork_server.sh index a71ced6f..ce7deb36 100755 --- a/src/scripts/mork_server.sh +++ b/src/scripts/mork_server.sh @@ -1,20 +1,16 @@ #!/bin/bash -set -eoux pipefail +set -euo pipefail IMAGE_NAME="trueagi/das:mork-server-1.1.0" -CONTAINER_NAME="das-mork-server-$(date +%Y%m%d%H%M%S)" +MORK_PORT="${DAS_MORK_PORT:-40022}" +CONTAINER_NAME="das-mork-server-${MORK_PORT}" -docker run --rm \ +docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true + +docker run -d --rm \ --name="${CONTAINER_NAME}" \ --network host \ -e MORK_SERVER_ADDR=0.0.0.0 \ - -e MORK_SERVER_PORT=${DAS_MORK_PORT:-40022} \ + -e MORK_SERVER_PORT="${MORK_PORT}" \ "${IMAGE_NAME}" "$@" - -sleep 1 - -if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then - echo "Removing existing container: ${CONTAINER_NAME}" - _=$(docker rm -f "${CONTAINER_NAME}" 2>&1 > /dev/null || true) -fi diff --git a/src/tests/cpp/BUILD b/src/tests/cpp/BUILD index 2ca1c7f8..619a1ea3 100644 --- a/src/tests/cpp/BUILD +++ b/src/tests/cpp/BUILD @@ -942,7 +942,7 @@ cc_test( cc_test( name = "postgreswrapper_test", - size = "small", + size = "medium", srcs = [ "postgreswrapper_test.cc", ], @@ -971,7 +971,7 @@ cc_test( cc_test( name = "morkwrapper_test", - size = "small", + size = "medium", srcs = [ "morkwrapper_test.cc", ], @@ -1001,7 +1001,7 @@ cc_test( cc_test( name = "adapterdb_test", - size = "small", + size = "medium", srcs = [ "adapterdb_test.cc", ], diff --git a/src/tests/cpp/adapterdb_test.cc b/src/tests/cpp/adapterdb_test.cc index b87ebb3c..54a0040f 100644 --- a/src/tests/cpp/adapterdb_test.cc +++ b/src/tests/cpp/adapterdb_test.cc @@ -34,7 +34,7 @@ struct AdapterTestParams { }; void seed_mork_adapter_test_data() { - auto mork_client = make_shared("localhost:40022"); + auto mork_client = make_shared("localhost:40032"); const string similarity_seed = "(Similarity \"ent\" \"human\")"; const string inheritance_seed = "(Inheritance \"human\" \"mammal\")"; @@ -192,7 +192,7 @@ static const AdapterTestParams MorkParams = { R"( (Similarity "ent" $h) (Inheritance "human" $m))", - {{"host", "localhost"}, {"port", 40022}}, + {{"host", "localhost"}, {"port", 40032}}, "Mork", }; diff --git a/src/tests/cpp/morkdb_test.cc b/src/tests/cpp/morkdb_test.cc index 4491f9f7..a5c798d3 100644 --- a/src/tests/cpp/morkdb_test.cc +++ b/src/tests/cpp/morkdb_test.cc @@ -125,7 +125,7 @@ TEST_F(MorkDBTest, QueryForTargets) { } TEST_F(MorkDBTest, ConcurrentQueryForPattern) { - const int num_threads = 200; + const int num_threads = 8; vector threads; atomic success_count{0}; @@ -301,7 +301,7 @@ TEST_F(MorkDBTest, ConcurrentAddLinks) { int arity = 3; int chunck_size = 500; - const int num_threads = 100; + const int num_threads = 8; vector threads; atomic success_count{0}; diff --git a/src/tests/cpp/morkwrapper_test.cc b/src/tests/cpp/morkwrapper_test.cc index 08cb9ed3..9f871729 100644 --- a/src/tests/cpp/morkwrapper_test.cc +++ b/src/tests/cpp/morkwrapper_test.cc @@ -35,7 +35,7 @@ using namespace processor; class MorkWrapperTest : public ::testing::Test { protected: string TEST_HOST = "localhost"; - int TEST_PORT = 40022; + int TEST_PORT = 40032; string INVALID_HOST = "invalid.host"; int INVALID_PORT = 99999; diff --git a/src/tests/integration/cpp/BUILD b/src/tests/integration/cpp/BUILD index 4f0fd87e..e7420ad2 100644 --- a/src/tests/integration/cpp/BUILD +++ b/src/tests/integration/cpp/BUILD @@ -11,6 +11,7 @@ cc_test( "-Iexternal/gtest/googletest", ], linkstatic = 1, + tags = ["integration"], deps = [ "//tests/integration/cpp/helpers:process_helper", "//tests/integration/cpp/helpers:test_helper",