diff --git a/.wordlist.txt b/.wordlist.txt index dce8bd43a1..fd69bf388d 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -1,46 +1,91 @@ +AHU +APIs +ARP +Auth +BACnet +BAMBI +BMS +BOS +CLA +CLI +ClearBlade +CloudRun +CoAP +CoV +DBO +DBP +DNS +Dev +Dockerized +ETHMAC +GCP +GCR +GCS +GHCR +GKE +GiB +GitLab +Github +GraphQL +IAM +IPv +IoT +JDK +JS +JUnit +JVM +JWT +Kubernetes +MQTT +MacOS +OIDC +OSS +Onboarding +PUBACKs +Prem +Proxied +PubSub +QoS +RPC +RSA +Readme +Reconciler +SDK +Schemas +Stackdriver +TLS +TODO +UDMI +UDMI's +UDMIF +UDMIS +UI +UNK +Writeback acked addr -AHU anagement api -APIs -ARP ascii auth -Auth backend backoff bacnet -BACnet -BAMBI bambi blobset -BMS -BOS breakpoint buildingsiot centric -CLA -ClearBlade -CLI -CloudRun -CoAP comms config coreutils -CoV cron datapoint -DBO -DBP designator deterministically dev -Dev directionality distroless -DNS -Dockerized dockerized downlink enablement @@ -51,64 +96,44 @@ envs etag etags ethmac -ETHMAC evice faucetsdn fieldbus gcloud -GCP -GCR -GCS gencode -GHCR ghcr -GiB github -Github -GitLab gittools -GKE googlegroups -GraphQL +hardcoded hostname http https -IAM idToken integrations invoker io iot -IoT ipv -IPv -JDK -JS json -JUnit -JWT keygen kube kubectl -Kubernetes localhost localnet logentry loglevel macos -MacOS md +microservices modbus -MQTT mydomain namespace niversal +nmap npm nterface -OIDC onboarding -Onboarding -OSS pagent parameterization parsers @@ -116,66 +141,46 @@ pointname pointset postfix prem -Prem prepend prepended programmatically provisioner proxied -Proxied proxying -PUBACKs pubber -PubSub pubsub -QoS -Readme -Reconciler reconciler repo -RPC -RSA runDev runtime schemas -Schemas -SDK setpoint sha sharding src -Stackdriver stateStatus +subFolder +subType +subTypes subblock subblocks subcomponent subcomponents subdirectory subflows -subFolder -subType -subTypes systemd terraform terraformed testability -TLS -TODO transactional transpile transpiles txt udmi -UDMI udmif -UDMIF -UDMI's -UDMIS -UI unconfigured underspecified uniqs -UNK unprocessable unwriteable uplink @@ -183,6 +188,4 @@ url validator writeable writeback -Writeback yaml -microservices diff --git a/docs/specs/sequences/HOW_TO_TEST.md b/docs/specs/sequences/HOW_TO_TEST.md new file mode 100644 index 0000000000..1f5f05d51d --- /dev/null +++ b/docs/specs/sequences/HOW_TO_TEST.md @@ -0,0 +1,68 @@ +[**UDMI**](../../../) / [**Docs**](../../) / [**Specs**](../) / [Sequences](#) + +# Adding a Sequencer Test + +This guide explains how to add a new sequencer test to the UDMI Java validator and what steps are necessary to ensure it passes Continuous Integration (CI). + +## 1. Create the Test + +Sequencer tests are located in `validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/`. +To create a new test, locate the appropriate sequence class (e.g., `SystemSequences.java`) or create a new one. + +Add your test method, annotated with `@Test`, `@Feature`, and `@Summary`. For example, a trivial test: + +```java + @Test(timeout = TWO_MINUTES_MS) + @Feature(stage = ALPHA, bucket = SYSTEM) + @Summary("Trivial test to check testing infrastructure") + public void trivial_test() { + checkThat("this is a trivial test", () -> true); + } +``` + +*Note: The sequencer tests share the same JVM instance. Avoid using `static` variables for test state to prevent cross-test leakage and flaky behavior.* + +## 2. Update Golden and Expected Files + +Just adding the test is not sufficient. You need to update the expected outputs and golden files used by the CI testing infrastructure. + +### A. Update `docs/specs/sequences/generated.md` + +This file documents all the tests. To update it, run: +```bash +bin/gencode_seq +``` +This will append your new test to the documentation. + +### B. Update `etc/sequencer.out` + +This golden file contains the expected outputs for all tests. You must manually add your test's expected output or regenerate it by running the sequencer locally and verifying the output. + +When manually adding an entry to `etc/sequencer.out`, **ensure the contents are sorted correctly** so they match the generated `out/sequencer.out` during testing. Use the following command to sort the file: +```bash +LC_ALL=C sort -k 3,4 -o etc/sequencer.out etc/sequencer.out +``` + +Example of a test entry: +```text +RESULT pass system.mode trivial_test ALPHA 10/10 Sequence complete +``` + +If you add, modify, or remove a test, this file must reflect those changes. + +### C. Verify with `bin/test_sequcheck` + +To verify that your newly added output matches the tests, run: +```bash +bin/test_sequcheck +``` +This command compares `out/sequencer.out` against `etc/sequencer.out`. It will fail if your test is not in the expected file or if the file isn't sorted correctly. + +## 3. Dealing with Flaky Tests + +When encountering unexpected test failures, you should **double-check them**, as flaky tests are common in complex integration test suites. + +* Use a **unique test name**. This makes it easy to trace any particular CI failure to your newly added test versus an existing, flaky test. +* **Discovery tests:** If testing discovery events (e.g., nmap scans), use polling loops instead of hardcoded `time.sleep()` to prevent flaky test failures due to variable execution times. +* **Timeouts:** If your test waits for a long-running operation (like a device restart or delayed connection), make sure to use `DEFAULT_LOOP_TIMEOUT` rather than the shorter `DEFAULT_WAIT_TIME` to prevent premature test timeouts. +* **State Leakage:** As mentioned earlier, avoid modifying `static` fields. Config transactions leaking across test runs executing in the same JVM session will cause flaky failures in subsequent tests. diff --git a/docs/specs/sequences/generated.md b/docs/specs/sequences/generated.md index 45917faecc..cf7d2cd9b1 100644 --- a/docs/specs/sequences/generated.md +++ b/docs/specs/sequences/generated.md @@ -68,6 +68,7 @@ Some caveats: * [state_software](#state_software-stable): Check that a device publishes correct software information in state messages * [system_last_update](#system_last_update-stable): Check that last_update state is correctly set in response to a config update. * [system_mode_restart](#system_mode_restart-preview): Restart and connect to same endpoint and expect it returns. +* [trivial_test](#trivial_test-stable) * [valid_serial_no](#valid_serial_no-stable) ## bad_point_ref (PREVIEW) @@ -661,6 +662,9 @@ Restart and connect to same endpoint and expect it returns. 1. Wait for last_start is newer than previous last_start Test passed. +## trivial_test (STABLE) + +1. Trivial test to check testing infrastructure ## valid_serial_no (STABLE) diff --git a/docs/specs/sequences/readme.md b/docs/specs/sequences/readme.md index 3d0b183ab8..46d3cf0780 100644 --- a/docs/specs/sequences/readme.md +++ b/docs/specs/sequences/readme.md @@ -14,3 +14,7 @@ More explanatory detail for some of the key functional specifications: - [Discovery](discovery.md): Flow for on-prem network and point discovery. - [Endpoint Reconfiguration](endpoint_reconfiguration.md): Ability to reconfigure endpoint (e.g. MQTT) from the cloud - [Writeback](writeback.md): Ability to control on-prem resources from the cloud. + +## Test Development + +- [How To Add Tests](HOW_TO_TEST.md): A guide on creating a new sequencer test and correctly updating the golden files so that it passes CI checking. diff --git a/docs/tools/sequencer.md b/docs/tools/sequencer.md index 0a04fe43f9..e399a7dd40 100644 --- a/docs/tools/sequencer.md +++ b/docs/tools/sequencer.md @@ -9,6 +9,10 @@ validates that the composition of sequential messages is compliant with the UDMI A [guide is given for using sequencer for complete device testing.](../guides/device_testing.md) +# Adding Sequencer Tests + +A [guide for adding tests and ensuring they pass Continuous Integration (CI).](../specs/sequences/HOW_TO_TEST.md) + # Sequencer Setup 1. Ensure you have a suitable MQTT broker setup. diff --git a/etc/sequencer.out b/etc/sequencer.out index 69bb49397d..6173ead718 100644 --- a/etc/sequencer.out +++ b/etc/sequencer.out @@ -42,14 +42,15 @@ RESULT pass system extra_config STABLE 10/10 Sequence complete RESULT pass system family_ether_addr PREVIEW 10/10 Sequence complete RESULT pass system family_ipv4_addr PREVIEW 10/10 Sequence complete RESULT skip system family_ipv6_addr PREVIEW 0/0 No ipv6 address defined in metadata -RESULT pass system.mode system_mode_restart PREVIEW 10/10 Sequence complete RESULT pass system state_make_model STABLE 10/10 Sequence complete RESULT pass system state_software STABLE 10/10 Sequence complete RESULT pass system system_last_update STABLE 11/11 Sequence complete CPBLTY pass system system_last_update.subblocks ALPHA 1/1 Capability supported RESULT pass system system_min_loglevel ALPHA 10/10 Sequence complete RESULT pass system too_much_state ALPHA 10/10 Sequence complete +RESULT pass system trivial_test STABLE 10/10 Sequence complete RESULT pass system valid_serial_no STABLE 10/10 Sequence complete +RESULT pass system.mode system_mode_restart PREVIEW 10/10 Sequence complete RESULT pass writeback writeback_failure ALPHA 10/10 Sequence complete RESULT pass writeback writeback_invalid ALPHA 10/10 Sequence complete RESULT pass writeback writeback_operation ALPHA 10/10 Sequence complete diff --git a/etc/sequencer_nostate.out b/etc/sequencer_nostate.out index 59911b343b..fb39208fb0 100644 --- a/etc/sequencer_nostate.out +++ b/etc/sequencer_nostate.out @@ -37,9 +37,10 @@ RESULT skip system extra_config STABLE 0/0 State testing disabled RESULT skip system family_ether_addr PREVIEW 0/0 State testing disabled RESULT skip system family_ipv4_addr PREVIEW 0/0 State testing disabled RESULT skip system family_ipv6_addr PREVIEW 0/0 State testing disabled -RESULT skip system.mode system_mode_restart PREVIEW 0/0 State testing disabled RESULT skip system state_make_model STABLE 0/0 State testing disabled RESULT skip system state_software STABLE 0/0 State testing disabled RESULT skip system system_last_update STABLE 0/0 State testing disabled CPBLTY skip system system_last_update.subblocks ALPHA 0/0 Never executed +RESULT skip system trivial_test STABLE 0/0 State testing disabled RESULT skip system valid_serial_no STABLE 0/0 State testing disabled +RESULT skip system.mode system_mode_restart PREVIEW 0/0 State testing disabled diff --git a/etc/shell_common.sh b/etc/shell_common.sh index 6048e08d00..84af0fa0ca 100644 --- a/etc/shell_common.sh +++ b/etc/shell_common.sh @@ -8,7 +8,7 @@ set -eu set -o pipefail # Force consistent sort order and other processing things -export LC_ALL=en_US.UTF-8 +export LC_ALL=C function search { egrep -q "$1" $2 || fail Could not find in $2: $1 diff --git a/validator/sequences/trivial_test/sequence.log b/validator/sequences/trivial_test/sequence.log new file mode 100644 index 0000000000..09c4e6eb63 --- /dev/null +++ b/validator/sequences/trivial_test/sequence.log @@ -0,0 +1 @@ +TIMESTAMP NOTICE RESULT pass system trivial_test STABLE 10/10 Sequence complete diff --git a/validator/sequences/trivial_test/sequence.md b/validator/sequences/trivial_test/sequence.md new file mode 100644 index 0000000000..8baabedec8 --- /dev/null +++ b/validator/sequences/trivial_test/sequence.md @@ -0,0 +1,3 @@ +## trivial_test (STABLE) + +1. Trivial test to check testing infrastructure diff --git a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/SystemSequences.java b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/SystemSequences.java index 0b0ce45420..7e81f9286d 100644 --- a/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/SystemSequences.java +++ b/validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/SystemSequences.java @@ -83,6 +83,14 @@ public void too_much_state() { () -> numStateUpdates <= STATE_LIMIT_THRESHOLD); // TODO: Check that the state timestamp is close to current time. } + + @Test(timeout = TWO_MINUTES_MS) + @Feature(stage = STABLE, bucket = SYSTEM) + @Summary("Trivial test to check testing infrastructure") + public void trivial_test() { + checkThat("this is a trivial test", () -> true); + } + }