MCU8MASS-545 Add all tests besides gps tracker and add timeout for entries

Author: M65649

test/test.py

@@ -21,7 +21,7 @@
     "bodvoltage=1v9,bodmode=disabled,eesave=enable,resetpin=reset,"\
     "millis=tcb2,startuptime=8,wiremode=mors2,printf=full"
 
-TIMEOUT = 10
+TIMEOUT = 30
 
 
 def retrieve_examples(examples_directory):
@@ -85,17 +85,25 @@ def test(example, test_data, serial_handle):
         expectation = entry.get("expectation", None)
         command = entry.get("command", None)
         repeat = entry.get("repeat", 1)
+        timeout = entry.get("timeout", TIMEOUT)
 
-        for _ in range(0, repeat):
+        if not expectation:
+            return (False, f"No expeted value defined for entry: {entry}")
+
+        for i in range(0, repeat):
             if command != None:
                 command_stripped = command.strip("\r")
                 logging.info(f"\tTesting command: {command_stripped}")
 
                 serial_handle.write(str.encode(command))
                 serial_handle.flush()
 
-            # Read until line feed
-            output = serial_handle.read_until().decode("utf-8")
+            # Read until line feed or timeout
+            start = time.time()
+            output = ""
+
+            while time.time() - start < timeout and output == "":
+                output = serial_handle.read_until().decode("utf-8")
 
             response = re.search(expectation, output)
 
@@ -148,9 +156,15 @@ def test(example, test_data, serial_handle):
                         default="tests.json")
 
     parser.add_argument("-e",
-                        "--examplesdir",
+                        "--examples",
+                        type=str,
+                        help="Examples to test (all or some specific example, e.g. sandbox)",
+                        default="all")
+
+    parser.add_argument("-s",
+                        "--sketchdir",
                         type=str,
-                        help="Relative path to the example directory",
+                        help="Relative path to the example directory with the example sketches",
                         default="../examples")
 
     parser.add_argument("-b",
@@ -182,7 +196,7 @@ def test(example, test_data, serial_handle):
             for name in files:
                 os.remove(os.path.join(root, name))
 
-    examples = retrieve_examples(arguments.examplesdir)
+    examples = retrieve_examples(arguments.sketchdir)
 
     test_config = {}
 
@@ -195,6 +209,9 @@ def test(example, test_data, serial_handle):
 
         example_name = os.path.splitext(os.path.basename(example))[0]
 
+        if example_name != arguments.examples and arguments.examples != "all":
+            continue
+
         if not example_name in test_config:
             examples_test_status[example_name] = {"status": "No test defined", "error": None}
             continue
@@ -218,7 +235,7 @@ def test(example, test_data, serial_handle):
                 if not error:
                     examples_test_status[example_name] = {"status": "Passed", "error": None}
                 else:
-                    examples_test_status[example_name] = {"status": "Passed", "error": error}
+                    examples_test_status[example_name] = {"status": "Not passed", "error": error}
 
         except serial.SerialException as exception:
             logging.error(f"Got exception while opening serial port: {exception}")
@@ -228,6 +245,8 @@ def test(example, test_data, serial_handle):
 
     backend.disconnect_from_tool()
 
+    all_tests_passed = True
+
     print("--------------- Test status ---------------")
     for example_name, entry in examples_test_status.items():
         status = entry["status"]
@@ -237,5 +256,9 @@ def test(example, test_data, serial_handle):
             print(f"{example_name:<30}: {status}")
         else:
             print(f"{example_name:<30}: {status} - {error}")
+            all_tests_passed = False
 
-    exit(0)
+    if all_tests_passed:
+        exit(0)
+    else:
+        exit(1)

test/tests.json

@@ -234,8 +234,8 @@
       }
     ]
   },
-  "mqtt_polling_aws-not-finished": {
-    "enabled": false,
+  "mqtt_polling_aws": {
+    "enabled": true,
     "tests": [
       {
         "expectation": "\\[INFO\\] Starting initialization of MQTT Polling for AWS"
@@ -269,8 +269,8 @@
       }
     ]
   },
-  "power_down-not-finished": {
-    "enabled": false,
+  "power_down": {
+    "enabled": true,
     "tests": [
       {
         "expectation": "\\[INFO\\] Connecting to operator.{0,}OK!"
@@ -282,7 +282,7 @@
         "expectation": "\\[INFO\\] Powering down..."
       },
       {
-        "timeout": 65,
+        "timeout": 90,
         "expectation": "\\[INFO\\] Connecting to operator.{0,}OK!"
       },
       {
@@ -292,5 +292,65 @@
         "expectation": "\\[INFO\\] Doing work..."
       }
     ]
+  },
+  "power_print_voltage": {
+    "enabled": true,
+    "tests": [
+      {
+        "expectation": "\\[INFO\\] Starting up example for printing voltage supplied to the board"
+      },
+      {
+        "repeat": 5,
+        "expectation": "\\[INFO\\] The voltage supplied is: [0-9]\\.[0-9]+"
+      }
+    ]
+  },
+  "power_save": {
+    "enabled": true,
+    "tests": [
+      {
+        "expectation": "\\[INFO\\] Connecting to operator.{0,}OK!"
+      },
+      {
+        "expectation": "\\[INFO\\] Connected to operator: (.*)"
+      },
+      {
+        "expectation": "\\[INFO\\] Power saving..."
+      },
+      {
+        "timeout": 300,
+        "expectation": "\\[INFO\\] Woke up!"
+      },
+      {
+        "expectation": "\\[INFO\\] Doing work..."
+      }
+    ]
+  },
+  "sandbox": {
+    "enabled": true,
+    "tests": [
+      {
+        "expectation": "\\[INFO\\] Starting sandbox / landing page procedure. Version = [0-9]+\\.[0-9]+\\.[0-9]+"
+      },
+      {
+        "expectation": "\\[INFO\\] Board name: [a-z0-9]+"
+      },
+      {
+        "expectation": "\\[INFO\\] Connecting to operator.{0,}OK!"
+      },
+      {
+        "expectation": "\\[INFO\\] Connected to operator: (.*)"
+      },
+      {
+        "expectation": "\\[INFO\\] Connecting to MQTT broker..."
+      },
+      {
+        "expectation": "\\[INFO\\] Connected to MQTT broker, subscribing to topic: \\$aws\/things\/[a-z0-9]+\/shadow\/update\/delta!"
+      },
+      {
+        "repeat": 2,
+        "expectation": "\\[INFO\\] Sending heartbeat"
+      }
+    ]
   }
-}
+}