Update ci to catch non zero return value for diffs

Author: mcbarton

.github/workflows/main.yml

@@ -355,19 +355,32 @@ jobs:
           # to run notebooks in it. If you try to run the notebooks before the website is
           # ready the ci python script will crash saying ti cannot access the url
           sleep 10
+          echo "Running xeus-cpp in Jupter Lite in Chrome"
           python -u scripts/automated-notebook-run-script.py --driver chrome --notebook ${{ matrix.notebook }} --kernel ${{ matrix.kernel }}
+          echo "Diff Notebooks Chrome"
           nbdiff notebooks/${{ matrix.notebook }} $HOME/Downloads/${{ matrix.notebook }} --ignore-id --ignore-metadata
-          rm $HOME/Downloads/${{ matrix.notebook }}
+          export CHROME_TESTS_RETURN_VALUE=$?
+          echo "Running xeus-cpp in Jupter Lite in Firefox"
           python -u scripts/automated-notebook-run-script.py --driver firefox --notebook ${{ matrix.notebook }} --kernel ${{ matrix.kernel }}
+          echo "Diff Notebooks Firefox"
           nbdiff notebooks/${{ matrix.notebook }} $HOME/Downloads/${{ matrix.notebook }} --ignore-id --ignore-metadata
-          rm $HOME/Downloads/${{ matrix.notebook }}      
+	  export FIREFOX_TESTS_RETURN_VALUE=$?
+	  rm $HOME/Downloads/${{ matrix.notebook }}      
+          export SAFARI_TESTS_RETURN_VALUE=0
           if [[ "${{ matrix.os }}" == "macos"* ]]; then
             python -m pip install PyAutoGUI
             python scripts/enable-downloads-safari-github-ci.py
             python -u scripts/automated-notebook-run-script.py --driver safari --notebook ${{ matrix.notebook }} --kernel ${{ matrix.kernel }}
+	    echo "Running xeus-cpp in Jupter Lite in  Safari"
+            echo "Diff Notebooks Safari"
             nbdiff notebooks/${{ matrix.notebook }} $HOME/Downloads/${{ matrix.notebook }} --ignore-id --ignore-metadata
+	    export SAFARI_TESTS_RETURN_VALUE=$?
             rm $HOME/Downloads/${{ matrix.notebook }}
           fi
+          if [[ $SAFARI_TESTS_RETURN_VALUE -ne 0 || $FIREFOX_TESTS_RETURN_VALUE -ne 0 || $CHROME_TESTS_RETURN_VALUE -ne 0 ]]; then
+            echo "Diff for Safari, Chrome or Firefox returned non zero value (could be more than one)"
+            exit 1
+          fi
         timeout-minutes: 15
 
       - name: Setup tmate session

scripts/automated-notebook-run-script.py

@@ -9,6 +9,7 @@
 from selenium.webdriver.common.keys import Keys
 import time
 import platform
+import sys
 
 def cell_is_waiting_for_input(driver):
     dialog_selectors = [
@@ -207,7 +208,7 @@ def main():
         elapsed = time.time() - start_time
         if elapsed > timeout:
             print(f"Timeout reached ({elapsed:.1f} seconds). Stopping loop.")
-            break
+            sys.exit(1)
 
         spans = driver.find_elements(By.CSS_SELECTOR, "span.jp-StatusBar-TextItem")
         status_span = spans[2]