Merge pull request #237 from stanfordnmbl/docker-stability

dev: docker qol changes

Author: carmichaelong

app.py

@@ -79,7 +79,8 @@
         continue
 
     if r.status_code == 404:
-        logging.info("...pulling " + workerType + " trials from " + API_URL)
+        logging.info(f"...pulling {workerType} trials from {API_URL} "
+                     f"using commit {getCommitHash()}")
         time.sleep(1)
 
         # When using autoscaling, we will remove the instance scale-in protection if it hasn't

docker/docker-compose.yaml

@@ -38,6 +38,7 @@ services:
       options:
         max-size: "100m"  # Rotate when the log reaches 10MB
         max-file: "7"    # Keep the last 7 log files
+    restart: on-failure:3
   mmpose:
     image: ${MMPOSE_IMAGE_NAME}
     volumes:
@@ -55,6 +56,7 @@ services:
       options:
         max-size: "100m"  # Rotate when the log reaches 10MB
         max-file: "7"    # Keep the last 7 log files
+    restart: on-failure:3
 
 volumes:
   data: {}

mmpose/loop_mmpose.py

@@ -69,8 +69,8 @@ def checkCudaPyTorch():
         if os.path.isfile(bboxPath):
             os.remove(bboxPath)
 
-        logging.info("Done. Cleaning up")
+        logging.info("mmpose: Done. Cleaning up")
 
     except:
-        logging.info("Pose detection failed.")
+        logging.info("mmpose: Pose detection failed.")
         os.remove(video_path)

openpose/loop_openpose.py

@@ -92,7 +92,7 @@ def getResolutionCommand(resolutionPoseDetection, horizontal):
         time.sleep(0.1)
         continue
 
-    logging.info("Processing...")
+    logging.info("Processing openpose...")
 
     if os.path.isdir(output_dir):
         shutil.rmtree(output_dir)
@@ -101,13 +101,18 @@ def getResolutionCommand(resolutionPoseDetection, horizontal):
     horizontal = getVideoOrientation(video_path)
     cmd_hr = getResolutionCommand(resolutionPoseDetection, horizontal)
 
-    check_cuda_device()
-    command = "/openpose/build/examples/openpose/openpose.bin\
-        --video {video_path}\
-        --display 0\
-        --write_json {output_dir}\
-        --render_pose 0{cmd_hr}".format(video_path=video_path, output_dir=output_dir, cmd_hr=cmd_hr)
-    os.system(command)
+    try: 
+        check_cuda_device()
+        command = "/openpose/build/examples/openpose/openpose.bin\
+            --video {video_path}\
+            --display 0\
+            --write_json {output_dir}\
+            --render_pose 0{cmd_hr}".format(video_path=video_path, output_dir=output_dir, cmd_hr=cmd_hr)
+        os.system(command)
 
-    logging.info("Done. Cleaning up")
-    os.remove(video_path)
+        logging.info("openpose: Done. Cleaning up")
+        os.remove(video_path)
+
+    except:
+        logging.info("openpose: Pose detection failed.")
+        os.remove(video_path)

utils.py

@@ -1586,7 +1586,7 @@ def checkCudaTF():
         sendStatusEmail(message=message)
         raise Exception("No GPU detected. Exiting.")
 
-def writeToJsonLog(path, new_dict, max_entries=1000):
+def writeToJsonLog(path, new_dict, max_entries=1000, indent=2):
     dir_name = os.path.dirname(path)
     if not os.path.exists(dir_name):
         os.makedirs(dir_name)
@@ -1603,7 +1603,7 @@ def writeToJsonLog(path, new_dict, max_entries=1000):
         data.pop(0)
 
     with open(path, 'w') as f:
-        json.dump(data, f)
+        json.dump(data, f, indent=indent)
 
 def writeToErrorLog(path, session_id, trial_id, error, stack, max_entries=1000):
     error_entry = {

utilsServer.py

@@ -6,6 +6,7 @@
 import logging
 import time
 import random
+import urllib
 
 from main import main
 from utils import getDataDirectory
@@ -500,28 +501,28 @@ def runTestSession(pose='all',isDocker=True,maxNumTries=3):
             logging.info("\n\n\nStatus check succeeded. \n\n")
             return
 
-        # Catch and re-enter while loop if it's an HTTPError (could be more
-        # than just 404 errors). Wait between 30 and 60 seconds before 
-        # retrying.
-        except requests.exceptions.HTTPError as e:
+        # Catch and re-enter while loop if it's an HTTPError or URLError 
+        # (could be more than just 404 errors). Wait between 30 and 60 seconds 
+        # before retrying.
+        except (requests.exceptions.HTTPError, urllib.error.URLError) as e:
             if numTries < maxNumTries:
-                logging.info(f"test trial failed on try #{numTries} due to HTTPError. Retrying.")
+                logging.info(f"test trial failed on try #{numTries} due to HTTPError or URLError. Retrying.")
                 wait_time = random.randint(30,60)
                 logging.info(f"waiting {wait_time} seconds then retrying...")
                 time.sleep(wait_time)
                 continue
             else:
-                logging.info(f"test trial failed on try #{numTries} due to HTTPError.")
+                logging.info(f"test trial failed on try #{numTries} due to HTTPError or URLError.")
                 # send email
-                message = "A backend OpenCap machine failed the status check (HTTPError). It has been stopped."
+                message = "A backend OpenCap machine failed the status check (HTTPError or URLError). It has been stopped."
                 sendStatusEmail(message=message)
-                raise Exception('Failed status check (HTTPError). Stopped.')
+                raise Exception('Failed status check (HTTPError or URLError). Stopped.')
 
         # Catch other errors and stop
         except:
             logging.info("test trial failed. stopping machine.")
             # send email
-            message = "A backend OpenCap machine failed the status check. It has been stopped."
+            message = "A backend OpenCap machine failed the status check (not HTTPError or URLError). It has been stopped."
             sendStatusEmail(message=message)
             raise Exception('Failed status check. Stopped.')