From 619a4dbdcf3d0c682921689627bb72f64891ec11 Mon Sep 17 00:00:00 2001 From: William Crouch Date: Sat, 25 Oct 2025 14:08:12 -0400 Subject: [PATCH 1/3] Reapply "Merge branch 'main' into feature/hoot-support" This reverts commit 426273e89fe128fda2dd18f38b504c585a53e85c. --- .dockerignore | 6 + .gitignore | 4 + docker-compose.yml | 37 ++++ docker-services/dataviz/Dockerfile | 18 ++ docker-services/dataviz/src/main.py | 1 + docker-services/dataviz/todo.txt | 2 + docker-services/ingest/Dockerfile | 23 +++ docker-services/ingest/build.sh | 9 + .../ingest/flashpoint-ingest.service | 11 ++ docker-services/ingest/newcontainer.sh | 9 + docker-services/ingest/requirements.txt | 1 + docker-services/ingest/src/main.py | 162 ++++++++++++++++++ docker-services/ingest/todo.txt | 2 + drive-backup.py | 105 ++++++++++++ ingest-requirements.txt | 2 + ingest_dir.sh | 4 + setup-db.py | 3 + viz-requirements.txt | 4 + 18 files changed, 403 insertions(+) create mode 100644 .dockerignore create mode 100644 docker-compose.yml create mode 100644 docker-services/dataviz/Dockerfile create mode 120000 docker-services/dataviz/src/main.py create mode 100644 docker-services/dataviz/todo.txt create mode 100644 docker-services/ingest/Dockerfile create mode 100755 docker-services/ingest/build.sh create mode 100644 docker-services/ingest/flashpoint-ingest.service create mode 100755 docker-services/ingest/newcontainer.sh create mode 100644 docker-services/ingest/requirements.txt create mode 100755 docker-services/ingest/src/main.py create mode 100644 docker-services/ingest/todo.txt create mode 100644 drive-backup.py create mode 100644 ingest-requirements.txt create mode 100755 ingest_dir.sh create mode 100644 setup-db.py create mode 100644 viz-requirements.txt diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a829363 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +**/docker/ +**/telemetry/ +Dockerfile-ingest +docker-compose.yml +.venv +.env \ No newline at end of file diff --git a/.gitignore b/.gitignore index e39c48d..608681b 100644 --- a/.gitignore +++ b/.gitignore @@ -122,6 +122,8 @@ celerybeat.pid # Environments .env .venv +.ingest-venv +.viz-venv env/ venv/ ENV/ @@ -163,4 +165,6 @@ cython_debug/ data/ converted_data/ db/ +telemetry/ +faketele/ old_2024_data/ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ead6ff2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,37 @@ +services: + ingest: + image: flashpoint-ingest + build: + dockerfile: ./docker-services/ingest/Dockerfile + volumes: + - telemetry:/app/telemetry + - database:/app/db + - convertedData:/app/converted_data + dataviz: + image: flashpoint-dataviz + ports: + - "8501:8501" + build: + dockerfile: ./docker-services/dataviz/Dockerfile + volumes: + - database:/app/db/ + - convertedData:/app/converted_data/ + +# if it failed to mount _data, make sure FLASHPOINT_DIR is correct and there is a telemetry, db, and converted_data directory +volumes: + telemetry: + driver_opts: + type: "none" + o: "bind" + device: "${FLASHPOINT_DIR}/telemetry" + database: + driver_opts: + type: "none" + o: "bind" + device: "${FLASHPOINT_DIR}/db" + convertedData: + driver_opts: + type: "none" + o: "bind" + device: "${FLASHPOINT_DIR}/converted_data" + \ No newline at end of file diff --git a/docker-services/dataviz/Dockerfile b/docker-services/dataviz/Dockerfile new file mode 100644 index 0000000..5d9a913 --- /dev/null +++ b/docker-services/dataviz/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.12.3-slim-bullseye + +#RUN apt-get update && apt-get install -y --no-install-recommends iputils-ping sshpass ssh +#RUN rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY . . + +RUN pip install --no-cache-dir -r docker-services/dataviz/requirements.txt +RUN pip install --no-cache-dir -r viz-requirements.txt + +RUN mkdir /app/db/ +RUN mkdir /app/converted_data/ + +EXPOSE 8501 +# eventually change to python3 docker-services/dataviz/src/main.py +CMD ["streamlit", "run", "viz.py"] \ No newline at end of file diff --git a/docker-services/dataviz/src/main.py b/docker-services/dataviz/src/main.py new file mode 120000 index 0000000..ebdca85 --- /dev/null +++ b/docker-services/dataviz/src/main.py @@ -0,0 +1 @@ +/home/ubuntu/Documents/flashpoint/viz.py \ No newline at end of file diff --git a/docker-services/dataviz/todo.txt b/docker-services/dataviz/todo.txt new file mode 100644 index 0000000..00e84c4 --- /dev/null +++ b/docker-services/dataviz/todo.txt @@ -0,0 +1,2 @@ +- pre-defined data viz app +- pygwalker app for data browsing diff --git a/docker-services/ingest/Dockerfile b/docker-services/ingest/Dockerfile new file mode 100644 index 0000000..57af930 --- /dev/null +++ b/docker-services/ingest/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.12.3-slim-bullseye + +RUN apt-get update && apt-get install -y --no-install-recommends sshpass ssh +RUN rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY . . + +RUN pip install --no-cache-dir -r docker-services/ingest/requirements.txt + +#RUN ls -lag && sleep 15 +RUN pip install --no-cache-dir -r ingest-requirements.txt + +RUN mkdir /app/telemetry/ +RUN mkdir /app/converted_data/ +RUN mkdir /app/db/ + +EXPOSE 22 + +RUN python3 setup-db.py + +CMD ["python", "docker-services/ingest/src/main.py"] \ No newline at end of file diff --git a/docker-services/ingest/build.sh b/docker-services/ingest/build.sh new file mode 100755 index 0000000..6c46def --- /dev/null +++ b/docker-services/ingest/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash +docker volume create --driver local \ + -o o=bind \ + -o type=none \ + -o device="/home/ubuntu/Ignite/flashpoint-docker/telemetry" \ + flashpoint-telemetry + +docker build -t flashpoint-ingest . + diff --git a/docker-services/ingest/flashpoint-ingest.service b/docker-services/ingest/flashpoint-ingest.service new file mode 100644 index 0000000..3d6a463 --- /dev/null +++ b/docker-services/ingest/flashpoint-ingest.service @@ -0,0 +1,11 @@ +[Unit] +Description=A Watchdog Ingest Docker container that will wait for the robot, the unload the files + +[Service] +Type=simple +ExecStart=export FLASHPOINT_DIR="/home/ignite/workspace/flashpoint/" && /usr/bin/docker compose up $FLASHPOINT_DIR +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/docker-services/ingest/newcontainer.sh b/docker-services/ingest/newcontainer.sh new file mode 100755 index 0000000..164c0c5 --- /dev/null +++ b/docker-services/ingest/newcontainer.sh @@ -0,0 +1,9 @@ +#!/bin/bash +docker create -it \ +--volume /var/lib/docker/volumes/flashpoint-telemetry:/app/telemetry/ \ +--name flashpoint-ingest flashpoint-ingest + +##!/bin/bash +#docker create -it \ # keep shell open even when no activity +#--volume /var/lib/docker/volumes/flashpoint-telemetry:/app/telemetry/ \ # specify volume:mount-point +#--name flashpoint-ingest flashpoint-ingest \ No newline at end of file diff --git a/docker-services/ingest/requirements.txt b/docker-services/ingest/requirements.txt new file mode 100644 index 0000000..f24de7f --- /dev/null +++ b/docker-services/ingest/requirements.txt @@ -0,0 +1 @@ +ping3==5.1.5 \ No newline at end of file diff --git a/docker-services/ingest/src/main.py b/docker-services/ingest/src/main.py new file mode 100755 index 0000000..5cfa42b --- /dev/null +++ b/docker-services/ingest/src/main.py @@ -0,0 +1,162 @@ +import subprocess +import time +import sys +from ping3 import ping # type: ignore + +botIP = "10.68.29.2" +botUser = "lvuser" +botPass = "" +teleDir = "/home/lvuser/logs" +allDir = teleDir+"/*" +botHostname = botUser+"@"+botIP +watchdogDelay = 5 +afterFoundDelay = 300 +timeSlot = 1 + +SPACESEPARATOR = " " + + +def check_ip_alive(ip_address): + try: + response = ping(ip_address, timeout=1) + return response is not None + except Exception: + return False + +passPrefix = [ + "sshpass", "-p", botPass, +] + +sshPrefix = passPrefix + [ + "ssh", botHostname, # login + "-o StrictHostKeyChecking=no", # no "check fingerprint" message/error + "-o UserKnownHostsFile=/dev/null", # don't save fingerprint +] + +getDate = [ + "date", # get date + "+%Y%m%d" # format into YYYYMMDD +] + +def retrieveLogs(): + date = int(subprocess.run(getDate, capture_output=True).stdout.decode()) + + listFiles = sshPrefix + [ + "ls", # list all files + "-1t", # one per line, in time order (newest first) + teleDir, # directory + ] + grepFilter = listFiles + ["|", + "find", # find all files + "|", # and then + "grep", f"*{date-timeSlot}" # filter to only files within specified days + ] + + lsRes = "" + lsErr = "" + if timeSlot == 0: + lsCmd = subprocess.run(listFiles, capture_output=True) + lsRes = lsCmd.stdout.decode() + lsErr = lsCmd.stderr.decode() + else: + lsCmd = subprocess.run(grepFilter, capture_output=True) + lsRes = lsCmd.stdout.decode() + lsErr = lsCmd.stderr.decode() + + + if len(lsErr.splitlines()) > 1: + print("Something failed: "+lsErr) + print("Logs: "+lsRes) + return + + if len(lsRes.splitlines()) == 0 and timeSlot != 0: + print(f"\nFailed to find any logs within {timeSlot} days") + return + elif len(lsRes.splitlines()) == 0 and timeSlot == 0: + print(f"There appears to be no logs in this directory on the host: {teleDir}") + return + + for line in lsRes.splitlines(): + print(line) + scpSuccessFlag = False + + scpCommand = passPrefix + [ + "scp", + "-o StrictHostKeyChecking=no", # no "check fingerprint" message/error + "-o UserKnownHostsFile=/dev/null", # don't save fingerprint + botHostname+":"+teleDir+"/"+line, "/app/telemetry/", # take one user@ip:/path/to/logs/ file and store in /app/telemetry/ + ] + + removeCommand = [ + "ssh", botHostname, # login + "-o StrictHostKeyChecking=no", # no "check fingerprint" message/error + "-o UserKnownHostsFile=/dev/null", # don't save fingerprint + "rm", "-f", f"{teleDir+"/"+line}", # delete copied file + ] + + res = subprocess.run(scpCommand, capture_output=True).stderr.decode() + scpRes = "" + print("Downloading file: "+line) + if "No such file or directory" in res: + scpRes = "Error: " + res + f"\n - {scpCommand}" + else: + scpRes = res + scpSuccessFlag = True + print("Retrieving logs:", scpRes) + + finalRemoveRes = "" + if scpSuccessFlag: + removeRes = subprocess.run(removeCommand, capture_output=True).stderr.decode() + if "No such file or directory" in removeRes: + finalRemoveRes = "Error: " + removeRes + f"\n - {removeCommand}" + elif removeRes is None: + finalRemoveRes = "null" + else: + finalRemoveRes = removeRes + print("Removing old logs:", finalRemoveRes) + print("\n\n") # separate + + # start ingest + print("Starting Ingest on files in ./telemetry") + ingestCMD = subprocess.run(["python3", "./ingest_dir.sh"], capture_output=True) + ingestRes = ingestCMD.stdout.decode() + ingestErr = ingestCMD.stderr.decode() + print(ingestRes) + print(ingestErr) + + """for fileName in lsRes.splitlines(): + rmCMD = subprocess.run(["rm", "./telemetry/"+fileName], capture_output=True) + rmRes = rmCMD.stdout.decode() + rmErr = rmCMD.stderr.decode() + print(rmRes) + print(rmErr)""" + +def main(): + global timeSlot + if len(sys.argv) > 1: + if "--time" in sys.argv and len(sys.argv) > 2: + timeSlot = int(sys.argv[2]) + if "--force" in sys.argv: + retrieveLogs() + return + elif len(sys.argv) == 2: + print("Usage: main.py [OPTIONS]\n") + print("Options:\n") + print("--time filter to files within X days") + print("--force force retrieval without pinging, will run only once") + return + print("Starting...") + while True: + pingRes = check_ip_alive(botIP) + if pingRes: + print("Found machine:", botIP) + retrieveLogs() + time.sleep(afterFoundDelay) + else: + print(f"Unable to ping address: {botIP}") + time.sleep(watchdogDelay) +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\nMonitoring stopped by user") diff --git a/docker-services/ingest/todo.txt b/docker-services/ingest/todo.txt new file mode 100644 index 0000000..e94ac7a --- /dev/null +++ b/docker-services/ingest/todo.txt @@ -0,0 +1,2 @@ +- watchdog to download logs +- file importer and metrics gen diff --git a/drive-backup.py b/drive-backup.py new file mode 100644 index 0000000..3f2e6d1 --- /dev/null +++ b/drive-backup.py @@ -0,0 +1,105 @@ +import subprocess +import sys +import time +import re + +drive_dir = "/home/ignite/GoogleDrive/" +flashpoint_dir = "/home/ignite/workspace/flashpoint/" + +compID_regex = r"[a-zA-Z][a-zA-Z][a-zA-Z][a-zA-Z][a-zA-Z]?[0-9]?_" # matches for any 4/5 letters, optional number, and underscore after + +invalids = [ + "FRC", "TBD", "FF", "rio" +] + +def backupLogs(): + list_files = [ + "ls", "-1", # list all files one per line + flashpoint_dir+"telemetry/" + ] + listRes = subprocess.run(list_files, capture_output=True).stdout.decode() + + for fileName in listRes.splitlines(): + print("File: "+fileName, end=" ") + + tmpFileName = fileName + for index in range(len(invalids)): + tmpFileName.replace(invalids[index], "") + + compID_match = re.search(compID_regex, tmpFileName) + drive_teledir = "" + if compID_match: + compID = "" + for i in range(compID_match.start(), compID_match.end()-1): # -1 removes underscore from matching + compID = compID+tmpFileName[i] + drive_teledir = drive_dir+"Programming/Telemetry/LandingZone/"+compID + print("Comp id: "+compID) + else: + drive_teledir = drive_dir+"Programming/Telemetry/LandingZone/nocomp" + print("Comp id: none") + + make_folder = [ + "mkdir", "-p", # make folder, all parent directories, and don't complain if existing + drive_teledir, + ] + mkdirRes = subprocess.run(make_folder, capture_output=True).stdout.decode() + if mkdirRes != "": + print(repr(mkdirRes)) # repr to force printing non-printing chars, such as newlines + + move_file = [ + "cp", flashpoint_dir+"telemetry/"+fileName, drive_teledir # move file to drive_directory + ] + moveCMD = subprocess.run(move_file, capture_output=True) + moveRes = moveCMD.stdout.decode() + moveErr = moveCMD.stdout.decode() + if moveRes != "": + print(repr(moveRes)) # repr to force printing non-printing chars, such as newlines + +def backupDB(): + print("Copying robot.db file...") + drive_db_dir = drive_dir+"Programming/Telemetry/LandingZone/dbbackups/" + + LT = time.localtime() + strDate = str(LT.tm_year)+str(LT.tm_mon)+str(LT.tm_mday) + strTime = str(LT.tm_hour)+str(LT.tm_min)+str(LT.tm_sec) + #print(strDate+"_"+strTime) + + fullNewFilePath = drive_db_dir+"backup_"+strDate+"_"+strTime+".db" + + copy_file = [ + "cp", flashpoint_dir+"db/robot.db", # copy the robot db file + fullNewFilePath + ] + + copyCMD = subprocess.run(copy_file, capture_output=True) + copyRes = copyCMD.stdout.decode() + copyErr = copyCMD.stderr.decode() + + if copyRes != "": + print(copyRes) + if copyErr != "": + print(copyErr) + +def main(): + print("Starting...") + if len(sys.argv) > 1: + if "--only-db" in sys.argv: + backupDB() + return + elif "--only-logs" in sys.argv: + backupLogs() + return + else: + backupLogs() + backupDB() + elif len(sys.argv) == 2: + print("Usage: drive-backup.py [OPTIONS]\n") + print("Options:\n") + print("--db backup database (robot.db) file") + return + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\nMonitoring stopped by user") \ No newline at end of file diff --git a/ingest-requirements.txt b/ingest-requirements.txt new file mode 100644 index 0000000..ceb1441 --- /dev/null +++ b/ingest-requirements.txt @@ -0,0 +1,2 @@ +pandas==2.2.3 +msgpack==1.1.0 diff --git a/ingest_dir.sh b/ingest_dir.sh new file mode 100755 index 0000000..b32a0f0 --- /dev/null +++ b/ingest_dir.sh @@ -0,0 +1,4 @@ +for file in telemetry/*.wpilog +do + python ingest_file.py "$file" db/robot.db +done diff --git a/setup-db.py b/setup-db.py new file mode 100644 index 0000000..53525d4 --- /dev/null +++ b/setup-db.py @@ -0,0 +1,3 @@ +from ingest_file import setup_db +c = setup_db("db/robot.db") +c.close() \ No newline at end of file diff --git a/viz-requirements.txt b/viz-requirements.txt new file mode 100644 index 0000000..da3364d --- /dev/null +++ b/viz-requirements.txt @@ -0,0 +1,4 @@ +pygwalker==0.4.9.15 +tornado==6.5.1 +streamlit==1.46.1 +msgpack==1.1.0 From f4eccf4adad1abb5929d5d7a3696e64009b18d73 Mon Sep 17 00:00:00 2001 From: William Crouch Date: Sat, 25 Oct 2025 14:09:40 -0400 Subject: [PATCH 2/3] Revert "Merge branch 'main' into feature/hoot-support" This reverts commit 4d8a0ab280bbbb3702feb545bfec25c6d4919fca, reversing changes made to 229adb75277884ffacea69a565306e7d0d6e0824. --- .dockerignore | 6 - .gitignore | 4 - docker-compose.yml | 37 ---- docker-services/dataviz/Dockerfile | 18 -- docker-services/dataviz/src/main.py | 1 - docker-services/dataviz/todo.txt | 2 - docker-services/ingest/Dockerfile | 23 --- docker-services/ingest/build.sh | 9 - .../ingest/flashpoint-ingest.service | 11 -- docker-services/ingest/newcontainer.sh | 9 - docker-services/ingest/requirements.txt | 1 - docker-services/ingest/src/main.py | 162 ------------------ docker-services/ingest/todo.txt | 2 - drive-backup.py | 105 ------------ ingest-requirements.txt | 2 - ingest_dir.sh | 4 - setup-db.py | 3 - viz-requirements.txt | 4 - 18 files changed, 403 deletions(-) delete mode 100644 .dockerignore delete mode 100644 docker-compose.yml delete mode 100644 docker-services/dataviz/Dockerfile delete mode 120000 docker-services/dataviz/src/main.py delete mode 100644 docker-services/dataviz/todo.txt delete mode 100644 docker-services/ingest/Dockerfile delete mode 100755 docker-services/ingest/build.sh delete mode 100644 docker-services/ingest/flashpoint-ingest.service delete mode 100755 docker-services/ingest/newcontainer.sh delete mode 100644 docker-services/ingest/requirements.txt delete mode 100755 docker-services/ingest/src/main.py delete mode 100644 docker-services/ingest/todo.txt delete mode 100644 drive-backup.py delete mode 100644 ingest-requirements.txt delete mode 100755 ingest_dir.sh delete mode 100644 setup-db.py delete mode 100644 viz-requirements.txt diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index a829363..0000000 --- a/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -**/docker/ -**/telemetry/ -Dockerfile-ingest -docker-compose.yml -.venv -.env \ No newline at end of file diff --git a/.gitignore b/.gitignore index 608681b..e39c48d 100644 --- a/.gitignore +++ b/.gitignore @@ -122,8 +122,6 @@ celerybeat.pid # Environments .env .venv -.ingest-venv -.viz-venv env/ venv/ ENV/ @@ -165,6 +163,4 @@ cython_debug/ data/ converted_data/ db/ -telemetry/ -faketele/ old_2024_data/ diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index ead6ff2..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,37 +0,0 @@ -services: - ingest: - image: flashpoint-ingest - build: - dockerfile: ./docker-services/ingest/Dockerfile - volumes: - - telemetry:/app/telemetry - - database:/app/db - - convertedData:/app/converted_data - dataviz: - image: flashpoint-dataviz - ports: - - "8501:8501" - build: - dockerfile: ./docker-services/dataviz/Dockerfile - volumes: - - database:/app/db/ - - convertedData:/app/converted_data/ - -# if it failed to mount _data, make sure FLASHPOINT_DIR is correct and there is a telemetry, db, and converted_data directory -volumes: - telemetry: - driver_opts: - type: "none" - o: "bind" - device: "${FLASHPOINT_DIR}/telemetry" - database: - driver_opts: - type: "none" - o: "bind" - device: "${FLASHPOINT_DIR}/db" - convertedData: - driver_opts: - type: "none" - o: "bind" - device: "${FLASHPOINT_DIR}/converted_data" - \ No newline at end of file diff --git a/docker-services/dataviz/Dockerfile b/docker-services/dataviz/Dockerfile deleted file mode 100644 index 5d9a913..0000000 --- a/docker-services/dataviz/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM python:3.12.3-slim-bullseye - -#RUN apt-get update && apt-get install -y --no-install-recommends iputils-ping sshpass ssh -#RUN rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -COPY . . - -RUN pip install --no-cache-dir -r docker-services/dataviz/requirements.txt -RUN pip install --no-cache-dir -r viz-requirements.txt - -RUN mkdir /app/db/ -RUN mkdir /app/converted_data/ - -EXPOSE 8501 -# eventually change to python3 docker-services/dataviz/src/main.py -CMD ["streamlit", "run", "viz.py"] \ No newline at end of file diff --git a/docker-services/dataviz/src/main.py b/docker-services/dataviz/src/main.py deleted file mode 120000 index ebdca85..0000000 --- a/docker-services/dataviz/src/main.py +++ /dev/null @@ -1 +0,0 @@ -/home/ubuntu/Documents/flashpoint/viz.py \ No newline at end of file diff --git a/docker-services/dataviz/todo.txt b/docker-services/dataviz/todo.txt deleted file mode 100644 index 00e84c4..0000000 --- a/docker-services/dataviz/todo.txt +++ /dev/null @@ -1,2 +0,0 @@ -- pre-defined data viz app -- pygwalker app for data browsing diff --git a/docker-services/ingest/Dockerfile b/docker-services/ingest/Dockerfile deleted file mode 100644 index 57af930..0000000 --- a/docker-services/ingest/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM python:3.12.3-slim-bullseye - -RUN apt-get update && apt-get install -y --no-install-recommends sshpass ssh -RUN rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -COPY . . - -RUN pip install --no-cache-dir -r docker-services/ingest/requirements.txt - -#RUN ls -lag && sleep 15 -RUN pip install --no-cache-dir -r ingest-requirements.txt - -RUN mkdir /app/telemetry/ -RUN mkdir /app/converted_data/ -RUN mkdir /app/db/ - -EXPOSE 22 - -RUN python3 setup-db.py - -CMD ["python", "docker-services/ingest/src/main.py"] \ No newline at end of file diff --git a/docker-services/ingest/build.sh b/docker-services/ingest/build.sh deleted file mode 100755 index 6c46def..0000000 --- a/docker-services/ingest/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -docker volume create --driver local \ - -o o=bind \ - -o type=none \ - -o device="/home/ubuntu/Ignite/flashpoint-docker/telemetry" \ - flashpoint-telemetry - -docker build -t flashpoint-ingest . - diff --git a/docker-services/ingest/flashpoint-ingest.service b/docker-services/ingest/flashpoint-ingest.service deleted file mode 100644 index 3d6a463..0000000 --- a/docker-services/ingest/flashpoint-ingest.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=A Watchdog Ingest Docker container that will wait for the robot, the unload the files - -[Service] -Type=simple -ExecStart=export FLASHPOINT_DIR="/home/ignite/workspace/flashpoint/" && /usr/bin/docker compose up $FLASHPOINT_DIR -Restart=always -RestartSec=10 - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/docker-services/ingest/newcontainer.sh b/docker-services/ingest/newcontainer.sh deleted file mode 100755 index 164c0c5..0000000 --- a/docker-services/ingest/newcontainer.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -docker create -it \ ---volume /var/lib/docker/volumes/flashpoint-telemetry:/app/telemetry/ \ ---name flashpoint-ingest flashpoint-ingest - -##!/bin/bash -#docker create -it \ # keep shell open even when no activity -#--volume /var/lib/docker/volumes/flashpoint-telemetry:/app/telemetry/ \ # specify volume:mount-point -#--name flashpoint-ingest flashpoint-ingest \ No newline at end of file diff --git a/docker-services/ingest/requirements.txt b/docker-services/ingest/requirements.txt deleted file mode 100644 index f24de7f..0000000 --- a/docker-services/ingest/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -ping3==5.1.5 \ No newline at end of file diff --git a/docker-services/ingest/src/main.py b/docker-services/ingest/src/main.py deleted file mode 100755 index 5cfa42b..0000000 --- a/docker-services/ingest/src/main.py +++ /dev/null @@ -1,162 +0,0 @@ -import subprocess -import time -import sys -from ping3 import ping # type: ignore - -botIP = "10.68.29.2" -botUser = "lvuser" -botPass = "" -teleDir = "/home/lvuser/logs" -allDir = teleDir+"/*" -botHostname = botUser+"@"+botIP -watchdogDelay = 5 -afterFoundDelay = 300 -timeSlot = 1 - -SPACESEPARATOR = " " - - -def check_ip_alive(ip_address): - try: - response = ping(ip_address, timeout=1) - return response is not None - except Exception: - return False - -passPrefix = [ - "sshpass", "-p", botPass, -] - -sshPrefix = passPrefix + [ - "ssh", botHostname, # login - "-o StrictHostKeyChecking=no", # no "check fingerprint" message/error - "-o UserKnownHostsFile=/dev/null", # don't save fingerprint -] - -getDate = [ - "date", # get date - "+%Y%m%d" # format into YYYYMMDD -] - -def retrieveLogs(): - date = int(subprocess.run(getDate, capture_output=True).stdout.decode()) - - listFiles = sshPrefix + [ - "ls", # list all files - "-1t", # one per line, in time order (newest first) - teleDir, # directory - ] - grepFilter = listFiles + ["|", - "find", # find all files - "|", # and then - "grep", f"*{date-timeSlot}" # filter to only files within specified days - ] - - lsRes = "" - lsErr = "" - if timeSlot == 0: - lsCmd = subprocess.run(listFiles, capture_output=True) - lsRes = lsCmd.stdout.decode() - lsErr = lsCmd.stderr.decode() - else: - lsCmd = subprocess.run(grepFilter, capture_output=True) - lsRes = lsCmd.stdout.decode() - lsErr = lsCmd.stderr.decode() - - - if len(lsErr.splitlines()) > 1: - print("Something failed: "+lsErr) - print("Logs: "+lsRes) - return - - if len(lsRes.splitlines()) == 0 and timeSlot != 0: - print(f"\nFailed to find any logs within {timeSlot} days") - return - elif len(lsRes.splitlines()) == 0 and timeSlot == 0: - print(f"There appears to be no logs in this directory on the host: {teleDir}") - return - - for line in lsRes.splitlines(): - print(line) - scpSuccessFlag = False - - scpCommand = passPrefix + [ - "scp", - "-o StrictHostKeyChecking=no", # no "check fingerprint" message/error - "-o UserKnownHostsFile=/dev/null", # don't save fingerprint - botHostname+":"+teleDir+"/"+line, "/app/telemetry/", # take one user@ip:/path/to/logs/ file and store in /app/telemetry/ - ] - - removeCommand = [ - "ssh", botHostname, # login - "-o StrictHostKeyChecking=no", # no "check fingerprint" message/error - "-o UserKnownHostsFile=/dev/null", # don't save fingerprint - "rm", "-f", f"{teleDir+"/"+line}", # delete copied file - ] - - res = subprocess.run(scpCommand, capture_output=True).stderr.decode() - scpRes = "" - print("Downloading file: "+line) - if "No such file or directory" in res: - scpRes = "Error: " + res + f"\n - {scpCommand}" - else: - scpRes = res - scpSuccessFlag = True - print("Retrieving logs:", scpRes) - - finalRemoveRes = "" - if scpSuccessFlag: - removeRes = subprocess.run(removeCommand, capture_output=True).stderr.decode() - if "No such file or directory" in removeRes: - finalRemoveRes = "Error: " + removeRes + f"\n - {removeCommand}" - elif removeRes is None: - finalRemoveRes = "null" - else: - finalRemoveRes = removeRes - print("Removing old logs:", finalRemoveRes) - print("\n\n") # separate - - # start ingest - print("Starting Ingest on files in ./telemetry") - ingestCMD = subprocess.run(["python3", "./ingest_dir.sh"], capture_output=True) - ingestRes = ingestCMD.stdout.decode() - ingestErr = ingestCMD.stderr.decode() - print(ingestRes) - print(ingestErr) - - """for fileName in lsRes.splitlines(): - rmCMD = subprocess.run(["rm", "./telemetry/"+fileName], capture_output=True) - rmRes = rmCMD.stdout.decode() - rmErr = rmCMD.stderr.decode() - print(rmRes) - print(rmErr)""" - -def main(): - global timeSlot - if len(sys.argv) > 1: - if "--time" in sys.argv and len(sys.argv) > 2: - timeSlot = int(sys.argv[2]) - if "--force" in sys.argv: - retrieveLogs() - return - elif len(sys.argv) == 2: - print("Usage: main.py [OPTIONS]\n") - print("Options:\n") - print("--time filter to files within X days") - print("--force force retrieval without pinging, will run only once") - return - print("Starting...") - while True: - pingRes = check_ip_alive(botIP) - if pingRes: - print("Found machine:", botIP) - retrieveLogs() - time.sleep(afterFoundDelay) - else: - print(f"Unable to ping address: {botIP}") - time.sleep(watchdogDelay) -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - print("\nMonitoring stopped by user") diff --git a/docker-services/ingest/todo.txt b/docker-services/ingest/todo.txt deleted file mode 100644 index e94ac7a..0000000 --- a/docker-services/ingest/todo.txt +++ /dev/null @@ -1,2 +0,0 @@ -- watchdog to download logs -- file importer and metrics gen diff --git a/drive-backup.py b/drive-backup.py deleted file mode 100644 index 3f2e6d1..0000000 --- a/drive-backup.py +++ /dev/null @@ -1,105 +0,0 @@ -import subprocess -import sys -import time -import re - -drive_dir = "/home/ignite/GoogleDrive/" -flashpoint_dir = "/home/ignite/workspace/flashpoint/" - -compID_regex = r"[a-zA-Z][a-zA-Z][a-zA-Z][a-zA-Z][a-zA-Z]?[0-9]?_" # matches for any 4/5 letters, optional number, and underscore after - -invalids = [ - "FRC", "TBD", "FF", "rio" -] - -def backupLogs(): - list_files = [ - "ls", "-1", # list all files one per line - flashpoint_dir+"telemetry/" - ] - listRes = subprocess.run(list_files, capture_output=True).stdout.decode() - - for fileName in listRes.splitlines(): - print("File: "+fileName, end=" ") - - tmpFileName = fileName - for index in range(len(invalids)): - tmpFileName.replace(invalids[index], "") - - compID_match = re.search(compID_regex, tmpFileName) - drive_teledir = "" - if compID_match: - compID = "" - for i in range(compID_match.start(), compID_match.end()-1): # -1 removes underscore from matching - compID = compID+tmpFileName[i] - drive_teledir = drive_dir+"Programming/Telemetry/LandingZone/"+compID - print("Comp id: "+compID) - else: - drive_teledir = drive_dir+"Programming/Telemetry/LandingZone/nocomp" - print("Comp id: none") - - make_folder = [ - "mkdir", "-p", # make folder, all parent directories, and don't complain if existing - drive_teledir, - ] - mkdirRes = subprocess.run(make_folder, capture_output=True).stdout.decode() - if mkdirRes != "": - print(repr(mkdirRes)) # repr to force printing non-printing chars, such as newlines - - move_file = [ - "cp", flashpoint_dir+"telemetry/"+fileName, drive_teledir # move file to drive_directory - ] - moveCMD = subprocess.run(move_file, capture_output=True) - moveRes = moveCMD.stdout.decode() - moveErr = moveCMD.stdout.decode() - if moveRes != "": - print(repr(moveRes)) # repr to force printing non-printing chars, such as newlines - -def backupDB(): - print("Copying robot.db file...") - drive_db_dir = drive_dir+"Programming/Telemetry/LandingZone/dbbackups/" - - LT = time.localtime() - strDate = str(LT.tm_year)+str(LT.tm_mon)+str(LT.tm_mday) - strTime = str(LT.tm_hour)+str(LT.tm_min)+str(LT.tm_sec) - #print(strDate+"_"+strTime) - - fullNewFilePath = drive_db_dir+"backup_"+strDate+"_"+strTime+".db" - - copy_file = [ - "cp", flashpoint_dir+"db/robot.db", # copy the robot db file - fullNewFilePath - ] - - copyCMD = subprocess.run(copy_file, capture_output=True) - copyRes = copyCMD.stdout.decode() - copyErr = copyCMD.stderr.decode() - - if copyRes != "": - print(copyRes) - if copyErr != "": - print(copyErr) - -def main(): - print("Starting...") - if len(sys.argv) > 1: - if "--only-db" in sys.argv: - backupDB() - return - elif "--only-logs" in sys.argv: - backupLogs() - return - else: - backupLogs() - backupDB() - elif len(sys.argv) == 2: - print("Usage: drive-backup.py [OPTIONS]\n") - print("Options:\n") - print("--db backup database (robot.db) file") - return - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - print("\nMonitoring stopped by user") \ No newline at end of file diff --git a/ingest-requirements.txt b/ingest-requirements.txt deleted file mode 100644 index ceb1441..0000000 --- a/ingest-requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pandas==2.2.3 -msgpack==1.1.0 diff --git a/ingest_dir.sh b/ingest_dir.sh deleted file mode 100755 index b32a0f0..0000000 --- a/ingest_dir.sh +++ /dev/null @@ -1,4 +0,0 @@ -for file in telemetry/*.wpilog -do - python ingest_file.py "$file" db/robot.db -done diff --git a/setup-db.py b/setup-db.py deleted file mode 100644 index 53525d4..0000000 --- a/setup-db.py +++ /dev/null @@ -1,3 +0,0 @@ -from ingest_file import setup_db -c = setup_db("db/robot.db") -c.close() \ No newline at end of file diff --git a/viz-requirements.txt b/viz-requirements.txt deleted file mode 100644 index da3364d..0000000 --- a/viz-requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -pygwalker==0.4.9.15 -tornado==6.5.1 -streamlit==1.46.1 -msgpack==1.1.0 From d5c06d6faa54db7becdc41b83d558466ffa3981c Mon Sep 17 00:00:00 2001 From: William Crouch Date: Sat, 25 Oct 2025 14:14:19 -0400 Subject: [PATCH 3/3] code used to create GRITS.db --- .gitignore | 1 + ingest_library.py | 2 +- manage_imports.py | 38 ++++++++++++++++++++++++++------------ 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index e39c48d..3081626 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,4 @@ data/ converted_data/ db/ old_2024_data/ +imported_files/ diff --git a/ingest_library.py b/ingest_library.py index bb0322d..51793e8 100644 --- a/ingest_library.py +++ b/ingest_library.py @@ -33,7 +33,7 @@ def update_file_metadata(connection, filename, hash, success): connection.rollback() def read_device_logfile(filepath): - if 'rio' in os.basename(filepath): + if 'rio' in os.path.basename(filepath): convert_folder = 'converted_rio_device_logs' else: convert_folder = 'converted_drive_device_logs' diff --git a/manage_imports.py b/manage_imports.py index 47094a1..14d33f6 100644 --- a/manage_imports.py +++ b/manage_imports.py @@ -1,29 +1,38 @@ import os import shutil +import sys + + +from ingest_match_logs import ingest_match_logs def manage_imports(): print("Managing imports") import_list = os.listdir("./imported_files") - + import_system_list = [] import_drive_list = [] import_rio_list = [] - + + os.makedirs("./data", exist_ok=True) + os.makedirs("./data/system_logs", exist_ok = True) + os.makedirs("./data/rio_device_logs", exist_ok = True) + os.makedirs("./data/drive_device_logs", exist_ok = True) + for file in import_list: pos = file.rfind(".") - if file[pos:] is "wpilog": + if file[pos:] == ".wpilog": import_system_list.append(file) - shutil.copy("./data/imported_files/" + file, "./data/system_logs") - elif file[pos:] is "hoot": + shutil.copy("./imported_files/" + file, "./data/system_logs") + elif file[pos:] == ".hoot": if "rio" in file: import_rio_list.append(file) - shutil.copy("./data/imported_files/" + file, "./data/rio_device_logs") + shutil.copy("./imported_files/" + file, "./data/rio_device_logs") else: import_drive_list.append(file) - shutil.copy("./data/imported_files/" + file, "./data/drive_device_logs") + shutil.copy("./imported_files/" + file, "./data/drive_device_logs") else: print("Unrecognized filetype is in import directory!") - os.remove("./imported_files/" + file) - + #os.remove("./imported_files/" + file) + #TODO handle exceptions ingest_dict = {} for file in import_system_list: @@ -35,8 +44,13 @@ def manage_imports(): for file in import_rio_list: match_id = file.split("_")[1] ingest_dict[match_id].append("./data/rio_device_logs/"+file) - + return ingest_dict - + if __name__ == '__main__': - manage_imports() \ No newline at end of file + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + ingest_dict = manage_imports() + for match in list(ingest_dict.values()): + ingest_match_logs(match[0], match[1], match[2],sys.argv[1]) \ No newline at end of file