Merge dev into testmove

.gitignore

@@ -1,15 +1,3 @@
-# ukmon specific
-ukmon.ini
-live.key
-archive.key
-addSftpUser.sh
-.firstrun
-domp4s
-extrascript
-dotimelapse
-.config.backup
-ukmon.ini
-
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -139,6 +127,18 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# ukmon specific
+ukmon.ini
+live.key
+archive.key
+addSftpUser.sh
+.firstrun
+domp4s
+extrascript
+dotimelapse
+.config.backup
+cameras.ini
 deploy.yml
 sample-crontab-entries.txt
 jsonToKeyFile.py

bumpver.toml

@@ -0,0 +1,26 @@
+[bumpver]
+current_version = "2026.01.04"
+version_pattern = "YYYY.MM.PATCH"
+commit_message = "bump version {old_version} -> {new_version}"
+tag_message = "{new_version}"
+tag_scope = "default"
+pre_commit_hook = ""
+post_commit_hook = ""
+commit = true
+tag = true
+push = true
+
+[bumpver.file_patterns]
+"fireballCollector.py" = [
+    "versionid = '{version}'"
+]
+"README.md" = [
+    'Version {version}',
+]
+"bumpver.toml" = [
+    'current_version = "{version}"',
+]
+#".github/workflows/build_tools.yml" = [
+#    'tag_name: {version}',
+#]
+

gatherLogs.sh

@@ -1,29 +1,61 @@
 #!/bin/bash
-cd ~/RMS_data/logs
-mkdir logtmp
-cd logtmp
-
 #
 # Script to gather the logfiles and upload them for debugging & analysis
 # Copyright (C) 2018-2023 Mark McIntyre
 #
-source /home/${LOGNAME}/source/ukmon-pitools/ukmon.ini
+myself=$(readlink -f $0)
+here="$( cd "$(dirname "$myself")" >/dev/null 2>&1 ; pwd -P )"
+
+[ "$1" == "" ] && echo "usage: ./gatherlogs.sh UKxxxxx" && exit
+CAMID=$1
+
+cd $HOME
+[ -d logtmp_$CAMID ] && rm -Rf logtmp_$CAMID
+mkdir -p logtmp_$CAMID
+cd logtmp_$CAMID
+
+source $here/ukmon.ini
+
+echo "gathering files"
+# gather system logs
 sudo cp /var/log/kern.log .
-sudo chown ${user}:${LOGNAME} kern.log
-cp /var/log/messages ./messages.log
-cp /home/${LOGNAME}/source/RMS/.config ./${LOCATION}.config
-cp /home/${LOGNAME}/source/RMS/platepar_cmn2010.cal ./${LOCATION}.cal
-cp /home/${LOGNAME}/source/ukmon-pitools/*.key .
-cp /home/${LOGNAME}/source/ukmon-pitools/*.ini .
+[ -f /var/log/messages ] && sudo cp /var/log/messages ./messages.log; 
+[ -f /var/log/syslog ] && sudo cp /var/log/syslog ./messages.log
+sudo chown ${LOGNAME}:${LOGNAME} ./*.log
+
+# find the RMS config and log location
+if [ -d /home/${LOGNAME}/source/Stations/${CAMID} ] ; then
+    rootdir=/home/${LOGNAME}/source/Stations/${CAMID}
+else
+    rootdir=/home/${LOGNAME}/source/RMS
+fi 
+echo rootdir is $rootdir
+rmscfg=$rootdir/.config
+datadir=$(python -c "import configparser,os;cfg=configparser.ConfigParser();cfg.read('$rmscfg');print(os.path.expanduser(cfg['Capture']['data_dir']))")
+logdir=$datadir/logs
+echo logdir is $logdir
+[ ! -d $logdir ] && logdir=~/RMS_data/logs
+
+# gather the RMS config and logs
+cp $rootdir/.config ./${CAMID}.config
+cp $rootdir/platepar_cmn2010.cal ./${CAMID}.cal
+cp $here/live.key ./${CAMID}.key
+cp $here/ukmon.ini .
+[ -f $here/cameras.ini ] && cp $here/cameras.ini .
 crontab -l > ./crontab.txt
-find  .. -maxdepth 1 -name "*.log*" -type f -mtime -5 -exec cp {} . \;
-ZIPFILE=/tmp/${LOCATION}_logs.tgz
-tar cvzf $ZIPFILE *.log* ${LOCATION}.config ${LOCATION}.cal crontab.txt *.key *.ini
+ls -1tr $logdir/log*.log* | tail -5 | while read i; do cp $i . ; done
+ls -1tr $logdir/uk*.log* | tail -5 | while read i; do cp $i . ; done
+
+# create a tarball and upload to the server
+echo "uploading logs"
+ZIPFILE=/tmp/${CAMID}_logs.tgz
+tar czf $ZIPFILE *.log* ${CAMID}.config ${CAMID}.cal crontab.txt *.key *.ini
 sftp -i $UKMONKEY -q logupload@$UKMONHELPER << EOF
 cd logs
 progress
 put $ZIPFILE 
 exit
 EOF
 cd ..
-rm -Rf logtmp
+rm -Rf logtmp_$CAMID
+echo "done"

liveMonitor.py

@@ -8,12 +8,13 @@
 import logging
 import RMS.ConfigReader as cr
 from stat import ST_INO
-from uploadToArchive import readKeyFile, readIniFile
-from ukmonPostProc import setupLogging
+from uploadToArchive import readKeyFile, readIniFile, getLatestKeys
+from ukmonPostProc import setupLogging, versionid
 
 
 log = logging.getLogger("ukmonlogger")
 
+
 timetowait = 30 # seconds to wait for a new line before deciding the log is stale
 
 # Images created more than this many seconds ago won't be uploaded. Prevents reuploads. 
@@ -47,35 +48,43 @@ def follow(fname, logf_ino):
             yield line.strip()
 
 
-def monitorLogFile(camloc, rmscfg):
+def monitorLogFile(camloc, stationid=None):
     """ This function monitors the latest RMS log file for meteor detections, convert the FF file
     to a jpg and upload it to the livestream.  
     This function is called from the shell script *liveMonitor.sh* and should not be called directly. 
     """
+    # get configuration
+    myloc = os.path.split(os.path.abspath(__file__))[0]
+    inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid)
+    if not inifvals:
+        log.error('ukmon.ini not present, aborting')
+        exit(1)
+    rmscfg = inifvals['RMSCFG']
+    if not os.path.isfile(rmscfg):
+        log.error('rms config file', rmscfg,'not present, aborting')
+        exit(1)
     cfg = cr.parse(os.path.expanduser(rmscfg))
 
     datadir = cfg.data_dir
     logdir = os.path.expanduser(os.path.join(datadir, cfg.log_dir))
 
     setupLogging(logdir, 'ukmonlive_')
 
-    log.info('--------------------------------')
-    log.info('    live feed started')
-    log.info('--------------------------------')
+    log.info('------------------------------------------')
+    log.info('    live feed started, version ' + versionid)
+    log.info('------------------------------------------')
 
     log.info('Camera location is {}'.format(camloc))
     log.info('RMS config file is {}'.format(rmscfg))
 
-    myloc = os.path.split(os.path.abspath(__file__))[0]
-
     # get credentials
-    inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'))
-    if not inifvals:
-        log.error('ukmon.ini not present, aborting')
-        exit(1)
+    if not os.path.isfile(os.path.join(myloc, 'live.key')):
+        if not getLatestKeys(myloc, cfg.stationID):
+            print('unable to get key for', inifvals['LOCATION'])
+            exit(1)
     keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals)
     if not keys:
-        log.error('config file not present, aborting')
+        log.error('unable to read AWS configuration, aborting')
         exit(1)
 
     keepon = True
@@ -142,11 +151,8 @@ def monitorLogFile(camloc, rmscfg):
 
 if __name__ == '__main__':
     if len(sys.argv) < 2:
-        print('LOCATION missing')
+        print('usage: python liveMonitor.py {ukmon_location} {rms_id}')
         exit(1)
-    if len(sys.argv) < 3:
-        rmscfg = os.path.expanduser('~/source/RMS/.config')
-    else:
-        rmscfg = sys.argv[2]
     camloc = sys.argv[1]
-    monitorLogFile(camloc, rmscfg)
+    stationid = sys.argv[2] if len(sys.argv) > 2 else None
+    monitorLogFile(camloc, stationid)

liveMonitor.sh

@@ -9,17 +9,21 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 
 # kill any existing livestream process
 pids=$(ps -ef | grep ${here}/liveMonitor | egrep -v "grep|$$" | awk '{print $2}')
-kill -9 $pids
+[ "$pids" != "" ] && kill -9 $pids
 
 source $here/ukmon.ini
-source $here/live.key
 
-if [ "$LOCATION" == "NOTCONFIGURED" ]; then
-    echo "station not configured, unable to continue" 
-    exit 1
-fi
-
-rmsdir=$(dirname $RMSCFG)
-cd $rmsdir
+cd ~/source/RMS
 export PYTHONPATH=$here:~/source/RMS
-python $here/liveMonitor.py $LOCATION $RMSCFG 
+if [ -f $here/cameras.ini ] ; then
+    cat $here/cameras.ini | grep = | while read i 
+    do 
+        python $here/liveMonitor.py ${i:7:60} ${i:0:6} &
+    done    
+else
+    if [ "$LOCATION" == "NOTCONFIGURED" ]; then
+        echo "station not configured, unable to continue" 
+        exit 1
+    fi
+    python $here/liveMonitor.py $LOCATION &
+fi

refreshTools.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # refresh ukmon-pitools
-# Copyright (C) 2018-2023 Mark McIntyre
+# Copyright (C) 2018- Mark McIntyre
 
 myself=$(readlink -f $0)
 here="$( cd "$(dirname "$myself")" >/dev/null 2>&1 ; pwd -P )"
@@ -22,75 +22,15 @@ python -c "import ukmonInstaller as pp ; pp.validateIni('${here}', '3.11.55.160'
 source $here/ukmon.ini
 
 echo "refreshing toolset"
+git config pull.ff only 
 git stash 
 git pull
 git stash apply
 
-# creating an ssh key if not already present
-if [ ! -f  ${UKMONKEY} ] ; then 
-    echo "creating ukmon ssh key"
-    ssh-keygen -t rsa -f ${UKMONKEY} -q -N ''
-    echo "Copy this public key and email it to newcamera@ukmeteornetwork.org, then "
-    echo "wait for confirmation its been installed and rerun this script"
-    echo ""
-    cat ${UKMONKEY}.pub
-    echo ""
-    read -p "Press any key to continue"
-fi
-
-# if the station is configured, retrieve the AWS keys and test connectivity. 
-if [[ "$LOCATION" != "NOTCONFIGURED"  && "$LOCATION" != "" ]] ; then
-    # check if RMS is still updating - its taking longer and longer
-    loopctr=0
-    echo "Checking RMS update not in progress"
-    while [ $loopctr -lt 10 ] ; do
-            [ -f $RMSCFG ] && break
-            echo "RMS update in progress or station not configured, trying again in a minute"
-            sleep 60
-            loopctr=$((loopctr + 1))
-    done
-    while [ $loopctr -lt 10 ] ; do
-            grep XX0001 $RMSCFG | grep stationID:
-            [ $? -eq 1 ] && break
-            echo "RMS update in progress or station not configured, trying again in a minute"
-            sleep 60
-            loopctr=$((loopctr + 1))
-    done
-    if [ $loopctr -eq 10 ] ; then
-            echo RMS update failed or long-running, unable to proceed
-            exit 1
-    else
-            echo all good proceeding
-    fi
-    echo "checking for ukmon config changes"
-    python -c "import ukmonInstaller as pp ; pp.getLatestKeys('${here}') ;"
-
-    if [ -f archive.key ] ; then \rm archive.key ; fi 
-
-    echo "checking the RMS config file, crontab and icons"
-    source ~/vRMS/bin/activate
-    source $here/ukmon.ini
-    cd $(dirname $RMSCFG)
-    export PYTHONPATH=$here:~/source/RMS
-    python -c "import ukmonInstaller as pp ; pp.installUkmonFeed('${RMSCFG}');"
-
-    echo "testing connections"
-    python $here/sendToLive.py test test
-    python $here/uploadToArchive.py test
-    echo "if you did not see two success messages contact us for advice" 
-    if [ "$DOCKER_RUNNING" != "true" ] ; then read -p "Press any key to finish" ; fi
-    echo "done"
-else
-    statid=$(grep stationID $RMSCFG | awk -F" " '{print $2}')
-    if [ "$statid" == "XX0001" ] ; then
-        echo "You must configure RMS before setting up the ukmon tools"
-    else
-        python -c "import ukmonInstaller as pp ; pp.addDesktopIcons('${here}', '${statid}');"
-        echo "Location missing - unable to continue. Please obtain a location code from the UKMON team,"
-        echo "then update the UKMON Config File using the desktop icon and rerun this script."
-    fi 
-    sleep 5
-    if [ "$DOCKER_RUNNING" != "true" ] ; then read -p "Press any key to end" ; fi
-    exit 1
-fi
+python -c "from ukmonInstaller import relocateGitRepo;relocateGitRepo()"
+git fetch
 
+echo "testing connections"
+python $here/sendToLive.py test test
+python $here/uploadToArchive.py test
+echo "if you did not see two success messages contact us for advice"