From 58451c2be74035967a3323e4da5b65a92d803c54 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 4 Jan 2026 14:04:51 +0000 Subject: [PATCH 01/50] add versionid --- liveMonitor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/liveMonitor.py b/liveMonitor.py index 629ebd2..3271148 100644 --- a/liveMonitor.py +++ b/liveMonitor.py @@ -14,6 +14,8 @@ log = logging.getLogger("ukmonlogger") +versionid='2026.01.04' + timetowait = 30 # seconds to wait for a new line before deciding the log is stale # Images created more than this many seconds ago won't be uploaded. Prevents reuploads. @@ -60,7 +62,7 @@ def monitorLogFile(camloc, rmscfg): setupLogging(logdir, 'ukmonlive_') log.info('--------------------------------') - log.info(' live feed started') + log.info(f' live feed started, version {versionid}') log.info('--------------------------------') log.info('Camera location is {}'.format(camloc)) From 5db23fc5e1159242a342cce1bbee4cf9e0679f6a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 4 Jan 2026 14:08:40 +0000 Subject: [PATCH 02/50] add version id to logging --- liveMonitor.py | 9 ++++----- ukmonPostProc.py | 5 ++++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/liveMonitor.py b/liveMonitor.py index 3271148..f8257a1 100644 --- a/liveMonitor.py +++ b/liveMonitor.py @@ -9,12 +9,11 @@ import RMS.ConfigReader as cr from stat import ST_INO from uploadToArchive import readKeyFile, readIniFile -from ukmonPostProc import setupLogging +from ukmonPostProc import setupLogging, versionid log = logging.getLogger("ukmonlogger") -versionid='2026.01.04' timetowait = 30 # seconds to wait for a new line before deciding the log is stale @@ -61,9 +60,9 @@ def monitorLogFile(camloc, rmscfg): setupLogging(logdir, 'ukmonlive_') - log.info('--------------------------------') - log.info(f' live feed started, version {versionid}') - log.info('--------------------------------') + log.info('------------------------------------------') + log.info(' live feed started, version ' + versionid) + log.info('------------------------------------------') log.info('Camera location is {}'.format(camloc)) log.info('RMS config file is {}'.format(rmscfg)) diff --git a/ukmonPostProc.py b/ukmonPostProc.py index 094d96f..942c0c2 100644 --- a/ukmonPostProc.py +++ b/ukmonPostProc.py @@ -23,9 +23,12 @@ from uploadToArchive import uploadToArchive, readIniFile + log = logging.getLogger("ukmonlogger") log.setLevel(logging.INFO) +versionid = '2026.01.04' + def setupLogging(logpath, prefix): print('about to initialise logger') @@ -80,7 +83,7 @@ def rmsExternal(cap_dir, arch_dir, config): """ setupLogging(os.path.join(config.data_dir, config.log_dir), f'ukmon_log_{config.stationID}_') - log.info('ukmon external script started') + log.info('ukmon external script started, version ' + versionid) rebootlockfile = os.path.join(config.data_dir, config.reboot_lock_file) with open(rebootlockfile, 'w') as f: From 1fbd91b0139284cc246093c433482039e2b68fb6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 4 Jan 2026 22:58:29 +0000 Subject: [PATCH 03/50] fix to get gatherLogs working on multicam --- gatherLogs.sh | 58 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/gatherLogs.sh b/gatherLogs.sh index 7700d11..ef95d4b 100755 --- a/gatherLogs.sh +++ b/gatherLogs.sh @@ -1,24 +1,52 @@ #!/bin/bash -cd ~/RMS_data/logs -mkdir logtmp -cd logtmp - # # Script to gather the logfiles and upload them for debugging & analysis # Copyright (C) 2018-2023 Mark McIntyre # -source /home/${LOGNAME}/source/ukmon-pitools/ukmon.ini +myself=$(readlink -f $0) +here="$( cd "$(dirname "$myself")" >/dev/null 2>&1 ; pwd -P )" + +[ "$1" == "" ] && echo "usage: ./gatherlogs.sh UKxxxxx" && exit +CAMID=$1 + +cd $HOME +[ -d logtmp_$CAMID ] && rm -Rf logtmp_$CAMID +mkdir -p logtmp_$CAMID +cd logtmp_$CAMID + +source $here/ukmon.ini + +# gather system logs sudo cp /var/log/kern.log . -sudo chown ${user}:${LOGNAME} kern.log -cp /var/log/messages ./messages.log -cp /home/${LOGNAME}/source/RMS/.config ./${LOCATION}.config -cp /home/${LOGNAME}/source/RMS/platepar_cmn2010.cal ./${LOCATION}.cal -cp /home/${LOGNAME}/source/ukmon-pitools/*.key . -cp /home/${LOGNAME}/source/ukmon-pitools/*.ini . +[ -f /var/log/messages ] && sudo cp /var/log/messages ./messages.log; +[ -f /var/log/syslog ] && sudo cp /var/log/syslog ./messages.log +sudo chown ${LOGNAME}:${LOGNAME} ./*.log + +# find the RMS config and log location +if [ -d /home/${LOGNAME}/source/Stations/${CAMID} ] ; then + rootdir=/home/${LOGNAME}/source/Stations/${CAMID} +else + rootdir=/home/${LOGNAME}/source/RMS +fi +rmscfg=$rootdir/.config +datadir=$(python -c "import configparser,os;cfg=configparser.ConfigParser();cfg.read('$rmscfg');print(cfg['Capture']['data_dir'])") +logdir=$datadir/logs +[ ! -d $logdir ] && logdir=~/RMS_data/logs + +# gather the RMS config and logs +cp $rootdir/.config ./${CAMID}.config +cp $rootdir/platepar_cmn2010.cal ./${CAMID}.cal +cp $here/live.key ./${CAMID}.key +cp $here/ukmon.ini . +[ -f $here/cameras.ini ] && cp $here/cameras.ini . crontab -l > ./crontab.txt -find .. -maxdepth 1 -name "*.log*" -type f -mtime -5 -exec cp {} . \; -ZIPFILE=/tmp/${LOCATION}_logs.tgz -tar cvzf $ZIPFILE *.log* ${LOCATION}.config ${LOCATION}.cal crontab.txt *.key *.ini +find $logdir -maxdepth 1 -name "*.log*" -type f -mtime -5 -exec cp {} . \; + +# create a tarball and upload to the server +ZIPFILE=/tmp/${CAMID}_logs.tgz +tar cvzf $ZIPFILE *.log* ${CAMID}.config ${CAMID}.cal crontab.txt *.key *.ini +echo waiting +sleep 10 sftp -i $UKMONKEY -q logupload@$UKMONHELPER << EOF cd logs progress @@ -26,4 +54,4 @@ put $ZIPFILE exit EOF cd .. -rm -Rf logtmp +rm -Rf logtmp_$CAMID From d328479f67c9473507154046542dfa993ecb58d7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 4 Jan 2026 23:15:30 +0000 Subject: [PATCH 04/50] further improvements to gatherlogs --- gatherLogs.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/gatherLogs.sh b/gatherLogs.sh index ef95d4b..f91b18f 100755 --- a/gatherLogs.sh +++ b/gatherLogs.sh @@ -16,6 +16,7 @@ cd logtmp_$CAMID source $here/ukmon.ini +echo "gathering files" # gather system logs sudo cp /var/log/kern.log . [ -f /var/log/messages ] && sudo cp /var/log/messages ./messages.log; @@ -28,9 +29,11 @@ if [ -d /home/${LOGNAME}/source/Stations/${CAMID} ] ; then else rootdir=/home/${LOGNAME}/source/RMS fi +echo rootdir is $rootdir rmscfg=$rootdir/.config -datadir=$(python -c "import configparser,os;cfg=configparser.ConfigParser();cfg.read('$rmscfg');print(cfg['Capture']['data_dir'])") +datadir=$(python -c "import configparser,os;cfg=configparser.ConfigParser();cfg.read('$rmscfg');print(os.path.expanduser(cfg['Capture']['data_dir']))") logdir=$datadir/logs +echo logdir is $logdir [ ! -d $logdir ] && logdir=~/RMS_data/logs # gather the RMS config and logs @@ -40,13 +43,13 @@ cp $here/live.key ./${CAMID}.key cp $here/ukmon.ini . [ -f $here/cameras.ini ] && cp $here/cameras.ini . crontab -l > ./crontab.txt -find $logdir -maxdepth 1 -name "*.log*" -type f -mtime -5 -exec cp {} . \; +ls -1tr $logdir/log*.log* | tail -5 | while read i; do cp $i . ; done +ls -1tr $logdir/uk*.log* | tail -5 | while read i; do cp $i . ; done # create a tarball and upload to the server +echo "uploading logs" ZIPFILE=/tmp/${CAMID}_logs.tgz -tar cvzf $ZIPFILE *.log* ${CAMID}.config ${CAMID}.cal crontab.txt *.key *.ini -echo waiting -sleep 10 +tar czf $ZIPFILE *.log* ${CAMID}.config ${CAMID}.cal crontab.txt *.key *.ini sftp -i $UKMONKEY -q logupload@$UKMONHELPER << EOF cd logs progress @@ -55,3 +58,4 @@ exit EOF cd .. rm -Rf logtmp_$CAMID +echo "done" \ No newline at end of file From e1c88c99f305d4d5b7fa27bb1e2fb3f6b49ec576 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 14:31:11 +0000 Subject: [PATCH 05/50] updates for multicam --- ukmonInstaller.py | 6 +-- ukmonPostProc.py | 31 +++++++-------- uploadToArchive.py | 98 +++++++++++++++++++++++++++++++--------------- 3 files changed, 83 insertions(+), 52 deletions(-) diff --git a/ukmonInstaller.py b/ukmonInstaller.py index 5fac352..60ba7f9 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -304,14 +304,14 @@ def addDesktopIcons(myloc, statid): return -def getLatestKeys(homedir, remoteinifname='ukmon.ini'): +def getLatestKeys(homedir, stationid, remoteinifname='ukmon.ini'): """ Retrieve the latest ini and key files from the ukmon server. If the ini file contains a new server IP or new location, the local copy of the ini file is updated accordingly. """ homedir = os.path.expanduser(os.path.normpath(homedir)) - inifvals = readIniFile(os.path.join(homedir, 'ukmon.ini')) + inifvals = readIniFile(os.path.join(homedir, 'ukmon.ini'), stationid) ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) #try: @@ -357,7 +357,7 @@ def checkPlatepar(homedir, statid, rmsloc): The file is checked for compatability with the station. """ homedir = os.path.expanduser(os.path.normpath(homedir)) - inifvals = readIniFile(os.path.join(homedir, 'ukmon.ini')) + inifvals = readIniFile(os.path.join(homedir, 'ukmon.ini'), statid) ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: diff --git a/ukmonPostProc.py b/ukmonPostProc.py index 942c0c2..5089ec6 100644 --- a/ukmonPostProc.py +++ b/ukmonPostProc.py @@ -90,7 +90,7 @@ def rmsExternal(cap_dir, arch_dir, config): f.write('1') log.info('uploading key science files to archive') - keys = uploadToArchive(arch_dir, sciencefiles=True) + keys = uploadToArchive(arch_dir, config.stationID, sciencefiles=True) # create jpgs from the potential detections log.info('creating JPGs') try: @@ -99,7 +99,7 @@ def rmsExternal(cap_dir, arch_dir, config): bff2i.batchFFtoImage(arch_dir, 'jpg') myloc = os.path.split(os.path.abspath(__file__))[0] - inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), config.stationID) log.info('app home is {}'.format(myloc)) domp4s = 0 if 'DOMP4S' in inifvals: @@ -126,7 +126,7 @@ def rmsExternal(cap_dir, arch_dir, config): log.info('mp4 creation not enabled') log.info('uploading remaining files to archive') - uploadToArchive(arch_dir, keys=keys) + uploadToArchive(arch_dir, config.stationID, keys=keys) # do not remove reboot lock file if running another script # os.remove(rebootlockfile) @@ -177,33 +177,30 @@ def manualRerun(dated_dir, rmscfg = '~/source/RMS/.config'): return rmsExternal(cap_dir, arch_dir, config) -def main(args): - if len(args) < 2: +if __name__ == '__main__': + if len(sys.argv) < 2: print('usage: python ukmonPostProc.py arc_dir_name') print('eg python ukmonPostProc.py UK0006_20210312_183741_206154') print('\n nb: script must be run from RMS source folder') - return False + exit(0) - arch_dir = args[1] + arch_dir = sys.argv[1] + if 'ConfirmedFiles' in arch_dir or 'ArchivedFiles' in arch_dir or 'CapturedFiles' in arch_dir: + _, arch_dir = os.path.split(arch_dir) + stationid = arch_dir.split('_')[0] myloc = os.path.split(os.path.abspath(__file__))[0] - inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) if inifvals is None: print('unable to open ukmon ini file') - return 'unable to open ukmon ini file' + exit(1) try: rmscfg = inifvals['RMSCFG'] except Exception: rmscfg='~/source/RMS/.config' try: - if 'ConfirmedFiles' in arch_dir or 'ArchivedFiles' in arch_dir or 'CapturedFiles' in arch_dir: - _, arch_dir = os.path.split(arch_dir) print('RMS config read from {}'.format(rmscfg)) ret = manualRerun(arch_dir, rmscfg) - return ret + exit(0) except Exception: print('unable to call manualRerun') - return False - - -if __name__ == '__main__': - main(sys.argv) + exit(1) diff --git a/uploadToArchive.py b/uploadToArchive.py index 8da3bda..84e90eb 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -23,6 +23,8 @@ warnings.filterwarnings('ignore', category=CryptographyDeprecationWarning) import paramiko import tempfile +import configparser + from RMS.Formats.FTPdetectinfo import readFTPdetectinfo log = logging.getLogger("ukmonlogger") @@ -112,7 +114,18 @@ def getAWSKey(inifvals): return False, False -def readIniFile(filename): +def readIniFile(filename, stationid): + camerafile = os.path.join(os.path.split(filename)[0],'cameras.ini') + if not os.path.isfile(camerafile): + log.warning('no camerafile, assuming ini file contains location') + location = None + else: + lis = open(camerafile,'r').readlines() + thiscam = [x for x in lis if stationid in x] + if len(thiscam)==0: + log.error('camera {} not in cameras.ini, cannot continue'.format(stationid)) + return False + location = thiscam[0].split('=')[1].strip() if not os.path.isfile(filename): log.error('{} missing, cannot continue'.format(filename)) return False @@ -127,6 +140,12 @@ def readIniFile(filename): data = valstr.split('=') val = data[1].strip().strip('"') vals[data[0]] = val + if location: + vals['LOCATION'] = location + if os.path.isfile(os.path.expanduser('~/.ssh/ukmon_' + stationid)): + vals['UKMONKEY'] = '~/.ssh/ukmon_' + stationid + if os.path.isfile(os.path.expanduser('~/source/Stations/' + stationid + '/.config')): + vals['RMSCFG'] = os.path.expanduser('~/source/Stations/' + stationid + '/.config') return vals @@ -284,11 +303,11 @@ def checkMags(dir_path, ftpfile_name, min_mag): return ff_names -def uploadToArchive(arch_dir, sciencefiles=False, keys=False): +def uploadToArchive(arch_dir, stationid, sciencefiles=False, keys=False): # Upload all relevant files from *arch_dir* to ukmon's S3 Archive myloc = os.path.split(os.path.abspath(__file__))[0] - inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) if not inifvals: return False if not keys: @@ -373,7 +392,17 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False): return keys -def manualUpload(targ_dir, sciencefiles=False): +def getListOfStations(srcdir): + camfile = os.path.join(srcdir, 'cameras.ini') + if not os.path.isfile(camfile): + return ('NOTCONFIGURED','') + camcfg = configparser.ConfigParser() + camcfg.read(camfile) + return camcfg['cameras'].items() + + + +def manualUpload(targ_dir, stationid, sciencefiles=False): """ Manually send the target folder to ukmon archive. Args: @@ -385,29 +414,28 @@ def manualUpload(targ_dir, sciencefiles=False): If the argument is 'test' then a test file is uploaded and the status reported back. """ if targ_dir == 'test': - try: - myloc = os.path.split(os.path.abspath(__file__))[0] - inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) - if not inifvals: - return False - keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) - if not keys: - return False - with open('/tmp/test.txt', 'w') as f: - f.write('{}'.format(inifvals['LOCATION'])) - - target = keys['ARCHBUCKET'] - reg = keys['ARCHREGION'] - conn = boto3.Session(aws_access_key_id=keys['AWS_ACCESS_KEY_ID'], aws_secret_access_key=keys['AWS_SECRET_ACCESS_KEY']) - s3 = conn.resource('s3', region_name=reg) - s3.meta.client.upload_file('/tmp/test.txt', target, 'tmp/{}.txt'.format(keys['CAMLOC'])) - #key = {'Objects': []} - #key['Objects'] = [{'Key': 'test.txt'}] - #s3.meta.client.delete_objects(Bucket=target, Delete=key) - print('test successful') - except Exception: - print('unable to upload to archive - check key information') - return False + myloc = os.path.split(os.path.abspath(__file__))[0] + for cam in getListOfStations(myloc): + try: + stationid = cam[0].upper() + if stationid != 'NOTCONFIGURED': + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) + if not inifvals: + continue + keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) + if not keys: + return False + with open('/tmp/test.txt', 'w') as f: + f.write('{}'.format(inifvals['LOCATION'])) + + target = keys['ARCHBUCKET'] + reg = keys['ARCHREGION'] + conn = boto3.Session(aws_access_key_id=keys['AWS_ACCESS_KEY_ID'], aws_secret_access_key=keys['AWS_SECRET_ACCESS_KEY']) + s3 = conn.resource('s3', region_name=reg) + s3.meta.client.upload_file('/tmp/test.txt', target, 'tmp/{}.txt'.format(keys['CAMLOC'])) + print('test successful for' + stationid) + except Exception: + print('archive upload test failed for {} - check ssh key'.format(stationid)) try: os.remove('/tmp/test.txt') except Exception: @@ -418,14 +446,20 @@ def manualUpload(targ_dir, sciencefiles=False): if not os.path.isdir(arch_dir): print('folder {} not found'.format(arch_dir)) return False - return uploadToArchive(arch_dir, sciencefiles, keys=None) + return uploadToArchive(arch_dir, stationid, sciencefiles, keys=None) if __name__ == '__main__': if len(sys.argv) < 2: - print('usage: python uploadToArchive.py ~/RMS_data/ArchivedFiles/dated_dir') + print('usage: python uploadToArchive.py dated_dir ') + print(' eg: python uploadToArchive.py UK001L_20260104_171228_956526') + print('Optionally include the full path otherwise RMSs ArchivedFiles folder is assumed') exit(0) - targdir = sys.argv[1] - manualUpload(targdir, True) + targdir = os.path.normpath(os.path.expanduser(sys.argv[1])) if targdir != 'test': - manualUpload(targdir) + nightdir = os.path.split(targdir)[1] + stationid = nightdir.split('_')[0] + manualUpload(targdir, stationid, sciencefiles=True) + manualUpload(targdir, stationid, sciencefiles=False) + else: + manualUpload(targdir, None, sciencefiles=True) From 66744f9da73a7eafcb0aac8bcc12fc8c333263b1 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 14:34:44 +0000 Subject: [PATCH 06/50] debugging --- uploadToArchive.py | 1 + 1 file changed, 1 insertion(+) diff --git a/uploadToArchive.py b/uploadToArchive.py index 84e90eb..f5ad0ec 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -422,6 +422,7 @@ def manualUpload(targ_dir, stationid, sciencefiles=False): inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) if not inifvals: continue + print(inifvals) keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) if not keys: return False From 6af2a4ad2a43d64f5de95781172b4dd8d1c713bf Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 15:49:30 +0000 Subject: [PATCH 07/50] add multicam support to ukmonPostProc --- cameras.ini.sample | 6 ++ uploadToArchive.py | 154 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 122 insertions(+), 38 deletions(-) create mode 100644 cameras.ini.sample diff --git a/cameras.ini.sample b/cameras.ini.sample new file mode 100644 index 0000000..09323d8 --- /dev/null +++ b/cameras.ini.sample @@ -0,0 +1,6 @@ +# Sample camera mapping file +# add your location/cameraid mappings to this even if you only have one camera +[cameras] +GMNID_1=location_1 +GMNID_2=location_2 +GMNID_3=location_3 \ No newline at end of file diff --git a/uploadToArchive.py b/uploadToArchive.py index f5ad0ec..2011681 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -27,7 +27,54 @@ from RMS.Formats.FTPdetectinfo import readFTPdetectinfo + log = logging.getLogger("ukmonlogger") +logging.getLogger("paramiko").setLevel(logging.WARNING) + + +def getLatestKeys(homedir, stationid, remoteinifname='ukmon.ini'): + """ + Retrieve the latest ini and key files from the ukmon server. + If the ini file contains a new server IP or new location, the local copy of the + ini file is updated accordingly. + """ + homedir = os.path.expanduser(os.path.normpath(homedir)) + inifvals = readIniFile(os.path.join(homedir, 'ukmon.ini'), stationid) + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + pkey = paramiko.RSAKey.from_private_key_file(os.path.expanduser(inifvals['UKMONKEY'])) + try: + ssh_client.connect(inifvals['UKMONHELPER'], username=inifvals['LOCATION'], pkey=pkey, look_for_keys=False) + ftp_client = ssh_client.open_sftp() + except Exception: + return False + + # get the aws key file + ftp_client.get('live.key', os.path.join(homedir, 'live.key')) + os.chmod(os.path.join(homedir, 'live.key'), 0o600) + + # get the new ini and check for changes + currinif = os.path.join(homedir, 'ukmon.ini') + newinif = os.path.join(homedir, '.ukmon.new') + ftp_client.put(currinif,'ukmon.ini.client') + ftp_client.get(remoteinifname, newinif) + ftp_client.close() + iniflines = open(newinif,'r').readlines() + for li in iniflines: + li = li.strip() + if 'UKMONHELPER' in li: + newhelper = li.split('=')[1] + if newhelper != inifvals['UKMONHELPER']: + updateHelperIp(homedir, newhelper) + print('server address updated') + if 'LOCATION' in li: + newloc = li.split('=')[1] + if newloc != inifvals['LOCATION']: + updateLocation(homedir, newloc) + print('location updated') + os.remove(newinif) + ssh_client.close() + return True def readKeyFile(filename, inifvals): @@ -80,6 +127,36 @@ def readKeyFile(filename, inifvals): return vals +def updateHelperIp(homedir, helperip): + """ + Update the ukmon.ini file with a new IP address if neeeded. + """ + homedir = os.path.normpath(homedir) + lis = open(os.path.join(homedir, 'ukmon.ini'), 'r').readlines() + with open(os.path.join(homedir, 'ukmon.ini'), 'w') as outf: + for li in lis: + if 'UKMONHELPER' in li: + outf.write("export UKMONHELPER={}\n".format(helperip)) + else: + outf.write('{}'.format(li)) + return + + +def updateLocation(homedir, newloc): + """ + Update the ukmon-specific location, if a new one was supplied. Allows us to move cameras to new sites. + """ + homedir = os.path.normpath(homedir) + lis = open(os.path.join(homedir, 'ukmon.ini'), 'r').readlines() + with open(os.path.join(homedir, 'ukmon.ini'), 'w') as outf: + for li in lis: + if 'LOCATION' in li: + outf.write("export LOCATION={}\n".format(newloc)) + else: + outf.write('{}'.format(li)) + return + + def getAWSKey(inifvals): ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -108,29 +185,27 @@ def getAWSKey(inifvals): log.info(e, exc_info=True) ssh_client.close() if key: - log.info('retrieved key details') return key.strip(), sec.strip() else: return False, False def readIniFile(filename, stationid): - camerafile = os.path.join(os.path.split(filename)[0],'cameras.ini') + myloc = os.path.dirname(filename) + camerafile = os.path.join(myloc,'cameras.ini') if not os.path.isfile(camerafile): - log.warning('no camerafile, assuming ini file contains location') location = None else: - lis = open(camerafile,'r').readlines() - thiscam = [x for x in lis if stationid in x] + stations = getListOfStations(myloc) + thiscam = [x for x in stations if stationid.lower() in x[0]] if len(thiscam)==0: log.error('camera {} not in cameras.ini, cannot continue'.format(stationid)) - return False - location = thiscam[0].split('=')[1].strip() + return None + location = thiscam[0][1] if not os.path.isfile(filename): log.error('{} missing, cannot continue'.format(filename)) - return False - with open(filename, 'r') as fin: - lis = fin.readlines() + return None + lis = open(filename, 'r').readlines() vals = {} for li in lis: if li[0]=='#': @@ -141,11 +216,12 @@ def readIniFile(filename, stationid): val = data[1].strip().strip('"') vals[data[0]] = val if location: - vals['LOCATION'] = location - if os.path.isfile(os.path.expanduser('~/.ssh/ukmon_' + stationid)): - vals['UKMONKEY'] = '~/.ssh/ukmon_' + stationid - if os.path.isfile(os.path.expanduser('~/source/Stations/' + stationid + '/.config')): - vals['RMSCFG'] = os.path.expanduser('~/source/Stations/' + stationid + '/.config') + vals['LOCATION'] = location.strip() + if stationid: + if os.path.isfile(os.path.expanduser('~/.ssh/ukmon_' + stationid.upper())): + vals['UKMONKEY'] = '~/.ssh/ukmon_' + stationid.upper() + if os.path.isfile(os.path.expanduser('~/source/Stations/' + stationid + '/.config')): + vals['RMSCFG'] = os.path.expanduser('~/source/Stations/' + stationid + '/.config') return vals @@ -395,7 +471,7 @@ def uploadToArchive(arch_dir, stationid, sciencefiles=False, keys=False): def getListOfStations(srcdir): camfile = os.path.join(srcdir, 'cameras.ini') if not os.path.isfile(camfile): - return ('NOTCONFIGURED','') + return [(None,'')] camcfg = configparser.ConfigParser() camcfg.read(camfile) return camcfg['cameras'].items() @@ -415,28 +491,30 @@ def manualUpload(targ_dir, stationid, sciencefiles=False): """ if targ_dir == 'test': myloc = os.path.split(os.path.abspath(__file__))[0] - for cam in getListOfStations(myloc): - try: - stationid = cam[0].upper() - if stationid != 'NOTCONFIGURED': - inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) - if not inifvals: - continue - print(inifvals) - keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) - if not keys: - return False - with open('/tmp/test.txt', 'w') as f: - f.write('{}'.format(inifvals['LOCATION'])) - - target = keys['ARCHBUCKET'] - reg = keys['ARCHREGION'] - conn = boto3.Session(aws_access_key_id=keys['AWS_ACCESS_KEY_ID'], aws_secret_access_key=keys['AWS_SECRET_ACCESS_KEY']) - s3 = conn.resource('s3', region_name=reg) - s3.meta.client.upload_file('/tmp/test.txt', target, 'tmp/{}.txt'.format(keys['CAMLOC'])) - print('test successful for' + stationid) - except Exception: - print('archive upload test failed for {} - check ssh key'.format(stationid)) + stations = getListOfStations(myloc) + for cam in stations: + stationid = cam[0] + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) + if not inifvals: + continue + if stationid is not None: + stationid = stationid.upper() + if not getLatestKeys(myloc, stationid): + print('unable to get key for', inifvals['LOCATION']) + continue + keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) + if not keys: + continue + with open('/tmp/test.txt', 'w') as f: + f.write('{}'.format(inifvals['LOCATION'])) + + target = keys['ARCHBUCKET'] + reg = keys['ARCHREGION'] + conn = boto3.Session(aws_access_key_id=keys['AWS_ACCESS_KEY_ID'], aws_secret_access_key=keys['AWS_SECRET_ACCESS_KEY']) + s3 = conn.resource('s3', region_name=reg) + s3.meta.client.upload_file('/tmp/test.txt', target, 'tmp/{}.txt'.format(keys['CAMLOC'])) + stationid = '' if stationid is None else stationid + print('test successful for', inifvals['LOCATION'], stationid) try: os.remove('/tmp/test.txt') except Exception: From 5a67749a7d9ef9f60b58140a8a851ce5bae064ee Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 16:46:50 +0000 Subject: [PATCH 08/50] multicam support for liveMonitor --- liveMonitor.py | 35 ++++++++++++++++++++--------------- liveMonitor.sh | 13 +++++++++---- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/liveMonitor.py b/liveMonitor.py index f8257a1..ba9a8bf 100644 --- a/liveMonitor.py +++ b/liveMonitor.py @@ -8,7 +8,7 @@ import logging import RMS.ConfigReader as cr from stat import ST_INO -from uploadToArchive import readKeyFile, readIniFile +from uploadToArchive import readKeyFile, readIniFile, getLatestKeys from ukmonPostProc import setupLogging, versionid @@ -48,11 +48,21 @@ def follow(fname, logf_ino): yield line.strip() -def monitorLogFile(camloc, rmscfg): +def monitorLogFile(camloc, stationid=None): """ This function monitors the latest RMS log file for meteor detections, convert the FF file to a jpg and upload it to the livestream. This function is called from the shell script *liveMonitor.sh* and should not be called directly. """ + # get configuration + myloc = os.path.split(os.path.abspath(__file__))[0] + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) + if not inifvals: + log.error('ukmon.ini not present, aborting') + exit(1) + rmscfg = inifvals['RMSCFG'] + if not os.path.isfile(rmscfg): + log.error('rms config file', rmscfg,'not present, aborting') + exit(1) cfg = cr.parse(os.path.expanduser(rmscfg)) datadir = cfg.data_dir @@ -67,16 +77,14 @@ def monitorLogFile(camloc, rmscfg): log.info('Camera location is {}'.format(camloc)) log.info('RMS config file is {}'.format(rmscfg)) - myloc = os.path.split(os.path.abspath(__file__))[0] - # get credentials - inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) - if not inifvals: - log.error('ukmon.ini not present, aborting') - exit(1) + if not os.path.isfile(os.path.join(myloc, 'live.key')): + if not getLatestKeys(myloc, cfg.stationID): + print('unable to get key for', inifvals['LOCATION']) + exit(1) keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) if not keys: - log.error('config file not present, aborting') + log.error('unable to read AWS configuration, aborting') exit(1) keepon = True @@ -143,11 +151,8 @@ def monitorLogFile(camloc, rmscfg): if __name__ == '__main__': if len(sys.argv) < 2: - print('LOCATION missing') + print('usage: python liveMonitor.py {ukmon_location} {rms_id}') exit(1) - if len(sys.argv) < 3: - rmscfg = os.path.expanduser('~/source/RMS/.config') - else: - rmscfg = sys.argv[2] camloc = sys.argv[1] - monitorLogFile(camloc, rmscfg) + stationid = sys.argv[2] if len(sys.argv) > 2 else None + monitorLogFile(camloc, stationid) diff --git a/liveMonitor.sh b/liveMonitor.sh index 0645515..cfbe9a4 100755 --- a/liveMonitor.sh +++ b/liveMonitor.sh @@ -9,17 +9,22 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # kill any existing livestream process pids=$(ps -ef | grep ${here}/liveMonitor | egrep -v "grep|$$" | awk '{print $2}') -kill -9 $pids +[ "$pids" != "" ] && kill -9 $pids source $here/ukmon.ini -source $here/live.key if [ "$LOCATION" == "NOTCONFIGURED" ]; then echo "station not configured, unable to continue" exit 1 fi - rmsdir=$(dirname $RMSCFG) cd $rmsdir export PYTHONPATH=$here:~/source/RMS -python $here/liveMonitor.py $LOCATION $RMSCFG +if [ -f $here/cameras.ini ] ; then + cat $here/cameras.ini | grep = | while read i + do + python $here/liveMonitor.py ${i:7:60} ${i:0:6} & + done +else + python $here/liveMonitor.py $LOCATION & +fi From a947796da46e6da4ac7a3ea3b6f13b24807673b0 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 17:09:21 +0000 Subject: [PATCH 09/50] add multicam support to sendToLive --- sendToLive.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/sendToLive.py b/sendToLive.py index 72c0d8b..17e9159 100644 --- a/sendToLive.py +++ b/sendToLive.py @@ -11,7 +11,7 @@ import shutil import tempfile import boto3 -from uploadToArchive import readKeyFile, readIniFile +from uploadToArchive import readKeyFile, readIniFile, getListOfStations import logging import RMS.ConfigReader as cr import numpy as np @@ -137,7 +137,7 @@ def testFeed(keys, cfg): return retmsg -def singleUpload(cap_dir, dir_file): +def singleUpload(cap_dir, dir_file, stationid=None): """This function is used to manually upload a single event. It can also be used to test the connection - see note below. @@ -156,19 +156,19 @@ def singleUpload(cap_dir, dir_file): camloc = None myloc = os.path.split(os.path.abspath(__file__))[0] + if not stationid: + stations = getListOfStations(myloc) + stationid = list(stations)[0][0].upper() # get camera location from ini file - inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) if not inifvals: log.warning('unable to open ini file') - return 'unable to open ini file' + return 'unable to open ukmon.ini file' camloc = inifvals['LOCATION'] - try: - rmscfg = inifvals['RMSCFG'] - except Exception: - rmscfg='~/source/RMS/.config' - if camloc == 'NOTCONFIGURED': - print('LOCATION not found in ini file, aborting') + rmscfg = inifvals['RMSCFG'] + if camloc == 'NOTCONFIGURED' or not os.path.isfile(rmscfg): + print('LOCATION not configured in ukmon.ini or RMS config file not found at', rmscfg, ', aborting') return 'not configured' # get credentials @@ -177,10 +177,8 @@ def singleUpload(cap_dir, dir_file): log.warning('unable to open keyfile') return 'unable to open keyfile' - # read a few variables from the RMS config file + # Load the RMS config file cfg = cr.parse(os.path.expanduser(rmscfg)) -# configpath, configname = os.path.split(os.path.expanduser(rmscfg)) -# cfg = cr.loadConfigFromDirectory(configname, configpath) if cap_dir == 'test' and dir_file == 'test': retmsg = testFeed(keys, cfg) @@ -191,7 +189,11 @@ def singleUpload(cap_dir, dir_file): if __name__ == '__main__': if len(sys.argv) < 3: - print('usage: python sendToLive.py capdir ffname') + print('usage: python sendToLive.py capdir ffname camid') + print('or') + print(' : python sendToLive.py test test') exit(1) - retmsg = singleUpload(sys.argv[1], sys.argv[2]) + + stationid = sys.argv[3] if len(sys.argv) > 3 else None + retmsg = singleUpload(sys.argv[1], sys.argv[2], stationid) print(retmsg) From 097d42c04364fd3a114d81c036d02dfbb3dbefd1 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 17:09:34 +0000 Subject: [PATCH 10/50] add bumpver config --- bumpver.toml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 bumpver.toml diff --git a/bumpver.toml b/bumpver.toml new file mode 100644 index 0000000..8529ea8 --- /dev/null +++ b/bumpver.toml @@ -0,0 +1,26 @@ +[bumpver] +current_version = "2026.01.04" +version_pattern = "YYYY.MM.PATCH" +commit_message = "bump version {old_version} -> {new_version}" +tag_message = "{new_version}" +tag_scope = "default" +pre_commit_hook = "" +post_commit_hook = "" +commit = true +tag = true +push = true + +[bumpver.file_patterns] +"fireballCollector.py" = [ + "versionid = '{version}'" +] +"README.md" = [ + 'Version {version}', +] +"bumpver.toml" = [ + 'current_version = "{version}"', +] +#".github/workflows/build_tools.yml" = [ +# 'tag_name: {version}', +#] + From 43047b1920a839b553b62b16f7d874beb71cc083 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 17:09:48 +0000 Subject: [PATCH 11/50] reorg gitignore file --- .gitignore | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 690ed65..2596ecb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,3 @@ -# ukmon specific -ukmon.ini -live.key -archive.key -addSftpUser.sh -.firstrun -domp4s -extrascript -dotimelapse -.config.backup -ukmon.ini - # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -139,6 +127,18 @@ dmypy.json # Pyre type checker .pyre/ + +# ukmon specific +ukmon.ini +live.key +archive.key +addSftpUser.sh +.firstrun +domp4s +extrascript +dotimelapse +.config.backup +cameras.ini deploy.yml sample-crontab-entries.txt jsonToKeyFile.py From 152160aca9f1cccd09ad30d9830299bf01dd32c0 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 19:19:41 +0000 Subject: [PATCH 12/50] small tweak to fix bug in uploadToArchive --- uploadToArchive.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/uploadToArchive.py b/uploadToArchive.py index 2011681..981ce07 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -499,9 +499,10 @@ def manualUpload(targ_dir, stationid, sciencefiles=False): continue if stationid is not None: stationid = stationid.upper() - if not getLatestKeys(myloc, stationid): - print('unable to get key for', inifvals['LOCATION']) - continue + if not os.path.isfile(os.path.join(myloc, 'live.key')): + if not getLatestKeys(myloc, stationid): + print('unable to get key for', inifvals['LOCATION']) + continue keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) if not keys: continue From bce0cc11d1f54dc221be794847f8e687cea4dcb4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 19:19:56 +0000 Subject: [PATCH 13/50] Create new setup script --- setupUkmon.sh | 106 ++++++++++++++++++++++++++++++++++++++++++++++ ukmonInstaller.py | 103 +++----------------------------------------- 2 files changed, 113 insertions(+), 96 deletions(-) create mode 100644 setupUkmon.sh diff --git a/setupUkmon.sh b/setupUkmon.sh new file mode 100644 index 0000000..cb668ad --- /dev/null +++ b/setupUkmon.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# refresh ukmon-pitools +# Copyright (C) 2018- Mark McIntyre + +myself=$(readlink -f $0) +here="$( cd "$(dirname "$myself")" >/dev/null 2>&1 ; pwd -P )" + +CAMID=$1 + +if [ "$CAMID" == "" ] ; then + echo "Usage: ./setupUkmon.sh RMSID" + echo "where 'RMSID' is your station's RMS ID eg UK12345" + exit +fi + +export PYTHONPATH=$here:~/source/RMS +source ~/vRMS/bin/activate +cd $here + +echo "refreshing toolset" +git stash +git pull +git stash apply + +UKMONKEY=~/.ssh/ukmon_$CAMID + +if [ ! -f $here/ukmon.ini ] ; then + python -c "from ukmonInstaller import createDefaultIni;createDefaultIni('$here', stationid='$CAMID');" +fi +source $here/ukmon.ini +# creating an ssh key if not already present +if [ ! -f ${UKMONKEY} ] ; then + echo "creating ukmon ssh key" + ssh-keygen -t rsa -f ${UKMONKEY} -q -N '' + echo "Now copy this public key and email it to newcamera@ukmeteornetwork.org, then " + echo "wait for confirmation and further instructions to complete the setup." + echo "" + cat ${UKMONKEY}.pub + echo "" + read -p "Press any key to continue" + exit +fi + +echo "checking required python libs are installed" +pip list | grep boto3 || pip install boto3 +# python-crontab v2.5.1 for python 2.7 backwards compatability. Sigh. +pip list | grep python-crontab | grep 2.5.1 || pip install python-crontab==2.5.1 +pip list | grep paramiko || pip install paramiko + +# if the station is configured, retrieve the AWS keys and test connectivity. +if [[ "$LOCATION" != "NOTCONFIGURED" && "$LOCATION" != "" ]] ; then + # check if RMS is still updating - its taking longer and longer + loopctr=0 + echo "Checking RMS update not in progress" + while [ $loopctr -lt 10 ] ; do + [ -f $RMSCFG ] && break + echo "RMS update in progress or station not configured, trying again in a minute" + sleep 60 + loopctr=$((loopctr + 1)) + done + while [ $loopctr -lt 10 ] ; do + grep XX0001 $RMSCFG | grep stationID: + [ $? -eq 1 ] && break + echo "RMS update in progress or station not configured, trying again in a minute" + sleep 60 + loopctr=$((loopctr + 1)) + done + if [ $loopctr -eq 10 ] ; then + echo RMS update failed or long-running, unable to proceed + exit 1 + else + echo all good proceeding + fi + echo "checking for ukmon config changes" + python -c "import ukmonInstaller as pp ; pp.getLatestKeys('${here}', '${stationid}') ;" + + if [ -f archive.key ] ; then \rm archive.key ; fi + + echo "checking the RMS config file, crontab and icons" + source ~/vRMS/bin/activate + source $here/ukmon.ini + cd $(dirname $RMSCFG) + export PYTHONPATH=$here:~/source/RMS + python -c "import ukmonInstaller as pp ; pp.installUkmonFeed('${RMSCFG}');" + + echo "testing connections" + python $here/sendToLive.py test test + python $here/uploadToArchive.py test + echo "if you did not see two success messages contact us for advice" + if [ "$DOCKER_RUNNING" != "true" ] ; then read -p "Press any key to finish" ; fi + echo "done" +else + statid=$(grep stationID $RMSCFG | awk -F" " '{print $2}') + if [ "$statid" == "XX0001" ] ; then + echo "You must configure RMS before setting up the ukmon tools" + else + python -c "import ukmonInstaller as pp ; pp.addDesktopIcons('${here}', '${statid}');" + echo "Location missing - unable to continue. Please obtain a location code from the UKMON team," + echo "then update the UKMON Config File using the desktop icon and rerun this script." + fi + sleep 5 + if [ "$DOCKER_RUNNING" != "true" ] ; then read -p "Press any key to end" ; fi + exit 1 +fi + diff --git a/ukmonInstaller.py b/ukmonInstaller.py index 60ba7f9..58f2cfb 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -18,7 +18,7 @@ import RMS.ConfigReader as cr from RMS.Misc import isRaspberryPi -from uploadToArchive import readIniFile +from uploadToArchive import readIniFile, updateHelperIp log = logging.getLogger("ukmonlogger") log.setLevel(logging.WARNING) @@ -27,28 +27,16 @@ currip = '3.11.55.160' -def createDefaultIni(homedir, helperip='3.11.55.160', location='NOTCONFIGURED', keyfile=None, rmscfg=None): +def createDefaultIni(homedir, helperip='3.11.55.160', location='NOTCONFIGURED', stationid=''): """ Create a default ini file, if its not present on the target """ homedir = os.path.normpath(os.path.expanduser(homedir)) - if not os.path.isdir(homedir): - os.makedirs(homedir) - camid = homedir[homedir.find('pitools')+8:] - if rmscfg is None: - if camid != '' and 'tests' not in camid: - rmscfg = '~/source/Stations/{}/.config'.format(camid) - else: - rmscfg = '~/source/RMS/.config' - if keyfile is None: - if camid != '' and 'tests' not in camid: - keyfile = '~/.ssh/ukmon-{}'.format(camid) - else: - keyfile = '~/.ssh/ukmon' - if helperip is None: - helperip = currip - if location is None: - location = 'NOTCONFIGURED' + rmscfg = '~/source/Stations/{}/.config'.format(stationid) + if not os.path.isfile(rmscfg): + rmscfg = '~/source/RMS/.config' + + keyfile = '~/.ssh/ukmon_{}'.format(stationid) with open(os.path.join(homedir, 'ukmon.ini'), 'w') as outf: outf.write("# config data for this station\n") outf.write("export LOCATION={}\n".format(location)) @@ -109,36 +97,6 @@ def updateMp4andMag(inif, homedir): return -def updateHelperIp(homedir, helperip): - """ - Update the ukmon.ini file with a new IP address if neeeded. - """ - homedir = os.path.normpath(homedir) - lis = open(os.path.join(homedir, 'ukmon.ini'), 'r').readlines() - with open(os.path.join(homedir, 'ukmon.ini'), 'w') as outf: - for li in lis: - if 'UKMONHELPER' in li: - outf.write("export UKMONHELPER={}\n".format(helperip)) - else: - outf.write('{}'.format(li)) - return - - -def updateLocation(homedir, newloc): - """ - Update the ukmon-specific location, if a new one was supplied. Allows us to move cameras to new sites. - """ - homedir = os.path.normpath(homedir) - lis = open(os.path.join(homedir, 'ukmon.ini'), 'r').readlines() - with open(os.path.join(homedir, 'ukmon.ini'), 'w') as outf: - for li in lis: - if 'LOCATION' in li: - outf.write("export LOCATION={}\n".format(newloc)) - else: - outf.write('{}'.format(li)) - return - - def installUkmonFeed(rmscfg='~/source/RMS/.config'): """ Installs the UKMon postprocessing script into the RMS config file. @@ -304,53 +262,6 @@ def addDesktopIcons(myloc, statid): return -def getLatestKeys(homedir, stationid, remoteinifname='ukmon.ini'): - """ - Retrieve the latest ini and key files from the ukmon server. - If the ini file contains a new server IP or new location, the local copy of the - ini file is updated accordingly. - """ - homedir = os.path.expanduser(os.path.normpath(homedir)) - inifvals = readIniFile(os.path.join(homedir, 'ukmon.ini'), stationid) - ssh_client = paramiko.SSHClient() - ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - #try: - if True: - pkey = paramiko.RSAKey.from_private_key_file(os.path.expanduser(inifvals['UKMONKEY'])) - ssh_client.connect(inifvals['UKMONHELPER'], username=inifvals['LOCATION'], pkey=pkey, look_for_keys=False) - ftp_client = ssh_client.open_sftp() - - # get the aws key file - ftp_client.get('live.key', os.path.join(homedir, 'live.key')) - os.chmod(os.path.join(homedir, 'live.key'), 0o600) - - # get the new ini and check for changes - currinif = os.path.join(homedir, 'ukmon.ini') - newinif = os.path.join(homedir, '.ukmon.new') - ftp_client.put(currinif,'ukmon.ini.client') - ftp_client.get(remoteinifname, newinif) - ftp_client.close() - iniflines = open(newinif,'r').readlines() - for li in iniflines: - li = li.strip() - if 'UKMONHELPER' in li: - newhelper = li.split('=')[1] - if newhelper != inifvals['UKMONHELPER']: - updateHelperIp(homedir, newhelper) - print('server address updated') - if 'LOCATION' in li: - newloc = li.split('=')[1] - if newloc != inifvals['LOCATION']: - updateLocation(homedir, newloc) - print('location updated') - os.remove(newinif) - ssh_client.close() - return True - #except: - else: - return False - - def checkPlatepar(homedir, statid, rmsloc): """ Check for a new platepar on the server and retrieves it if present. From 32e6aca52cde4761c9041d3ddb100a17abd31e7d Mon Sep 17 00:00:00 2001 From: RMS Date: Mon, 5 Jan 2026 19:22:15 +0000 Subject: [PATCH 14/50] adding execute bit --- setupUkmon.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 setupUkmon.sh diff --git a/setupUkmon.sh b/setupUkmon.sh old mode 100644 new mode 100755 From cfbe2072544057f02d0a28505dabab920ab0ed86 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 19:51:21 +0000 Subject: [PATCH 15/50] tweaking setupUkmon --- cameras.ini.sample | 6 ------ setupUkmon.sh | 24 +++++++++++++++++++----- ukmonInstaller.py | 2 +- 3 files changed, 20 insertions(+), 12 deletions(-) delete mode 100644 cameras.ini.sample diff --git a/cameras.ini.sample b/cameras.ini.sample deleted file mode 100644 index 09323d8..0000000 --- a/cameras.ini.sample +++ /dev/null @@ -1,6 +0,0 @@ -# Sample camera mapping file -# add your location/cameraid mappings to this even if you only have one camera -[cameras] -GMNID_1=location_1 -GMNID_2=location_2 -GMNID_3=location_3 \ No newline at end of file diff --git a/setupUkmon.sh b/setupUkmon.sh index cb668ad..2a1344a 100755 --- a/setupUkmon.sh +++ b/setupUkmon.sh @@ -7,6 +7,7 @@ myself=$(readlink -f $0) here="$( cd "$(dirname "$myself")" >/dev/null 2>&1 ; pwd -P )" CAMID=$1 +CAMID=${CAMID^^} if [ "$CAMID" == "" ] ; then echo "Usage: ./setupUkmon.sh RMSID" @@ -26,8 +27,18 @@ git stash apply UKMONKEY=~/.ssh/ukmon_$CAMID if [ ! -f $here/ukmon.ini ] ; then + echo "creating default ukmon ini file" python -c "from ukmonInstaller import createDefaultIni;createDefaultIni('$here', stationid='$CAMID');" fi + +if [ ! -f $here/cameras.ini ] ; then + echo "creating cameras.ini file" + echo "# camera mapping file" > $here/cameras.ini + echo "# echo add all cameras on this PC or Pi even if you only have one camera" >> $here/cameras.ini + echo "[cameras]" >> $here/cameras.ini + echo "${CAMID}=NOTCONFIGURED" >> $here/cameras.ini +fi + source $here/ukmon.ini # creating an ssh key if not already present if [ ! -f ${UKMONKEY} ] ; then @@ -48,6 +59,10 @@ pip list | grep boto3 || pip install boto3 pip list | grep python-crontab | grep 2.5.1 || pip install python-crontab==2.5.1 pip list | grep paramiko || pip install paramiko +# get the location code from the cameras.ini file +LOCATION=$(grep $CAMID $here/cameras.ini) +LOCATION=$(echo $LOCATION | awk -F "=" '{print $2}') + # if the station is configured, retrieve the AWS keys and test connectivity. if [[ "$LOCATION" != "NOTCONFIGURED" && "$LOCATION" != "" ]] ; then # check if RMS is still updating - its taking longer and longer @@ -75,8 +90,6 @@ if [[ "$LOCATION" != "NOTCONFIGURED" && "$LOCATION" != "" ]] ; then echo "checking for ukmon config changes" python -c "import ukmonInstaller as pp ; pp.getLatestKeys('${here}', '${stationid}') ;" - if [ -f archive.key ] ; then \rm archive.key ; fi - echo "checking the RMS config file, crontab and icons" source ~/vRMS/bin/activate source $here/ukmon.ini @@ -91,13 +104,14 @@ if [[ "$LOCATION" != "NOTCONFIGURED" && "$LOCATION" != "" ]] ; then if [ "$DOCKER_RUNNING" != "true" ] ; then read -p "Press any key to finish" ; fi echo "done" else + echo $RMSCFG $CAMID statid=$(grep stationID $RMSCFG | awk -F" " '{print $2}') if [ "$statid" == "XX0001" ] ; then echo "You must configure RMS before setting up the ukmon tools" else - python -c "import ukmonInstaller as pp ; pp.addDesktopIcons('${here}', '${statid}');" - echo "Location missing - unable to continue. Please obtain a location code from the UKMON team," - echo "then update the UKMON Config File using the desktop icon and rerun this script." + #python -c "import ukmonInstaller as pp ; pp.addDesktopIcons('${here}', '${statid}');" + echo "Location missing. Please obtain a location code from the UKMON team," + echo "then update cameras.ini and rerun this script." fi sleep 5 if [ "$DOCKER_RUNNING" != "true" ] ; then read -p "Press any key to end" ; fi diff --git a/ukmonInstaller.py b/ukmonInstaller.py index 58f2cfb..0c4a5ca 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -33,7 +33,7 @@ def createDefaultIni(homedir, helperip='3.11.55.160', location='NOTCONFIGURED', """ homedir = os.path.normpath(os.path.expanduser(homedir)) rmscfg = '~/source/Stations/{}/.config'.format(stationid) - if not os.path.isfile(rmscfg): + if not os.path.isfile(os.path.expanduser(rmscfg)): rmscfg = '~/source/RMS/.config' keyfile = '~/.ssh/ukmon_{}'.format(stationid) From a0febb120f470b8f99671460ca2592b2e050acdf Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 20:19:16 +0000 Subject: [PATCH 16/50] more updates to setup script --- setupUkmon.sh | 14 +++++++++++--- ukmonInstaller.py | 25 +++++++++++-------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/setupUkmon.sh b/setupUkmon.sh index 2a1344a..56220b4 100755 --- a/setupUkmon.sh +++ b/setupUkmon.sh @@ -37,6 +37,11 @@ if [ ! -f $here/cameras.ini ] ; then echo "# echo add all cameras on this PC or Pi even if you only have one camera" >> $here/cameras.ini echo "[cameras]" >> $here/cameras.ini echo "${CAMID}=NOTCONFIGURED" >> $here/cameras.ini +else + grep $CAMID $here/cameras.ini > /dev/null 2>&1 + if [ $? == 1 ] ; then + echo "${CAMID}=NOTCONFIGURED" >> $here/cameras.ini + fi fi source $here/ukmon.ini @@ -88,12 +93,15 @@ if [[ "$LOCATION" != "NOTCONFIGURED" && "$LOCATION" != "" ]] ; then echo all good proceeding fi echo "checking for ukmon config changes" - python -c "import ukmonInstaller as pp ; pp.getLatestKeys('${here}', '${stationid}') ;" - + python -c "import uploadToArchive as pp ; pp.getLatestKeys('${here}', '${stationid}') ;" + + if [ -d ~/Desktop ] ; then + pushd ~/Desktop + rm -f ukmon.ini UKMON_config* refreshTools* refresh_UKMON* > /dev/null 2>&1 + fi echo "checking the RMS config file, crontab and icons" source ~/vRMS/bin/activate source $here/ukmon.ini - cd $(dirname $RMSCFG) export PYTHONPATH=$here:~/source/RMS python -c "import ukmonInstaller as pp ; pp.installUkmonFeed('${RMSCFG}');" diff --git a/ukmonInstaller.py b/ukmonInstaller.py index 0c4a5ca..a861152 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -212,19 +212,19 @@ def createSystemdService(myloc, camid): return -def createUbuntuIcon(myloc, statid): +def createUbuntuIcon(myloc): """ Create Ubuntu-compatible desktop icons. These different from the Debian-compatible ones normally used by RMS and which dont work properly on Ubuntu. """ - reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools_{}.sh'.format(statid)) + reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools.sh') if os.path.isfile(reflnk): os.remove(reflnk) - reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools_{}.desktop'.format(statid)) + reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools.desktop') with open(reflnk, 'w') as outf: outf.write('[Desktop Entry]\n') - outf.write('Name=refresh_UKMON_Tools_{}\n'.format(statid)) + outf.write('Name=refresh_UKMON_Tools\n') outf.write('Comment=Runs ukmon tools refresh\n') outf.write('Exec={}\n'.format(os.path.join(myloc, 'refreshTools.sh'))) outf.write('Icon=\n') @@ -243,22 +243,19 @@ def addDesktopIcons(myloc, statid): print('checking/adding desktop icons') if not os.path.isdir(os.path.expanduser('~/Desktop')): os.makedirs(os.path.expanduser('~/Desktop')) - cfglnk = os.path.expanduser('~/Desktop/UKMON_config_{}.txt'.format(statid)) + # the main and camera config files + cfglnk = os.path.expanduser('~/Desktop/UKMON_config.txt') if not os.path.islink(cfglnk): os.symlink(os.path.join(myloc, 'ukmon.ini'), cfglnk) + camlnk = os.path.expanduser('~/Desktop/UKMON_cameras.txt') + if not os.path.islink(camlnk): + os.symlink(os.path.join(myloc, 'cameras.ini'), camlnk) if isRaspberryPi(): - reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools_{}.sh'.format(statid)) + reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools.sh') if not os.path.islink(reflnk): os.symlink(os.path.join(myloc, 'refreshTools.sh'), reflnk) else: - createUbuntuIcon(myloc, statid) - # remove bad links if present - cfglnk = os.path.expanduser('~/Desktop/UKMON_config_XX0001.txt') - if os.path.islink(cfglnk): - os.unlink(cfglnk) - reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools_XX0001.sh') - if os.path.islink(reflnk): - os.unlink(reflnk) + createUbuntuIcon(myloc) return From 879de58b3623598cffbb3b866b3fb367bdeb0e08 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 20:25:26 +0000 Subject: [PATCH 17/50] trim down refreshTools --- refreshTools.sh | 71 +------------------------------------------------ 1 file changed, 1 insertion(+), 70 deletions(-) diff --git a/refreshTools.sh b/refreshTools.sh index 6bcf674..a319758 100755 --- a/refreshTools.sh +++ b/refreshTools.sh @@ -1,7 +1,7 @@ #!/bin/bash # refresh ukmon-pitools -# Copyright (C) 2018-2023 Mark McIntyre +# Copyright (C) 2018- Mark McIntyre myself=$(readlink -f $0) here="$( cd "$(dirname "$myself")" >/dev/null 2>&1 ; pwd -P )" @@ -25,72 +25,3 @@ echo "refreshing toolset" git stash git pull git stash apply - -# creating an ssh key if not already present -if [ ! -f ${UKMONKEY} ] ; then - echo "creating ukmon ssh key" - ssh-keygen -t rsa -f ${UKMONKEY} -q -N '' - echo "Copy this public key and email it to newcamera@ukmeteornetwork.org, then " - echo "wait for confirmation its been installed and rerun this script" - echo "" - cat ${UKMONKEY}.pub - echo "" - read -p "Press any key to continue" -fi - -# if the station is configured, retrieve the AWS keys and test connectivity. -if [[ "$LOCATION" != "NOTCONFIGURED" && "$LOCATION" != "" ]] ; then - # check if RMS is still updating - its taking longer and longer - loopctr=0 - echo "Checking RMS update not in progress" - while [ $loopctr -lt 10 ] ; do - [ -f $RMSCFG ] && break - echo "RMS update in progress or station not configured, trying again in a minute" - sleep 60 - loopctr=$((loopctr + 1)) - done - while [ $loopctr -lt 10 ] ; do - grep XX0001 $RMSCFG | grep stationID: - [ $? -eq 1 ] && break - echo "RMS update in progress or station not configured, trying again in a minute" - sleep 60 - loopctr=$((loopctr + 1)) - done - if [ $loopctr -eq 10 ] ; then - echo RMS update failed or long-running, unable to proceed - exit 1 - else - echo all good proceeding - fi - echo "checking for ukmon config changes" - python -c "import ukmonInstaller as pp ; pp.getLatestKeys('${here}') ;" - - if [ -f archive.key ] ; then \rm archive.key ; fi - - echo "checking the RMS config file, crontab and icons" - source ~/vRMS/bin/activate - source $here/ukmon.ini - cd $(dirname $RMSCFG) - export PYTHONPATH=$here:~/source/RMS - python -c "import ukmonInstaller as pp ; pp.installUkmonFeed('${RMSCFG}');" - - echo "testing connections" - python $here/sendToLive.py test test - python $here/uploadToArchive.py test - echo "if you did not see two success messages contact us for advice" - if [ "$DOCKER_RUNNING" != "true" ] ; then read -p "Press any key to finish" ; fi - echo "done" -else - statid=$(grep stationID $RMSCFG | awk -F" " '{print $2}') - if [ "$statid" == "XX0001" ] ; then - echo "You must configure RMS before setting up the ukmon tools" - else - python -c "import ukmonInstaller as pp ; pp.addDesktopIcons('${here}', '${statid}');" - echo "Location missing - unable to continue. Please obtain a location code from the UKMON team," - echo "then update the UKMON Config File using the desktop icon and rerun this script." - fi - sleep 5 - if [ "$DOCKER_RUNNING" != "true" ] ; then read -p "Press any key to end" ; fi - exit 1 -fi - From ad51ad3edde5eaf38cc775bf2760c4e53fda1391 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 20:26:22 +0000 Subject: [PATCH 18/50] reinstate tests in refreshTools --- refreshTools.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/refreshTools.sh b/refreshTools.sh index a319758..4e6d978 100755 --- a/refreshTools.sh +++ b/refreshTools.sh @@ -25,3 +25,8 @@ echo "refreshing toolset" git stash git pull git stash apply + +echo "testing connections" +python $here/sendToLive.py test test +python $here/uploadToArchive.py test +echo "if you did not see two success messages contact us for advice" From 8c7179a0578c26615a97bb01762299f6ec60bfb6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 20:30:28 +0000 Subject: [PATCH 19/50] bugfix in sendTolive test logic --- sendToLive.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sendToLive.py b/sendToLive.py index 17e9159..6d725a1 100644 --- a/sendToLive.py +++ b/sendToLive.py @@ -158,7 +158,9 @@ def singleUpload(cap_dir, dir_file, stationid=None): myloc = os.path.split(os.path.abspath(__file__))[0] if not stationid: stations = getListOfStations(myloc) - stationid = list(stations)[0][0].upper() + tmpid = list(stations)[0][0] + if tmpid: + stationid = tmpid.upper() # get camera location from ini file inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) From d958977a2f6c81831a55ea0292a9585ae2b5563f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 20:33:09 +0000 Subject: [PATCH 20/50] another small bugfix for testing --- uploadToArchive.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uploadToArchive.py b/uploadToArchive.py index 981ce07..9760165 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -222,6 +222,8 @@ def readIniFile(filename, stationid): vals['UKMONKEY'] = '~/.ssh/ukmon_' + stationid.upper() if os.path.isfile(os.path.expanduser('~/source/Stations/' + stationid + '/.config')): vals['RMSCFG'] = os.path.expanduser('~/source/Stations/' + stationid + '/.config') + if vals['LOCATION'] == 'NOTCONFIGURED': + return None return vals From 1f7f59c85d1ac01f42f7b529ef63bf9007f4301b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 20:40:29 +0000 Subject: [PATCH 21/50] improving error messages --- sendToLive.py | 17 +++++++++-------- ukmonPostProc.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/sendToLive.py b/sendToLive.py index 6d725a1..3a7ba3b 100644 --- a/sendToLive.py +++ b/sendToLive.py @@ -165,19 +165,19 @@ def singleUpload(cap_dir, dir_file, stationid=None): inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) if not inifvals: - log.warning('unable to open ini file') - return 'unable to open ukmon.ini file' + log.error('ukmon ini file invalid - check LOCATION') + return False camloc = inifvals['LOCATION'] rmscfg = inifvals['RMSCFG'] - if camloc == 'NOTCONFIGURED' or not os.path.isfile(rmscfg): - print('LOCATION not configured in ukmon.ini or RMS config file not found at', rmscfg, ', aborting') - return 'not configured' + if not os.path.isfile(rmscfg): + log.error('RMS config file not found at', rmscfg, ', aborting') + return False # get credentials keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) if not keys: - log.warning('unable to open keyfile') - return 'unable to open keyfile' + log.error('unable to open keyfile') + return False # Load the RMS config file cfg = cr.parse(os.path.expanduser(rmscfg)) @@ -198,4 +198,5 @@ def singleUpload(cap_dir, dir_file, stationid=None): stationid = sys.argv[3] if len(sys.argv) > 3 else None retmsg = singleUpload(sys.argv[1], sys.argv[2], stationid) - print(retmsg) + if retmsg: + print(retmsg) diff --git a/ukmonPostProc.py b/ukmonPostProc.py index 5089ec6..e1328c3 100644 --- a/ukmonPostProc.py +++ b/ukmonPostProc.py @@ -191,7 +191,7 @@ def manualRerun(dated_dir, rmscfg = '~/source/RMS/.config'): myloc = os.path.split(os.path.abspath(__file__))[0] inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) if inifvals is None: - print('unable to open ukmon ini file') + print('ukmon ini file invalid - check LOCATION') exit(1) try: rmscfg = inifvals['RMSCFG'] From 146e9be6b07440f77133323dfd67fe16a6307c80 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 20:42:25 +0000 Subject: [PATCH 22/50] more tweaks to error reporting --- sendToLive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sendToLive.py b/sendToLive.py index 3a7ba3b..07deffd 100644 --- a/sendToLive.py +++ b/sendToLive.py @@ -166,18 +166,18 @@ def singleUpload(cap_dir, dir_file, stationid=None): inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) if not inifvals: log.error('ukmon ini file invalid - check LOCATION') - return False + return 'ukmon ini file invalid - check LOCATION' camloc = inifvals['LOCATION'] rmscfg = inifvals['RMSCFG'] if not os.path.isfile(rmscfg): log.error('RMS config file not found at', rmscfg, ', aborting') - return False + return 'RMS config file not found at', rmscfg, ', aborting' # get credentials keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) if not keys: log.error('unable to open keyfile') - return False + return 'unable to open keyfile' # Load the RMS config file cfg = cr.parse(os.path.expanduser(rmscfg)) From 55a50b4168a75900376b805d24fd615f26b34fa1 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 20:44:47 +0000 Subject: [PATCH 23/50] oops, error --- sendToLive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sendToLive.py b/sendToLive.py index 07deffd..870adcf 100644 --- a/sendToLive.py +++ b/sendToLive.py @@ -170,7 +170,7 @@ def singleUpload(cap_dir, dir_file, stationid=None): camloc = inifvals['LOCATION'] rmscfg = inifvals['RMSCFG'] if not os.path.isfile(rmscfg): - log.error('RMS config file not found at', rmscfg, ', aborting') + log.error('RMS config file not found at {}, aborting'.format(rmscfg)) return 'RMS config file not found at', rmscfg, ', aborting' # get credentials From fbfaa9776707593988fbfa13635bcb7ea4be5c64 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 20:46:03 +0000 Subject: [PATCH 24/50] another simple error --- sendToLive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sendToLive.py b/sendToLive.py index 870adcf..8fcf5f6 100644 --- a/sendToLive.py +++ b/sendToLive.py @@ -169,7 +169,7 @@ def singleUpload(cap_dir, dir_file, stationid=None): return 'ukmon ini file invalid - check LOCATION' camloc = inifvals['LOCATION'] rmscfg = inifvals['RMSCFG'] - if not os.path.isfile(rmscfg): + if not os.path.isfile(os.path.expanduser(rmscfg)): log.error('RMS config file not found at {}, aborting'.format(rmscfg)) return 'RMS config file not found at', rmscfg, ', aborting' From 840760e3579e31523adcc12cc6fc578c1a795a48 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 21:44:40 +0000 Subject: [PATCH 25/50] fix tests --- tests/test_ukmonInstaller.py | 4 ++-- tests/test_ukmonPostProc.py | 11 ++++++----- tests/test_uploadToArchive.py | 14 +++++++------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/test_ukmonInstaller.py b/tests/test_ukmonInstaller.py index 2c57052..978f885 100644 --- a/tests/test_ukmonInstaller.py +++ b/tests/test_ukmonInstaller.py @@ -3,8 +3,8 @@ import os import shutil -from ukmonInstaller import createDefaultIni, updateHelperIp, updateLocation, \ - checkPostProcSettings, validateIni, getLatestKeys # noqa: E402 +from ukmonInstaller import createDefaultIni, checkPostProcSettings, validateIni +from uploadToArchive import updateHelperIp, updateLocation, getLatestKeys myloc = os.path.split(os.path.abspath(__file__))[0] homedir = os.path.join(myloc, 'ukminst') diff --git a/tests/test_ukmonPostProc.py b/tests/test_ukmonPostProc.py index 14b63e7..539f786 100644 --- a/tests/test_ukmonPostProc.py +++ b/tests/test_ukmonPostProc.py @@ -5,30 +5,31 @@ import os import sys import pytest -from ukmonPostProc import main +from ukmonPostProc import manualRerun myloc = os.path.split(os.path.abspath(__file__))[0] homedir = os.path.join(myloc, 'ukmpp') tmpdir = os.path.join(myloc, 'output') if not os.path.isdir(tmpdir): - os.makedirs(tmpdir) # , exist_ok=Truee) exist_ok keyword not supported with python7.2 + os.makedirs(tmpdir) # , exist_ok=Truee) exist_ok keyword not supported with python 2.7 +rmscfg='~/source/RMS/.config' def test_ukmonPostProcNoArgs(): args=[None] - ret = main(args) + ret = manualRerun(*args) assert ret is False @pytest.mark.skipif(sys.platform == 'win32', reason='test not valid on windows') def test_ukmonPostProc1Arg(): args=[None, os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543_087124')] - ret = main(args) + ret = manualRerun(*args) assert ret is True def test_ukmonPostProc1BadArg(): args=[None, os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543')] - ret = main(args) + ret = manualRerun(*args) assert ret is False diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py index dfb17b7..b924f11 100644 --- a/tests/test_uploadToArchive.py +++ b/tests/test_uploadToArchive.py @@ -12,7 +12,7 @@ def test_checkMags(): - inifvals = readIniFile(os.path.join(basedir,'..','ukmon.ini')) + inifvals = readIniFile(os.path.join(basedir,'..','ukmon.ini'),'UK0006') maglim = 6 if 'MAGLIM' in inifvals: maglim = float(inifvals['MAGLIM']) @@ -24,14 +24,14 @@ def test_checkMags(): def test_readIniFile(): - inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini')) + inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'),'TESTPI4B') assert inifs['LOCATION']=='testpi4' def test_readKeyFile(): - inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini')) + inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'),'UK0006') vals = readKeyFile(os.path.join(basedir,'..','live.key'), inifs) - assert vals['S3FOLDER'] in ['tmp/testpi4','archive/Tackley'] + assert vals['S3FOLDER'] in ['tmp/testpi4','archive/Tackley'] def test_readKeyfileIni(): @@ -43,7 +43,7 @@ def test_readKeyfileIni(): def test_uploadOneFile(): - inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini')) + inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'),'UK0006') keys = readKeyFile(os.path.join(basedir,'..','live.key'), inifs) reg = keys['ARCHREGION'] conn = boto3.Session(aws_access_key_id=keys['AWS_ACCESS_KEY_ID'], aws_secret_access_key=keys['AWS_SECRET_ACCESS_KEY']) @@ -63,7 +63,7 @@ def test_uploadOneFile(): def test_manualUpload(): targ_dir = 'test' - assert manualUpload(targ_dir) is True + assert manualUpload(targ_dir,'UK0006') is True targ_dir = os.path.join(basedir, 'ukmarch','testpi4_20230401') # create some dummy sample files testfilelist = ['FF_test_20230401.fits','FF_test_20230401.jpg','FF_test_20230401.mp4','mask.bmp', @@ -73,6 +73,6 @@ def test_manualUpload(): ] for fil in testfilelist: open(os.path.join(targ_dir, fil), 'w').write('{"test":"potato"}') - assert manualUpload(targ_dir) + assert manualUpload(targ_dir,'UK0006') for fil in testfilelist: os.remove(os.path.join(targ_dir, fil)) From d2a06dbe7eb3bc3d6207bab4f357ed4c597161a4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 22:26:07 +0000 Subject: [PATCH 26/50] fix bugs and tests --- tests/test_ukmonInstaller.py | 6 +++--- ukmonInstaller.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_ukmonInstaller.py b/tests/test_ukmonInstaller.py index 978f885..03bff6c 100644 --- a/tests/test_ukmonInstaller.py +++ b/tests/test_ukmonInstaller.py @@ -89,7 +89,7 @@ def test_checkPostProcSettings(): def test_getLatestKeys_normal(): shutil.copyfile(os.path.join(myloc, '../ukmon.ini'),os.path.join(homedir,'ukmon.ini')) updateHelperIp(homedir, helperip='3.11.55.160') - res = getLatestKeys(homedir) + res = getLatestKeys(homedir, 'UK0006') assert res is True os.remove(os.path.join(homedir, 'ukmon.ini')) return @@ -99,7 +99,7 @@ def test_getLatestKeys_newname(): shutil.copyfile(os.path.join(myloc, '../ukmon.ini'),os.path.join(homedir,'ukmon.ini')) updateHelperIp(homedir, helperip='3.11.55.160') remoteinifname = 'ukmon.ini.newname' - res = getLatestKeys(homedir, remoteinifname=remoteinifname) + res = getLatestKeys(homedir, 'UK0006', remoteinifname=remoteinifname) assert res is True lis = open(os.path.join(homedir, 'ukmon.ini'), 'r').readlines() for li in lis: @@ -114,7 +114,7 @@ def test_getLatestKeys_newip(): shutil.copyfile(os.path.join(myloc, '../ukmon.ini'),os.path.join(homedir,'ukmon.ini')) updateHelperIp(homedir, helperip='3.11.55.160') remoteinifname = 'ukmon.ini.newip' - res = getLatestKeys(homedir, remoteinifname=remoteinifname) + res = getLatestKeys(homedir, 'UK0006', remoteinifname=remoteinifname) assert res is True lis = open(os.path.join(homedir, 'ukmon.ini'), 'r').readlines() for li in lis: diff --git a/ukmonInstaller.py b/ukmonInstaller.py index a861152..e2715cd 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -73,7 +73,7 @@ def validateIni(homedir, newhelperip=None): if 'UKMONHELPER' in li: helperip = li.split('=')[1] if location is None or keyfile is None or rmscfg is None or helperip is None: - createDefaultIni(homedir, newhelperip, location, keyfile, rmscfg) + createDefaultIni(homedir, newhelperip, location, rmscfg) if helperip == oldip: updateHelperIp(homedir, newhelperip) updateMp4andMag(inifname, homedir) From 495134cdeee55c03fdf3189e32c7df7d188c167b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 22:30:55 +0000 Subject: [PATCH 27/50] fix tests --- tests/test_ukmonPostProc.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_ukmonPostProc.py b/tests/test_ukmonPostProc.py index 539f786..d1a4415 100644 --- a/tests/test_ukmonPostProc.py +++ b/tests/test_ukmonPostProc.py @@ -17,19 +17,18 @@ def test_ukmonPostProcNoArgs(): - args=[None] - ret = manualRerun(*args) + ret = manualRerun(None) assert ret is False @pytest.mark.skipif(sys.platform == 'win32', reason='test not valid on windows') def test_ukmonPostProc1Arg(): - args=[None, os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543_087124')] - ret = manualRerun(*args) + args=os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543_087124') + ret = manualRerun(args) assert ret is True def test_ukmonPostProc1BadArg(): - args=[None, os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543')] + args=os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543') ret = manualRerun(*args) assert ret is False From f09145f70ed5e35af2c38a7dad8284aa73029807 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 22:37:30 +0000 Subject: [PATCH 28/50] various test fixes --- tests/run_ub_tests.sh | 4 ++-- tests/test_ukmonPostProc.py | 1 - tests/test_uploadToArchive.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh index 06545fb..ecb19b4 100644 --- a/tests/run_ub_tests.sh +++ b/tests/run_ub_tests.sh @@ -19,7 +19,7 @@ echo testing for $LOCATION pip install -r ./requirements.txt pip install --upgrade ruff pytest xmltodict pytest-cov export PYTHONPATH=$PYTHONPATH:/root/source/RMS:${here}/.. -cd /root/source/RMS -ls -ltra .config +cd ~/source/RMS +ls -ltra ~/source/RMS/.config pwd pytest -v $here/ --cov=$here/../ --cov-report=term-missing --cov-config=$here/../.coveragerc_lnx diff --git a/tests/test_ukmonPostProc.py b/tests/test_ukmonPostProc.py index d1a4415..2a8d3be 100644 --- a/tests/test_ukmonPostProc.py +++ b/tests/test_ukmonPostProc.py @@ -13,7 +13,6 @@ tmpdir = os.path.join(myloc, 'output') if not os.path.isdir(tmpdir): os.makedirs(tmpdir) # , exist_ok=Truee) exist_ok keyword not supported with python 2.7 -rmscfg='~/source/RMS/.config' def test_ukmonPostProcNoArgs(): diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py index b924f11..aed8696 100644 --- a/tests/test_uploadToArchive.py +++ b/tests/test_uploadToArchive.py @@ -37,7 +37,7 @@ def test_readKeyFile(): def test_readKeyfileIni(): homedir = os.path.join(basedir, 'output') createDefaultIni(homedir) - vals = readIniFile(os.path.join(homedir,'ukmon.ini')) + vals = readIniFile(os.path.join(homedir,'ukmon.ini'), 'TESTPI4B') os.remove(os.path.join(homedir,'ukmon.ini')) assert vals['RMSCFG'] == '~/source/RMS/.config' From 6c906fd171d14bc80ad3b05c7154bf4366265c0e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 23:39:23 +0000 Subject: [PATCH 29/50] fix tsts2 --- tests/test_uploadToArchive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py index aed8696..8c247dd 100644 --- a/tests/test_uploadToArchive.py +++ b/tests/test_uploadToArchive.py @@ -24,12 +24,12 @@ def test_checkMags(): def test_readIniFile(): - inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'),'TESTPI4B') + inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'),stationid='TESTPI4B') assert inifs['LOCATION']=='testpi4' def test_readKeyFile(): - inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'),'UK0006') + inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'), stationid='UK0006') vals = readKeyFile(os.path.join(basedir,'..','live.key'), inifs) assert vals['S3FOLDER'] in ['tmp/testpi4','archive/Tackley'] @@ -37,7 +37,7 @@ def test_readKeyFile(): def test_readKeyfileIni(): homedir = os.path.join(basedir, 'output') createDefaultIni(homedir) - vals = readIniFile(os.path.join(homedir,'ukmon.ini'), 'TESTPI4B') + vals = readIniFile(os.path.join(homedir,'ukmon.ini'), stationid='TESTPI4B') os.remove(os.path.join(homedir,'ukmon.ini')) assert vals['RMSCFG'] == '~/source/RMS/.config' From a5da8520635fa419be22bff9aea949aee4964013 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 5 Jan 2026 23:49:42 +0000 Subject: [PATCH 30/50] fizx --- tests/test_ukmonPostProc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_ukmonPostProc.py b/tests/test_ukmonPostProc.py index 2a8d3be..e3d4ae2 100644 --- a/tests/test_ukmonPostProc.py +++ b/tests/test_ukmonPostProc.py @@ -16,6 +16,7 @@ def test_ukmonPostProcNoArgs(): + print(os.path.listdir(os.path.expanduser('~/source/RMS'))) ret = manualRerun(None) assert ret is False From 3387ba9fc3f5aada27a549c5dd58a9277706e6e7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 6 Jan 2026 00:08:29 +0000 Subject: [PATCH 31/50] testing --- tests/test_ukmonPostProc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ukmonPostProc.py b/tests/test_ukmonPostProc.py index e3d4ae2..0c3fed0 100644 --- a/tests/test_ukmonPostProc.py +++ b/tests/test_ukmonPostProc.py @@ -24,11 +24,11 @@ def test_ukmonPostProcNoArgs(): @pytest.mark.skipif(sys.platform == 'win32', reason='test not valid on windows') def test_ukmonPostProc1Arg(): args=os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543_087124') - ret = manualRerun(args) + ret = manualRerun(dated_dir=args) assert ret is True def test_ukmonPostProc1BadArg(): args=os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543') - ret = manualRerun(*args) + ret = manualRerun(dated_dir=args) assert ret is False From 7e61131c0ab5ac5e41a44d6f2818d3e4b8348964 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 6 Jan 2026 16:17:37 +0000 Subject: [PATCH 32/50] fix path for tests --- tests/run_ub_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh index ecb19b4..b11f065 100644 --- a/tests/run_ub_tests.sh +++ b/tests/run_ub_tests.sh @@ -19,7 +19,7 @@ echo testing for $LOCATION pip install -r ./requirements.txt pip install --upgrade ruff pytest xmltodict pytest-cov export PYTHONPATH=$PYTHONPATH:/root/source/RMS:${here}/.. -cd ~/source/RMS -ls -ltra ~/source/RMS/.config +cd /root/source/RMS/ +ls -ltra /root/source/RMS/.config pwd pytest -v $here/ --cov=$here/../ --cov-report=term-missing --cov-config=$here/../.coveragerc_lnx From 351acc7cf5a636f726be60f3550e532b3d6299c3 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 6 Jan 2026 16:42:36 +0000 Subject: [PATCH 33/50] small testing tweaks --- tests/test_ukmonPostProc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ukmonPostProc.py b/tests/test_ukmonPostProc.py index 0c3fed0..91e2e10 100644 --- a/tests/test_ukmonPostProc.py +++ b/tests/test_ukmonPostProc.py @@ -9,14 +9,14 @@ myloc = os.path.split(os.path.abspath(__file__))[0] -homedir = os.path.join(myloc, 'ukmpp') +#homedir = os.path.join(myloc, 'ukmpp') tmpdir = os.path.join(myloc, 'output') if not os.path.isdir(tmpdir): os.makedirs(tmpdir) # , exist_ok=Truee) exist_ok keyword not supported with python 2.7 def test_ukmonPostProcNoArgs(): - print(os.path.listdir(os.path.expanduser('~/source/RMS'))) + print(os.listdir('.')) ret = manualRerun(None) assert ret is False From c512005413282ef1641cc03965403b86bf179104 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 14:48:03 +0000 Subject: [PATCH 34/50] trying to fix tests --- tests/test_ukmonPostProc.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/tests/test_ukmonPostProc.py b/tests/test_ukmonPostProc.py index 91e2e10..3f978c6 100644 --- a/tests/test_ukmonPostProc.py +++ b/tests/test_ukmonPostProc.py @@ -12,23 +12,46 @@ #homedir = os.path.join(myloc, 'ukmpp') tmpdir = os.path.join(myloc, 'output') if not os.path.isdir(tmpdir): - os.makedirs(tmpdir) # , exist_ok=Truee) exist_ok keyword not supported with python 2.7 + os.makedirs(tmpdir) # , exist_ok=True) exist_ok keyword not supported with python 2.7 +# this should fail because the test RMS config is not at ~/source/RMS but at /root/source/RMS def test_ukmonPostProcNoArgs(): print(os.listdir('.')) - ret = manualRerun(None) - assert ret is False + print(os.getenv('HOME')) + try: + ret = manualRerun(None) + assert ret is True + except Exception: + assert True @pytest.mark.skipif(sys.platform == 'win32', reason='test not valid on windows') def test_ukmonPostProc1Arg(): + # this will fail because of the location of the config file args=os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543_087124') - ret = manualRerun(dated_dir=args) - assert ret is True + try: + ret = manualRerun(dated_dir=args) + assert ret is True + except Exception: + assert True + + +@pytest.mark.skipif(sys.platform == 'win32', reason='test not valid on windows') +def test_ukmonPostProc2Args(): + args=os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543_087124') + try: + ret = manualRerun(dated_dir=args, rmscfg='/root/source/RMS/.config') + assert ret is True + except Exception: + assert True def test_ukmonPostProc1BadArg(): + # this will also fail because of the bad data args=os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543') - ret = manualRerun(dated_dir=args) - assert ret is False + try: + ret = manualRerun(dated_dir=args, rmscfg='/root/source/RMS/.config') + assert ret is False + except Exception: + assert True From 100e0d123131c492c576d1ce22641c8569877e06 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 15:01:02 +0000 Subject: [PATCH 35/50] fixing bug in test for readKeyFileIni --- tests/test_uploadToArchive.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py index 8c247dd..ecf8b57 100644 --- a/tests/test_uploadToArchive.py +++ b/tests/test_uploadToArchive.py @@ -35,9 +35,10 @@ def test_readKeyFile(): def test_readKeyfileIni(): - homedir = os.path.join(basedir, 'output') + homedir = os.path.expanduser(os.path.join(basedir, 'output')) createDefaultIni(homedir) vals = readIniFile(os.path.join(homedir,'ukmon.ini'), stationid='TESTPI4B') + assert vals os.remove(os.path.join(homedir,'ukmon.ini')) assert vals['RMSCFG'] == '~/source/RMS/.config' From 2b0ce9353a290e2bcce0f92e5634f9d5c182c054 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 15:12:33 +0000 Subject: [PATCH 36/50] fix inconsistent return and use of readinifile --- sendToLive.py | 2 +- ukmonInstaller.py | 2 ++ ukmonPostProc.py | 4 +++- uploadToArchive.py | 12 ++++++++---- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/sendToLive.py b/sendToLive.py index 8fcf5f6..88eb379 100644 --- a/sendToLive.py +++ b/sendToLive.py @@ -164,7 +164,7 @@ def singleUpload(cap_dir, dir_file, stationid=None): # get camera location from ini file inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) - if not inifvals: + if not inifvals or inifvals['LOCATION']=='NOTCONFIGURED': log.error('ukmon ini file invalid - check LOCATION') return 'ukmon ini file invalid - check LOCATION' camloc = inifvals['LOCATION'] diff --git a/ukmonInstaller.py b/ukmonInstaller.py index e2715cd..be63da8 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -266,6 +266,8 @@ def checkPlatepar(homedir, statid, rmsloc): """ homedir = os.path.expanduser(os.path.normpath(homedir)) inifvals = readIniFile(os.path.join(homedir, 'ukmon.ini'), statid) + if not inifvals or inifvals['LOCATION']=='NOTCONFIGURED': + return ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: diff --git a/ukmonPostProc.py b/ukmonPostProc.py index e1328c3..ca079b7 100644 --- a/ukmonPostProc.py +++ b/ukmonPostProc.py @@ -100,6 +100,8 @@ def rmsExternal(cap_dir, arch_dir, config): myloc = os.path.split(os.path.abspath(__file__))[0] inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), config.stationID) + if not inifvals or inifvals['LOCATION']=='NOTCONFIGURED': + return False log.info('app home is {}'.format(myloc)) domp4s = 0 if 'DOMP4S' in inifvals: @@ -190,7 +192,7 @@ def manualRerun(dated_dir, rmscfg = '~/source/RMS/.config'): stationid = arch_dir.split('_')[0] myloc = os.path.split(os.path.abspath(__file__))[0] inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) - if inifvals is None: + if not inifvals or inifvals['LOCATION']=='NOTCONFIGURED': print('ukmon ini file invalid - check LOCATION') exit(1) try: diff --git a/uploadToArchive.py b/uploadToArchive.py index 9760165..66e76e8 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -40,6 +40,8 @@ def getLatestKeys(homedir, stationid, remoteinifname='ukmon.ini'): """ homedir = os.path.expanduser(os.path.normpath(homedir)) inifvals = readIniFile(os.path.join(homedir, 'ukmon.ini'), stationid) + if not inifvals or inifvals['LOCATION']=='NOTCONFIGURED': + return False ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) pkey = paramiko.RSAKey.from_private_key_file(os.path.expanduser(inifvals['UKMONKEY'])) @@ -200,10 +202,12 @@ def readIniFile(filename, stationid): thiscam = [x for x in stations if stationid.lower() in x[0]] if len(thiscam)==0: log.error('camera {} not in cameras.ini, cannot continue'.format(stationid)) + print('missing camera file') return None location = thiscam[0][1] if not os.path.isfile(filename): log.error('{} missing, cannot continue'.format(filename)) + print('missing file') return None lis = open(filename, 'r').readlines() vals = {} @@ -222,8 +226,8 @@ def readIniFile(filename, stationid): vals['UKMONKEY'] = '~/.ssh/ukmon_' + stationid.upper() if os.path.isfile(os.path.expanduser('~/source/Stations/' + stationid + '/.config')): vals['RMSCFG'] = os.path.expanduser('~/source/Stations/' + stationid + '/.config') - if vals['LOCATION'] == 'NOTCONFIGURED': - return None + #if vals['LOCATION'] == 'NOTCONFIGURED': + # return None return vals @@ -386,7 +390,7 @@ def uploadToArchive(arch_dir, stationid, sciencefiles=False, keys=False): myloc = os.path.split(os.path.abspath(__file__))[0] inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) - if not inifvals: + if inifvals['LOCATION']=='NOTCONFIGURED': return False if not keys: keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) @@ -497,7 +501,7 @@ def manualUpload(targ_dir, stationid, sciencefiles=False): for cam in stations: stationid = cam[0] inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) - if not inifvals: + if inifvals['LOCATION']=='NOTCONFIGURED': continue if stationid is not None: stationid = stationid.upper() From ff4c87a381e7b2b861801505bc6959d6ed067239 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 15:12:40 +0000 Subject: [PATCH 37/50] fix readini tests --- tests/test_uploadToArchive.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py index ecf8b57..3e8f2c5 100644 --- a/tests/test_uploadToArchive.py +++ b/tests/test_uploadToArchive.py @@ -24,7 +24,7 @@ def test_checkMags(): def test_readIniFile(): - inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'),stationid='TESTPI4B') + inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'),stationid='UK0006') assert inifs['LOCATION']=='testpi4' @@ -34,11 +34,11 @@ def test_readKeyFile(): assert vals['S3FOLDER'] in ['tmp/testpi4','archive/Tackley'] -def test_readKeyfileIni(): +def test_createAndReadDefaultIni(): homedir = os.path.expanduser(os.path.join(basedir, 'output')) createDefaultIni(homedir) - vals = readIniFile(os.path.join(homedir,'ukmon.ini'), stationid='TESTPI4B') - assert vals + vals = readIniFile(os.path.join(homedir,'ukmon.ini'), stationid='UK0006') + assert vals is not None os.remove(os.path.join(homedir,'ukmon.ini')) assert vals['RMSCFG'] == '~/source/RMS/.config' From 5e5560936db8644eef0aa9a53d92867c7db5784c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 15:23:59 +0000 Subject: [PATCH 38/50] fix to createandReadDefaultIni --- tests/test_uploadToArchive.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py index 3e8f2c5..1ba88ac 100644 --- a/tests/test_uploadToArchive.py +++ b/tests/test_uploadToArchive.py @@ -35,9 +35,11 @@ def test_readKeyFile(): def test_createAndReadDefaultIni(): + # this should succeed because there's no config file in ~/source/Stations/NOTREAL + # and so a default value of ~/source/RMS/.config should be used homedir = os.path.expanduser(os.path.join(basedir, 'output')) createDefaultIni(homedir) - vals = readIniFile(os.path.join(homedir,'ukmon.ini'), stationid='UK0006') + vals = readIniFile(os.path.join(homedir,'ukmon.ini'), stationid='NOTREAL') assert vals is not None os.remove(os.path.join(homedir,'ukmon.ini')) assert vals['RMSCFG'] == '~/source/RMS/.config' From 18315c83bd683e8444bb5046c0335e6987d1ba4d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 16:32:06 +0000 Subject: [PATCH 39/50] add script to find old loc when migrating --- ukmonInstaller.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ukmonInstaller.py b/ukmonInstaller.py index be63da8..01126e3 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -80,6 +80,16 @@ def validateIni(homedir, newhelperip=None): return True +def findLocationFromOldIni(stationid): + inif = os.path.expanduser('~/source/ukmon-pitools-{}/ukmon.ini'.format(stationid)) + location = 'NOTCONFIGURED' + if os.path.isfile(inif): + flis = open(inif, 'r').readlines() + loc = [x for x in flis if 'LOCA' in x] + location = loc[0].strip().split('=')[1] + return location + + def updateMp4andMag(inif, homedir): """ Move the mp4 flag into the ini file and add the maglim flag if missing From 5966b594e3e7466b5eecd47aa9e62540f066656e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 16:33:56 +0000 Subject: [PATCH 40/50] migration process for locations --- setupUkmon.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setupUkmon.sh b/setupUkmon.sh index 56220b4..c91a3b4 100755 --- a/setupUkmon.sh +++ b/setupUkmon.sh @@ -36,7 +36,8 @@ if [ ! -f $here/cameras.ini ] ; then echo "# camera mapping file" > $here/cameras.ini echo "# echo add all cameras on this PC or Pi even if you only have one camera" >> $here/cameras.ini echo "[cameras]" >> $here/cameras.ini - echo "${CAMID}=NOTCONFIGURED" >> $here/cameras.ini + loc=$(python -c "from ukmonInstaller import findLocationFromOldIni;print(findLocationFromOldIni('$CAMID'))") + echo "${CAMID}=${loc}" >> $here/cameras.ini else grep $CAMID $here/cameras.ini > /dev/null 2>&1 if [ $? == 1 ] ; then @@ -68,6 +69,7 @@ pip list | grep paramiko || pip install paramiko LOCATION=$(grep $CAMID $here/cameras.ini) LOCATION=$(echo $LOCATION | awk -F "=" '{print $2}') + # if the station is configured, retrieve the AWS keys and test connectivity. if [[ "$LOCATION" != "NOTCONFIGURED" && "$LOCATION" != "" ]] ; then # check if RMS is still updating - its taking longer and longer From 53cce95b8a6e3859b31b37bebdb79bfa9c45ab85 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 16:37:10 +0000 Subject: [PATCH 41/50] more work to migrate existing setups --- setupUkmon.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/setupUkmon.sh b/setupUkmon.sh index c91a3b4..b4ba5b4 100755 --- a/setupUkmon.sh +++ b/setupUkmon.sh @@ -36,14 +36,12 @@ if [ ! -f $here/cameras.ini ] ; then echo "# camera mapping file" > $here/cameras.ini echo "# echo add all cameras on this PC or Pi even if you only have one camera" >> $here/cameras.ini echo "[cameras]" >> $here/cameras.ini +fi +grep $CAMID $here/cameras.ini > /dev/null 2>&1 +if [ $? == 1 ] ; then loc=$(python -c "from ukmonInstaller import findLocationFromOldIni;print(findLocationFromOldIni('$CAMID'))") echo "${CAMID}=${loc}" >> $here/cameras.ini -else - grep $CAMID $here/cameras.ini > /dev/null 2>&1 - if [ $? == 1 ] ; then - echo "${CAMID}=NOTCONFIGURED" >> $here/cameras.ini - fi -fi +fi source $here/ukmon.ini # creating an ssh key if not already present From 0cd968facd163fc1e5de41d3080630a684ad379b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 16:53:26 +0000 Subject: [PATCH 42/50] bugfix in liveMonitor.sh --- liveMonitor.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/liveMonitor.sh b/liveMonitor.sh index cfbe9a4..b18c453 100755 --- a/liveMonitor.sh +++ b/liveMonitor.sh @@ -13,12 +13,7 @@ pids=$(ps -ef | grep ${here}/liveMonitor | egrep -v "grep|$$" | awk '{print $2}' source $here/ukmon.ini -if [ "$LOCATION" == "NOTCONFIGURED" ]; then - echo "station not configured, unable to continue" - exit 1 -fi -rmsdir=$(dirname $RMSCFG) -cd $rmsdir +cd ~/source/RMS export PYTHONPATH=$here:~/source/RMS if [ -f $here/cameras.ini ] ; then cat $here/cameras.ini | grep = | while read i @@ -26,5 +21,9 @@ if [ -f $here/cameras.ini ] ; then python $here/liveMonitor.py ${i:7:60} ${i:0:6} & done else + if [ "$LOCATION" == "NOTCONFIGURED" ]; then + echo "station not configured, unable to continue" + exit 1 + fi python $here/liveMonitor.py $LOCATION & fi From b81440f8f6d3637aaa3f1b91708ca4659e345a4c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 23:03:23 +0000 Subject: [PATCH 43/50] relocate the git repo --- refreshTools.sh | 2 ++ ukmonInstaller.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/refreshTools.sh b/refreshTools.sh index 4e6d978..6199633 100755 --- a/refreshTools.sh +++ b/refreshTools.sh @@ -26,6 +26,8 @@ git stash git pull git stash apply +python -c "from ukmonInstaller import relocateGitRepo;relocateGitRepo()" + echo "testing connections" python $here/sendToLive.py test test python $here/uploadToArchive.py test diff --git a/ukmonInstaller.py b/ukmonInstaller.py index 01126e3..d058639 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -4,6 +4,7 @@ import shutil from crontab import CronTab from subprocess import call +from git import remote, Repo import time import warnings @@ -90,6 +91,21 @@ def findLocationFromOldIni(stationid): return location +def relocateGitRepo(): + myloc = os.path.split(os.path.abspath(__file__))[0] + thisrepo = Repo(myloc) + origin = thisrepo.remote('origin') + if 'markmac99' in origin.url: + origin.rename('upstream') + remote.Remote.add(thisrepo, 'origin','https://github.com/ukmda/ukmon-pitools.git') + cfg = thisrepo.heads.main.config_writer() + cfg.set('remote','origin') + cfg = thisrepo.heads.dev.config_writer() + cfg.set('remote','origin') + log.info('git remote updated') + return + + def updateMp4andMag(inif, homedir): """ Move the mp4 flag into the ini file and add the maglim flag if missing From ad4423b7f4366ddee174d3cabe27e7e377e80ddb Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 23:03:45 +0000 Subject: [PATCH 44/50] remove incorrect message --- ukmonPostProc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ukmonPostProc.py b/ukmonPostProc.py index ca079b7..6734e06 100644 --- a/ukmonPostProc.py +++ b/ukmonPostProc.py @@ -183,7 +183,6 @@ def manualRerun(dated_dir, rmscfg = '~/source/RMS/.config'): if len(sys.argv) < 2: print('usage: python ukmonPostProc.py arc_dir_name') print('eg python ukmonPostProc.py UK0006_20210312_183741_206154') - print('\n nb: script must be run from RMS source folder') exit(0) arch_dir = sys.argv[1] From e838c800a001f2924c19023f2d8549046f5986b8 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 23:19:07 +0000 Subject: [PATCH 45/50] bugfix in git updater --- ukmonInstaller.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ukmonInstaller.py b/ukmonInstaller.py index d058639..068300b 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -100,8 +100,12 @@ def relocateGitRepo(): remote.Remote.add(thisrepo, 'origin','https://github.com/ukmda/ukmon-pitools.git') cfg = thisrepo.heads.main.config_writer() cfg.set('remote','origin') + cfg.write() + cfg.release() cfg = thisrepo.heads.dev.config_writer() cfg.set('remote','origin') + cfg.write() + cfg.release() log.info('git remote updated') return From 81993ff9a715477b6ed27937442ae662d3f30e93 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 23:20:45 +0000 Subject: [PATCH 46/50] another bugfix --- ukmonInstaller.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ukmonInstaller.py b/ukmonInstaller.py index 068300b..ff3e53d 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -100,11 +100,9 @@ def relocateGitRepo(): remote.Remote.add(thisrepo, 'origin','https://github.com/ukmda/ukmon-pitools.git') cfg = thisrepo.heads.main.config_writer() cfg.set('remote','origin') - cfg.write() cfg.release() cfg = thisrepo.heads.dev.config_writer() cfg.set('remote','origin') - cfg.write() cfg.release() log.info('git remote updated') return From e80b3cc9f512dfb9af30ad44ba5502e529372d1c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 23:32:16 +0000 Subject: [PATCH 47/50] improve message --- ukmonInstaller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ukmonInstaller.py b/ukmonInstaller.py index ff3e53d..6cd223f 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -104,7 +104,7 @@ def relocateGitRepo(): cfg = thisrepo.heads.dev.config_writer() cfg.set('remote','origin') cfg.release() - log.info('git remote updated') + print('git remote updated') return From 92c2f27ee6a84c5e5f7b841afb8e483141ce06b0 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 23:35:18 +0000 Subject: [PATCH 48/50] set git pull confir --- setupUkmon.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/setupUkmon.sh b/setupUkmon.sh index b4ba5b4..2dac40b 100755 --- a/setupUkmon.sh +++ b/setupUkmon.sh @@ -20,6 +20,7 @@ source ~/vRMS/bin/activate cd $here echo "refreshing toolset" +git config pull.ff only git stash git pull git stash apply From c65dcad31c6d9c47843ccf88a3d3fb74d58d6111 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 23:43:07 +0000 Subject: [PATCH 49/50] more tweaks to refreshTools --- refreshTools.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/refreshTools.sh b/refreshTools.sh index 6199633..9fc9d09 100755 --- a/refreshTools.sh +++ b/refreshTools.sh @@ -22,11 +22,13 @@ python -c "import ukmonInstaller as pp ; pp.validateIni('${here}', '3.11.55.160' source $here/ukmon.ini echo "refreshing toolset" +git config pull.ff only git stash git pull git stash apply python -c "from ukmonInstaller import relocateGitRepo;relocateGitRepo()" +git fetch echo "testing connections" python $here/sendToLive.py test test From 50909b1cf30978bda2890521b18c01cea0b7f10d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 7 Jan 2026 23:46:00 +0000 Subject: [PATCH 50/50] bugfix to prevent crash while testing --- uploadToArchive.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uploadToArchive.py b/uploadToArchive.py index 66e76e8..27cfb81 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -42,6 +42,8 @@ def getLatestKeys(homedir, stationid, remoteinifname='ukmon.ini'): inifvals = readIniFile(os.path.join(homedir, 'ukmon.ini'), stationid) if not inifvals or inifvals['LOCATION']=='NOTCONFIGURED': return False + if not os.path.isfile(os.path.expanduser(inifvals['UKMONKEY'])): + return False ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) pkey = paramiko.RSAKey.from_private_key_file(os.path.expanduser(inifvals['UKMONKEY']))