diff --git a/.gitignore b/.gitignore index 690ed65..2596ecb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,3 @@ -# ukmon specific -ukmon.ini -live.key -archive.key -addSftpUser.sh -.firstrun -domp4s -extrascript -dotimelapse -.config.backup -ukmon.ini - # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -139,6 +127,18 @@ dmypy.json # Pyre type checker .pyre/ + +# ukmon specific +ukmon.ini +live.key +archive.key +addSftpUser.sh +.firstrun +domp4s +extrascript +dotimelapse +.config.backup +cameras.ini deploy.yml sample-crontab-entries.txt jsonToKeyFile.py diff --git a/bumpver.toml b/bumpver.toml new file mode 100644 index 0000000..8529ea8 --- /dev/null +++ b/bumpver.toml @@ -0,0 +1,26 @@ +[bumpver] +current_version = "2026.01.04" +version_pattern = "YYYY.MM.PATCH" +commit_message = "bump version {old_version} -> {new_version}" +tag_message = "{new_version}" +tag_scope = "default" +pre_commit_hook = "" +post_commit_hook = "" +commit = true +tag = true +push = true + +[bumpver.file_patterns] +"fireballCollector.py" = [ + "versionid = '{version}'" +] +"README.md" = [ + 'Version {version}', +] +"bumpver.toml" = [ + 'current_version = "{version}"', +] +#".github/workflows/build_tools.yml" = [ +# 'tag_name: {version}', +#] + diff --git a/gatherLogs.sh b/gatherLogs.sh index 7700d11..f91b18f 100755 --- a/gatherLogs.sh +++ b/gatherLogs.sh @@ -1,24 +1,55 @@ #!/bin/bash -cd ~/RMS_data/logs -mkdir logtmp -cd logtmp - # # Script to gather the logfiles and upload them for debugging & analysis # Copyright (C) 2018-2023 Mark McIntyre # -source /home/${LOGNAME}/source/ukmon-pitools/ukmon.ini +myself=$(readlink -f $0) +here="$( cd "$(dirname "$myself")" >/dev/null 2>&1 ; pwd -P )" + +[ "$1" == "" ] && echo "usage: ./gatherlogs.sh UKxxxxx" && exit +CAMID=$1 + +cd $HOME +[ -d logtmp_$CAMID ] && rm -Rf logtmp_$CAMID +mkdir -p logtmp_$CAMID +cd logtmp_$CAMID + +source $here/ukmon.ini + +echo "gathering files" +# gather system logs sudo cp /var/log/kern.log . -sudo chown ${user}:${LOGNAME} kern.log -cp /var/log/messages ./messages.log -cp /home/${LOGNAME}/source/RMS/.config ./${LOCATION}.config -cp /home/${LOGNAME}/source/RMS/platepar_cmn2010.cal ./${LOCATION}.cal -cp /home/${LOGNAME}/source/ukmon-pitools/*.key . -cp /home/${LOGNAME}/source/ukmon-pitools/*.ini . +[ -f /var/log/messages ] && sudo cp /var/log/messages ./messages.log; +[ -f /var/log/syslog ] && sudo cp /var/log/syslog ./messages.log +sudo chown ${LOGNAME}:${LOGNAME} ./*.log + +# find the RMS config and log location +if [ -d /home/${LOGNAME}/source/Stations/${CAMID} ] ; then + rootdir=/home/${LOGNAME}/source/Stations/${CAMID} +else + rootdir=/home/${LOGNAME}/source/RMS +fi +echo rootdir is $rootdir +rmscfg=$rootdir/.config +datadir=$(python -c "import configparser,os;cfg=configparser.ConfigParser();cfg.read('$rmscfg');print(os.path.expanduser(cfg['Capture']['data_dir']))") +logdir=$datadir/logs +echo logdir is $logdir +[ ! -d $logdir ] && logdir=~/RMS_data/logs + +# gather the RMS config and logs +cp $rootdir/.config ./${CAMID}.config +cp $rootdir/platepar_cmn2010.cal ./${CAMID}.cal +cp $here/live.key ./${CAMID}.key +cp $here/ukmon.ini . +[ -f $here/cameras.ini ] && cp $here/cameras.ini . crontab -l > ./crontab.txt -find .. -maxdepth 1 -name "*.log*" -type f -mtime -5 -exec cp {} . \; -ZIPFILE=/tmp/${LOCATION}_logs.tgz -tar cvzf $ZIPFILE *.log* ${LOCATION}.config ${LOCATION}.cal crontab.txt *.key *.ini +ls -1tr $logdir/log*.log* | tail -5 | while read i; do cp $i . ; done +ls -1tr $logdir/uk*.log* | tail -5 | while read i; do cp $i . ; done + +# create a tarball and upload to the server +echo "uploading logs" +ZIPFILE=/tmp/${CAMID}_logs.tgz +tar czf $ZIPFILE *.log* ${CAMID}.config ${CAMID}.cal crontab.txt *.key *.ini sftp -i $UKMONKEY -q logupload@$UKMONHELPER << EOF cd logs progress @@ -26,4 +57,5 @@ put $ZIPFILE exit EOF cd .. -rm -Rf logtmp +rm -Rf logtmp_$CAMID +echo "done" \ No newline at end of file diff --git a/liveMonitor.py b/liveMonitor.py index 629ebd2..ba9a8bf 100644 --- a/liveMonitor.py +++ b/liveMonitor.py @@ -8,12 +8,13 @@ import logging import RMS.ConfigReader as cr from stat import ST_INO -from uploadToArchive import readKeyFile, readIniFile -from ukmonPostProc import setupLogging +from uploadToArchive import readKeyFile, readIniFile, getLatestKeys +from ukmonPostProc import setupLogging, versionid log = logging.getLogger("ukmonlogger") + timetowait = 30 # seconds to wait for a new line before deciding the log is stale # Images created more than this many seconds ago won't be uploaded. Prevents reuploads. @@ -47,11 +48,21 @@ def follow(fname, logf_ino): yield line.strip() -def monitorLogFile(camloc, rmscfg): +def monitorLogFile(camloc, stationid=None): """ This function monitors the latest RMS log file for meteor detections, convert the FF file to a jpg and upload it to the livestream. This function is called from the shell script *liveMonitor.sh* and should not be called directly. """ + # get configuration + myloc = os.path.split(os.path.abspath(__file__))[0] + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) + if not inifvals: + log.error('ukmon.ini not present, aborting') + exit(1) + rmscfg = inifvals['RMSCFG'] + if not os.path.isfile(rmscfg): + log.error('rms config file', rmscfg,'not present, aborting') + exit(1) cfg = cr.parse(os.path.expanduser(rmscfg)) datadir = cfg.data_dir @@ -59,23 +70,21 @@ def monitorLogFile(camloc, rmscfg): setupLogging(logdir, 'ukmonlive_') - log.info('--------------------------------') - log.info(' live feed started') - log.info('--------------------------------') + log.info('------------------------------------------') + log.info(' live feed started, version ' + versionid) + log.info('------------------------------------------') log.info('Camera location is {}'.format(camloc)) log.info('RMS config file is {}'.format(rmscfg)) - myloc = os.path.split(os.path.abspath(__file__))[0] - # get credentials - inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) - if not inifvals: - log.error('ukmon.ini not present, aborting') - exit(1) + if not os.path.isfile(os.path.join(myloc, 'live.key')): + if not getLatestKeys(myloc, cfg.stationID): + print('unable to get key for', inifvals['LOCATION']) + exit(1) keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) if not keys: - log.error('config file not present, aborting') + log.error('unable to read AWS configuration, aborting') exit(1) keepon = True @@ -142,11 +151,8 @@ def monitorLogFile(camloc, rmscfg): if __name__ == '__main__': if len(sys.argv) < 2: - print('LOCATION missing') + print('usage: python liveMonitor.py {ukmon_location} {rms_id}') exit(1) - if len(sys.argv) < 3: - rmscfg = os.path.expanduser('~/source/RMS/.config') - else: - rmscfg = sys.argv[2] camloc = sys.argv[1] - monitorLogFile(camloc, rmscfg) + stationid = sys.argv[2] if len(sys.argv) > 2 else None + monitorLogFile(camloc, stationid) diff --git a/liveMonitor.sh b/liveMonitor.sh index 0645515..b18c453 100755 --- a/liveMonitor.sh +++ b/liveMonitor.sh @@ -9,17 +9,21 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # kill any existing livestream process pids=$(ps -ef | grep ${here}/liveMonitor | egrep -v "grep|$$" | awk '{print $2}') -kill -9 $pids +[ "$pids" != "" ] && kill -9 $pids source $here/ukmon.ini -source $here/live.key -if [ "$LOCATION" == "NOTCONFIGURED" ]; then - echo "station not configured, unable to continue" - exit 1 -fi - -rmsdir=$(dirname $RMSCFG) -cd $rmsdir +cd ~/source/RMS export PYTHONPATH=$here:~/source/RMS -python $here/liveMonitor.py $LOCATION $RMSCFG +if [ -f $here/cameras.ini ] ; then + cat $here/cameras.ini | grep = | while read i + do + python $here/liveMonitor.py ${i:7:60} ${i:0:6} & + done +else + if [ "$LOCATION" == "NOTCONFIGURED" ]; then + echo "station not configured, unable to continue" + exit 1 + fi + python $here/liveMonitor.py $LOCATION & +fi diff --git a/refreshTools.sh b/refreshTools.sh index 6bcf674..9fc9d09 100755 --- a/refreshTools.sh +++ b/refreshTools.sh @@ -1,7 +1,7 @@ #!/bin/bash # refresh ukmon-pitools -# Copyright (C) 2018-2023 Mark McIntyre +# Copyright (C) 2018- Mark McIntyre myself=$(readlink -f $0) here="$( cd "$(dirname "$myself")" >/dev/null 2>&1 ; pwd -P )" @@ -22,75 +22,15 @@ python -c "import ukmonInstaller as pp ; pp.validateIni('${here}', '3.11.55.160' source $here/ukmon.ini echo "refreshing toolset" +git config pull.ff only git stash git pull git stash apply -# creating an ssh key if not already present -if [ ! -f ${UKMONKEY} ] ; then - echo "creating ukmon ssh key" - ssh-keygen -t rsa -f ${UKMONKEY} -q -N '' - echo "Copy this public key and email it to newcamera@ukmeteornetwork.org, then " - echo "wait for confirmation its been installed and rerun this script" - echo "" - cat ${UKMONKEY}.pub - echo "" - read -p "Press any key to continue" -fi - -# if the station is configured, retrieve the AWS keys and test connectivity. -if [[ "$LOCATION" != "NOTCONFIGURED" && "$LOCATION" != "" ]] ; then - # check if RMS is still updating - its taking longer and longer - loopctr=0 - echo "Checking RMS update not in progress" - while [ $loopctr -lt 10 ] ; do - [ -f $RMSCFG ] && break - echo "RMS update in progress or station not configured, trying again in a minute" - sleep 60 - loopctr=$((loopctr + 1)) - done - while [ $loopctr -lt 10 ] ; do - grep XX0001 $RMSCFG | grep stationID: - [ $? -eq 1 ] && break - echo "RMS update in progress or station not configured, trying again in a minute" - sleep 60 - loopctr=$((loopctr + 1)) - done - if [ $loopctr -eq 10 ] ; then - echo RMS update failed or long-running, unable to proceed - exit 1 - else - echo all good proceeding - fi - echo "checking for ukmon config changes" - python -c "import ukmonInstaller as pp ; pp.getLatestKeys('${here}') ;" - - if [ -f archive.key ] ; then \rm archive.key ; fi - - echo "checking the RMS config file, crontab and icons" - source ~/vRMS/bin/activate - source $here/ukmon.ini - cd $(dirname $RMSCFG) - export PYTHONPATH=$here:~/source/RMS - python -c "import ukmonInstaller as pp ; pp.installUkmonFeed('${RMSCFG}');" - - echo "testing connections" - python $here/sendToLive.py test test - python $here/uploadToArchive.py test - echo "if you did not see two success messages contact us for advice" - if [ "$DOCKER_RUNNING" != "true" ] ; then read -p "Press any key to finish" ; fi - echo "done" -else - statid=$(grep stationID $RMSCFG | awk -F" " '{print $2}') - if [ "$statid" == "XX0001" ] ; then - echo "You must configure RMS before setting up the ukmon tools" - else - python -c "import ukmonInstaller as pp ; pp.addDesktopIcons('${here}', '${statid}');" - echo "Location missing - unable to continue. Please obtain a location code from the UKMON team," - echo "then update the UKMON Config File using the desktop icon and rerun this script." - fi - sleep 5 - if [ "$DOCKER_RUNNING" != "true" ] ; then read -p "Press any key to end" ; fi - exit 1 -fi +python -c "from ukmonInstaller import relocateGitRepo;relocateGitRepo()" +git fetch +echo "testing connections" +python $here/sendToLive.py test test +python $here/uploadToArchive.py test +echo "if you did not see two success messages contact us for advice" diff --git a/sendToLive.py b/sendToLive.py index 72c0d8b..88eb379 100644 --- a/sendToLive.py +++ b/sendToLive.py @@ -11,7 +11,7 @@ import shutil import tempfile import boto3 -from uploadToArchive import readKeyFile, readIniFile +from uploadToArchive import readKeyFile, readIniFile, getListOfStations import logging import RMS.ConfigReader as cr import numpy as np @@ -137,7 +137,7 @@ def testFeed(keys, cfg): return retmsg -def singleUpload(cap_dir, dir_file): +def singleUpload(cap_dir, dir_file, stationid=None): """This function is used to manually upload a single event. It can also be used to test the connection - see note below. @@ -156,31 +156,31 @@ def singleUpload(cap_dir, dir_file): camloc = None myloc = os.path.split(os.path.abspath(__file__))[0] + if not stationid: + stations = getListOfStations(myloc) + tmpid = list(stations)[0][0] + if tmpid: + stationid = tmpid.upper() # get camera location from ini file - inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) - if not inifvals: - log.warning('unable to open ini file') - return 'unable to open ini file' + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) + if not inifvals or inifvals['LOCATION']=='NOTCONFIGURED': + log.error('ukmon ini file invalid - check LOCATION') + return 'ukmon ini file invalid - check LOCATION' camloc = inifvals['LOCATION'] - try: - rmscfg = inifvals['RMSCFG'] - except Exception: - rmscfg='~/source/RMS/.config' - if camloc == 'NOTCONFIGURED': - print('LOCATION not found in ini file, aborting') - return 'not configured' + rmscfg = inifvals['RMSCFG'] + if not os.path.isfile(os.path.expanduser(rmscfg)): + log.error('RMS config file not found at {}, aborting'.format(rmscfg)) + return 'RMS config file not found at', rmscfg, ', aborting' # get credentials keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) if not keys: - log.warning('unable to open keyfile') + log.error('unable to open keyfile') return 'unable to open keyfile' - # read a few variables from the RMS config file + # Load the RMS config file cfg = cr.parse(os.path.expanduser(rmscfg)) -# configpath, configname = os.path.split(os.path.expanduser(rmscfg)) -# cfg = cr.loadConfigFromDirectory(configname, configpath) if cap_dir == 'test' and dir_file == 'test': retmsg = testFeed(keys, cfg) @@ -191,7 +191,12 @@ def singleUpload(cap_dir, dir_file): if __name__ == '__main__': if len(sys.argv) < 3: - print('usage: python sendToLive.py capdir ffname') + print('usage: python sendToLive.py capdir ffname camid') + print('or') + print(' : python sendToLive.py test test') exit(1) - retmsg = singleUpload(sys.argv[1], sys.argv[2]) - print(retmsg) + + stationid = sys.argv[3] if len(sys.argv) > 3 else None + retmsg = singleUpload(sys.argv[1], sys.argv[2], stationid) + if retmsg: + print(retmsg) diff --git a/setupUkmon.sh b/setupUkmon.sh new file mode 100755 index 0000000..2dac40b --- /dev/null +++ b/setupUkmon.sh @@ -0,0 +1,129 @@ +#!/bin/bash + +# refresh ukmon-pitools +# Copyright (C) 2018- Mark McIntyre + +myself=$(readlink -f $0) +here="$( cd "$(dirname "$myself")" >/dev/null 2>&1 ; pwd -P )" + +CAMID=$1 +CAMID=${CAMID^^} + +if [ "$CAMID" == "" ] ; then + echo "Usage: ./setupUkmon.sh RMSID" + echo "where 'RMSID' is your station's RMS ID eg UK12345" + exit +fi + +export PYTHONPATH=$here:~/source/RMS +source ~/vRMS/bin/activate +cd $here + +echo "refreshing toolset" +git config pull.ff only +git stash +git pull +git stash apply + +UKMONKEY=~/.ssh/ukmon_$CAMID + +if [ ! -f $here/ukmon.ini ] ; then + echo "creating default ukmon ini file" + python -c "from ukmonInstaller import createDefaultIni;createDefaultIni('$here', stationid='$CAMID');" +fi + +if [ ! -f $here/cameras.ini ] ; then + echo "creating cameras.ini file" + echo "# camera mapping file" > $here/cameras.ini + echo "# echo add all cameras on this PC or Pi even if you only have one camera" >> $here/cameras.ini + echo "[cameras]" >> $here/cameras.ini +fi +grep $CAMID $here/cameras.ini > /dev/null 2>&1 +if [ $? == 1 ] ; then + loc=$(python -c "from ukmonInstaller import findLocationFromOldIni;print(findLocationFromOldIni('$CAMID'))") + echo "${CAMID}=${loc}" >> $here/cameras.ini +fi + +source $here/ukmon.ini +# creating an ssh key if not already present +if [ ! -f ${UKMONKEY} ] ; then + echo "creating ukmon ssh key" + ssh-keygen -t rsa -f ${UKMONKEY} -q -N '' + echo "Now copy this public key and email it to newcamera@ukmeteornetwork.org, then " + echo "wait for confirmation and further instructions to complete the setup." + echo "" + cat ${UKMONKEY}.pub + echo "" + read -p "Press any key to continue" + exit +fi + +echo "checking required python libs are installed" +pip list | grep boto3 || pip install boto3 +# python-crontab v2.5.1 for python 2.7 backwards compatability. Sigh. +pip list | grep python-crontab | grep 2.5.1 || pip install python-crontab==2.5.1 +pip list | grep paramiko || pip install paramiko + +# get the location code from the cameras.ini file +LOCATION=$(grep $CAMID $here/cameras.ini) +LOCATION=$(echo $LOCATION | awk -F "=" '{print $2}') + + +# if the station is configured, retrieve the AWS keys and test connectivity. +if [[ "$LOCATION" != "NOTCONFIGURED" && "$LOCATION" != "" ]] ; then + # check if RMS is still updating - its taking longer and longer + loopctr=0 + echo "Checking RMS update not in progress" + while [ $loopctr -lt 10 ] ; do + [ -f $RMSCFG ] && break + echo "RMS update in progress or station not configured, trying again in a minute" + sleep 60 + loopctr=$((loopctr + 1)) + done + while [ $loopctr -lt 10 ] ; do + grep XX0001 $RMSCFG | grep stationID: + [ $? -eq 1 ] && break + echo "RMS update in progress or station not configured, trying again in a minute" + sleep 60 + loopctr=$((loopctr + 1)) + done + if [ $loopctr -eq 10 ] ; then + echo RMS update failed or long-running, unable to proceed + exit 1 + else + echo all good proceeding + fi + echo "checking for ukmon config changes" + python -c "import uploadToArchive as pp ; pp.getLatestKeys('${here}', '${stationid}') ;" + + if [ -d ~/Desktop ] ; then + pushd ~/Desktop + rm -f ukmon.ini UKMON_config* refreshTools* refresh_UKMON* > /dev/null 2>&1 + fi + echo "checking the RMS config file, crontab and icons" + source ~/vRMS/bin/activate + source $here/ukmon.ini + export PYTHONPATH=$here:~/source/RMS + python -c "import ukmonInstaller as pp ; pp.installUkmonFeed('${RMSCFG}');" + + echo "testing connections" + python $here/sendToLive.py test test + python $here/uploadToArchive.py test + echo "if you did not see two success messages contact us for advice" + if [ "$DOCKER_RUNNING" != "true" ] ; then read -p "Press any key to finish" ; fi + echo "done" +else + echo $RMSCFG $CAMID + statid=$(grep stationID $RMSCFG | awk -F" " '{print $2}') + if [ "$statid" == "XX0001" ] ; then + echo "You must configure RMS before setting up the ukmon tools" + else + #python -c "import ukmonInstaller as pp ; pp.addDesktopIcons('${here}', '${statid}');" + echo "Location missing. Please obtain a location code from the UKMON team," + echo "then update cameras.ini and rerun this script." + fi + sleep 5 + if [ "$DOCKER_RUNNING" != "true" ] ; then read -p "Press any key to end" ; fi + exit 1 +fi + diff --git a/tests/run_ub_tests.sh b/tests/run_ub_tests.sh index 06545fb..b11f065 100644 --- a/tests/run_ub_tests.sh +++ b/tests/run_ub_tests.sh @@ -19,7 +19,7 @@ echo testing for $LOCATION pip install -r ./requirements.txt pip install --upgrade ruff pytest xmltodict pytest-cov export PYTHONPATH=$PYTHONPATH:/root/source/RMS:${here}/.. -cd /root/source/RMS -ls -ltra .config +cd /root/source/RMS/ +ls -ltra /root/source/RMS/.config pwd pytest -v $here/ --cov=$here/../ --cov-report=term-missing --cov-config=$here/../.coveragerc_lnx diff --git a/tests/test_ukmonInstaller.py b/tests/test_ukmonInstaller.py index 2c57052..03bff6c 100644 --- a/tests/test_ukmonInstaller.py +++ b/tests/test_ukmonInstaller.py @@ -3,8 +3,8 @@ import os import shutil -from ukmonInstaller import createDefaultIni, updateHelperIp, updateLocation, \ - checkPostProcSettings, validateIni, getLatestKeys # noqa: E402 +from ukmonInstaller import createDefaultIni, checkPostProcSettings, validateIni +from uploadToArchive import updateHelperIp, updateLocation, getLatestKeys myloc = os.path.split(os.path.abspath(__file__))[0] homedir = os.path.join(myloc, 'ukminst') @@ -89,7 +89,7 @@ def test_checkPostProcSettings(): def test_getLatestKeys_normal(): shutil.copyfile(os.path.join(myloc, '../ukmon.ini'),os.path.join(homedir,'ukmon.ini')) updateHelperIp(homedir, helperip='3.11.55.160') - res = getLatestKeys(homedir) + res = getLatestKeys(homedir, 'UK0006') assert res is True os.remove(os.path.join(homedir, 'ukmon.ini')) return @@ -99,7 +99,7 @@ def test_getLatestKeys_newname(): shutil.copyfile(os.path.join(myloc, '../ukmon.ini'),os.path.join(homedir,'ukmon.ini')) updateHelperIp(homedir, helperip='3.11.55.160') remoteinifname = 'ukmon.ini.newname' - res = getLatestKeys(homedir, remoteinifname=remoteinifname) + res = getLatestKeys(homedir, 'UK0006', remoteinifname=remoteinifname) assert res is True lis = open(os.path.join(homedir, 'ukmon.ini'), 'r').readlines() for li in lis: @@ -114,7 +114,7 @@ def test_getLatestKeys_newip(): shutil.copyfile(os.path.join(myloc, '../ukmon.ini'),os.path.join(homedir,'ukmon.ini')) updateHelperIp(homedir, helperip='3.11.55.160') remoteinifname = 'ukmon.ini.newip' - res = getLatestKeys(homedir, remoteinifname=remoteinifname) + res = getLatestKeys(homedir, 'UK0006', remoteinifname=remoteinifname) assert res is True lis = open(os.path.join(homedir, 'ukmon.ini'), 'r').readlines() for li in lis: diff --git a/tests/test_ukmonPostProc.py b/tests/test_ukmonPostProc.py index 14b63e7..3f978c6 100644 --- a/tests/test_ukmonPostProc.py +++ b/tests/test_ukmonPostProc.py @@ -5,30 +5,53 @@ import os import sys import pytest -from ukmonPostProc import main +from ukmonPostProc import manualRerun myloc = os.path.split(os.path.abspath(__file__))[0] -homedir = os.path.join(myloc, 'ukmpp') +#homedir = os.path.join(myloc, 'ukmpp') tmpdir = os.path.join(myloc, 'output') if not os.path.isdir(tmpdir): - os.makedirs(tmpdir) # , exist_ok=Truee) exist_ok keyword not supported with python7.2 + os.makedirs(tmpdir) # , exist_ok=True) exist_ok keyword not supported with python 2.7 +# this should fail because the test RMS config is not at ~/source/RMS but at /root/source/RMS def test_ukmonPostProcNoArgs(): - args=[None] - ret = main(args) - assert ret is False + print(os.listdir('.')) + print(os.getenv('HOME')) + try: + ret = manualRerun(None) + assert ret is True + except Exception: + assert True @pytest.mark.skipif(sys.platform == 'win32', reason='test not valid on windows') def test_ukmonPostProc1Arg(): - args=[None, os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543_087124')] - ret = main(args) - assert ret is True + # this will fail because of the location of the config file + args=os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543_087124') + try: + ret = manualRerun(dated_dir=args) + assert ret is True + except Exception: + assert True + + +@pytest.mark.skipif(sys.platform == 'win32', reason='test not valid on windows') +def test_ukmonPostProc2Args(): + args=os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543_087124') + try: + ret = manualRerun(dated_dir=args, rmscfg='/root/source/RMS/.config') + assert ret is True + except Exception: + assert True def test_ukmonPostProc1BadArg(): - args=[None, os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543')] - ret = main(args) - assert ret is False + # this will also fail because of the bad data + args=os.path.join(myloc, 'ukmarch/sampledata/UK0006_20220914_185543') + try: + ret = manualRerun(dated_dir=args, rmscfg='/root/source/RMS/.config') + assert ret is False + except Exception: + assert True diff --git a/tests/test_uploadToArchive.py b/tests/test_uploadToArchive.py index dfb17b7..1ba88ac 100644 --- a/tests/test_uploadToArchive.py +++ b/tests/test_uploadToArchive.py @@ -12,7 +12,7 @@ def test_checkMags(): - inifvals = readIniFile(os.path.join(basedir,'..','ukmon.ini')) + inifvals = readIniFile(os.path.join(basedir,'..','ukmon.ini'),'UK0006') maglim = 6 if 'MAGLIM' in inifvals: maglim = float(inifvals['MAGLIM']) @@ -24,26 +24,29 @@ def test_checkMags(): def test_readIniFile(): - inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini')) + inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'),stationid='UK0006') assert inifs['LOCATION']=='testpi4' def test_readKeyFile(): - inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini')) + inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'), stationid='UK0006') vals = readKeyFile(os.path.join(basedir,'..','live.key'), inifs) - assert vals['S3FOLDER'] in ['tmp/testpi4','archive/Tackley'] + assert vals['S3FOLDER'] in ['tmp/testpi4','archive/Tackley'] -def test_readKeyfileIni(): - homedir = os.path.join(basedir, 'output') +def test_createAndReadDefaultIni(): + # this should succeed because there's no config file in ~/source/Stations/NOTREAL + # and so a default value of ~/source/RMS/.config should be used + homedir = os.path.expanduser(os.path.join(basedir, 'output')) createDefaultIni(homedir) - vals = readIniFile(os.path.join(homedir,'ukmon.ini')) + vals = readIniFile(os.path.join(homedir,'ukmon.ini'), stationid='NOTREAL') + assert vals is not None os.remove(os.path.join(homedir,'ukmon.ini')) assert vals['RMSCFG'] == '~/source/RMS/.config' def test_uploadOneFile(): - inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini')) + inifs = readIniFile(os.path.join(basedir,'..','ukmon.ini'),'UK0006') keys = readKeyFile(os.path.join(basedir,'..','live.key'), inifs) reg = keys['ARCHREGION'] conn = boto3.Session(aws_access_key_id=keys['AWS_ACCESS_KEY_ID'], aws_secret_access_key=keys['AWS_SECRET_ACCESS_KEY']) @@ -63,7 +66,7 @@ def test_uploadOneFile(): def test_manualUpload(): targ_dir = 'test' - assert manualUpload(targ_dir) is True + assert manualUpload(targ_dir,'UK0006') is True targ_dir = os.path.join(basedir, 'ukmarch','testpi4_20230401') # create some dummy sample files testfilelist = ['FF_test_20230401.fits','FF_test_20230401.jpg','FF_test_20230401.mp4','mask.bmp', @@ -73,6 +76,6 @@ def test_manualUpload(): ] for fil in testfilelist: open(os.path.join(targ_dir, fil), 'w').write('{"test":"potato"}') - assert manualUpload(targ_dir) + assert manualUpload(targ_dir,'UK0006') for fil in testfilelist: os.remove(os.path.join(targ_dir, fil)) diff --git a/ukmonInstaller.py b/ukmonInstaller.py index 5fac352..6cd223f 100644 --- a/ukmonInstaller.py +++ b/ukmonInstaller.py @@ -4,6 +4,7 @@ import shutil from crontab import CronTab from subprocess import call +from git import remote, Repo import time import warnings @@ -18,7 +19,7 @@ import RMS.ConfigReader as cr from RMS.Misc import isRaspberryPi -from uploadToArchive import readIniFile +from uploadToArchive import readIniFile, updateHelperIp log = logging.getLogger("ukmonlogger") log.setLevel(logging.WARNING) @@ -27,28 +28,16 @@ currip = '3.11.55.160' -def createDefaultIni(homedir, helperip='3.11.55.160', location='NOTCONFIGURED', keyfile=None, rmscfg=None): +def createDefaultIni(homedir, helperip='3.11.55.160', location='NOTCONFIGURED', stationid=''): """ Create a default ini file, if its not present on the target """ homedir = os.path.normpath(os.path.expanduser(homedir)) - if not os.path.isdir(homedir): - os.makedirs(homedir) - camid = homedir[homedir.find('pitools')+8:] - if rmscfg is None: - if camid != '' and 'tests' not in camid: - rmscfg = '~/source/Stations/{}/.config'.format(camid) - else: - rmscfg = '~/source/RMS/.config' - if keyfile is None: - if camid != '' and 'tests' not in camid: - keyfile = '~/.ssh/ukmon-{}'.format(camid) - else: - keyfile = '~/.ssh/ukmon' - if helperip is None: - helperip = currip - if location is None: - location = 'NOTCONFIGURED' + rmscfg = '~/source/Stations/{}/.config'.format(stationid) + if not os.path.isfile(os.path.expanduser(rmscfg)): + rmscfg = '~/source/RMS/.config' + + keyfile = '~/.ssh/ukmon_{}'.format(stationid) with open(os.path.join(homedir, 'ukmon.ini'), 'w') as outf: outf.write("# config data for this station\n") outf.write("export LOCATION={}\n".format(location)) @@ -85,13 +74,40 @@ def validateIni(homedir, newhelperip=None): if 'UKMONHELPER' in li: helperip = li.split('=')[1] if location is None or keyfile is None or rmscfg is None or helperip is None: - createDefaultIni(homedir, newhelperip, location, keyfile, rmscfg) + createDefaultIni(homedir, newhelperip, location, rmscfg) if helperip == oldip: updateHelperIp(homedir, newhelperip) updateMp4andMag(inifname, homedir) return True +def findLocationFromOldIni(stationid): + inif = os.path.expanduser('~/source/ukmon-pitools-{}/ukmon.ini'.format(stationid)) + location = 'NOTCONFIGURED' + if os.path.isfile(inif): + flis = open(inif, 'r').readlines() + loc = [x for x in flis if 'LOCA' in x] + location = loc[0].strip().split('=')[1] + return location + + +def relocateGitRepo(): + myloc = os.path.split(os.path.abspath(__file__))[0] + thisrepo = Repo(myloc) + origin = thisrepo.remote('origin') + if 'markmac99' in origin.url: + origin.rename('upstream') + remote.Remote.add(thisrepo, 'origin','https://github.com/ukmda/ukmon-pitools.git') + cfg = thisrepo.heads.main.config_writer() + cfg.set('remote','origin') + cfg.release() + cfg = thisrepo.heads.dev.config_writer() + cfg.set('remote','origin') + cfg.release() + print('git remote updated') + return + + def updateMp4andMag(inif, homedir): """ Move the mp4 flag into the ini file and add the maglim flag if missing @@ -109,36 +125,6 @@ def updateMp4andMag(inif, homedir): return -def updateHelperIp(homedir, helperip): - """ - Update the ukmon.ini file with a new IP address if neeeded. - """ - homedir = os.path.normpath(homedir) - lis = open(os.path.join(homedir, 'ukmon.ini'), 'r').readlines() - with open(os.path.join(homedir, 'ukmon.ini'), 'w') as outf: - for li in lis: - if 'UKMONHELPER' in li: - outf.write("export UKMONHELPER={}\n".format(helperip)) - else: - outf.write('{}'.format(li)) - return - - -def updateLocation(homedir, newloc): - """ - Update the ukmon-specific location, if a new one was supplied. Allows us to move cameras to new sites. - """ - homedir = os.path.normpath(homedir) - lis = open(os.path.join(homedir, 'ukmon.ini'), 'r').readlines() - with open(os.path.join(homedir, 'ukmon.ini'), 'w') as outf: - for li in lis: - if 'LOCATION' in li: - outf.write("export LOCATION={}\n".format(newloc)) - else: - outf.write('{}'.format(li)) - return - - def installUkmonFeed(rmscfg='~/source/RMS/.config'): """ Installs the UKMon postprocessing script into the RMS config file. @@ -254,19 +240,19 @@ def createSystemdService(myloc, camid): return -def createUbuntuIcon(myloc, statid): +def createUbuntuIcon(myloc): """ Create Ubuntu-compatible desktop icons. These different from the Debian-compatible ones normally used by RMS and which dont work properly on Ubuntu. """ - reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools_{}.sh'.format(statid)) + reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools.sh') if os.path.isfile(reflnk): os.remove(reflnk) - reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools_{}.desktop'.format(statid)) + reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools.desktop') with open(reflnk, 'w') as outf: outf.write('[Desktop Entry]\n') - outf.write('Name=refresh_UKMON_Tools_{}\n'.format(statid)) + outf.write('Name=refresh_UKMON_Tools\n') outf.write('Comment=Runs ukmon tools refresh\n') outf.write('Exec={}\n'.format(os.path.join(myloc, 'refreshTools.sh'))) outf.write('Icon=\n') @@ -285,79 +271,31 @@ def addDesktopIcons(myloc, statid): print('checking/adding desktop icons') if not os.path.isdir(os.path.expanduser('~/Desktop')): os.makedirs(os.path.expanduser('~/Desktop')) - cfglnk = os.path.expanduser('~/Desktop/UKMON_config_{}.txt'.format(statid)) + # the main and camera config files + cfglnk = os.path.expanduser('~/Desktop/UKMON_config.txt') if not os.path.islink(cfglnk): os.symlink(os.path.join(myloc, 'ukmon.ini'), cfglnk) + camlnk = os.path.expanduser('~/Desktop/UKMON_cameras.txt') + if not os.path.islink(camlnk): + os.symlink(os.path.join(myloc, 'cameras.ini'), camlnk) if isRaspberryPi(): - reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools_{}.sh'.format(statid)) + reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools.sh') if not os.path.islink(reflnk): os.symlink(os.path.join(myloc, 'refreshTools.sh'), reflnk) else: - createUbuntuIcon(myloc, statid) - # remove bad links if present - cfglnk = os.path.expanduser('~/Desktop/UKMON_config_XX0001.txt') - if os.path.islink(cfglnk): - os.unlink(cfglnk) - reflnk = os.path.expanduser('~/Desktop/refresh_UKMON_tools_XX0001.sh') - if os.path.islink(reflnk): - os.unlink(reflnk) + createUbuntuIcon(myloc) return -def getLatestKeys(homedir, remoteinifname='ukmon.ini'): - """ - Retrieve the latest ini and key files from the ukmon server. - If the ini file contains a new server IP or new location, the local copy of the - ini file is updated accordingly. - """ - homedir = os.path.expanduser(os.path.normpath(homedir)) - inifvals = readIniFile(os.path.join(homedir, 'ukmon.ini')) - ssh_client = paramiko.SSHClient() - ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - #try: - if True: - pkey = paramiko.RSAKey.from_private_key_file(os.path.expanduser(inifvals['UKMONKEY'])) - ssh_client.connect(inifvals['UKMONHELPER'], username=inifvals['LOCATION'], pkey=pkey, look_for_keys=False) - ftp_client = ssh_client.open_sftp() - - # get the aws key file - ftp_client.get('live.key', os.path.join(homedir, 'live.key')) - os.chmod(os.path.join(homedir, 'live.key'), 0o600) - - # get the new ini and check for changes - currinif = os.path.join(homedir, 'ukmon.ini') - newinif = os.path.join(homedir, '.ukmon.new') - ftp_client.put(currinif,'ukmon.ini.client') - ftp_client.get(remoteinifname, newinif) - ftp_client.close() - iniflines = open(newinif,'r').readlines() - for li in iniflines: - li = li.strip() - if 'UKMONHELPER' in li: - newhelper = li.split('=')[1] - if newhelper != inifvals['UKMONHELPER']: - updateHelperIp(homedir, newhelper) - print('server address updated') - if 'LOCATION' in li: - newloc = li.split('=')[1] - if newloc != inifvals['LOCATION']: - updateLocation(homedir, newloc) - print('location updated') - os.remove(newinif) - ssh_client.close() - return True - #except: - else: - return False - - def checkPlatepar(homedir, statid, rmsloc): """ Check for a new platepar on the server and retrieves it if present. The file is checked for compatability with the station. """ homedir = os.path.expanduser(os.path.normpath(homedir)) - inifvals = readIniFile(os.path.join(homedir, 'ukmon.ini')) + inifvals = readIniFile(os.path.join(homedir, 'ukmon.ini'), statid) + if not inifvals or inifvals['LOCATION']=='NOTCONFIGURED': + return ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: diff --git a/ukmonPostProc.py b/ukmonPostProc.py index 094d96f..6734e06 100644 --- a/ukmonPostProc.py +++ b/ukmonPostProc.py @@ -23,9 +23,12 @@ from uploadToArchive import uploadToArchive, readIniFile + log = logging.getLogger("ukmonlogger") log.setLevel(logging.INFO) +versionid = '2026.01.04' + def setupLogging(logpath, prefix): print('about to initialise logger') @@ -80,14 +83,14 @@ def rmsExternal(cap_dir, arch_dir, config): """ setupLogging(os.path.join(config.data_dir, config.log_dir), f'ukmon_log_{config.stationID}_') - log.info('ukmon external script started') + log.info('ukmon external script started, version ' + versionid) rebootlockfile = os.path.join(config.data_dir, config.reboot_lock_file) with open(rebootlockfile, 'w') as f: f.write('1') log.info('uploading key science files to archive') - keys = uploadToArchive(arch_dir, sciencefiles=True) + keys = uploadToArchive(arch_dir, config.stationID, sciencefiles=True) # create jpgs from the potential detections log.info('creating JPGs') try: @@ -96,7 +99,9 @@ def rmsExternal(cap_dir, arch_dir, config): bff2i.batchFFtoImage(arch_dir, 'jpg') myloc = os.path.split(os.path.abspath(__file__))[0] - inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), config.stationID) + if not inifvals or inifvals['LOCATION']=='NOTCONFIGURED': + return False log.info('app home is {}'.format(myloc)) domp4s = 0 if 'DOMP4S' in inifvals: @@ -123,7 +128,7 @@ def rmsExternal(cap_dir, arch_dir, config): log.info('mp4 creation not enabled') log.info('uploading remaining files to archive') - uploadToArchive(arch_dir, keys=keys) + uploadToArchive(arch_dir, config.stationID, keys=keys) # do not remove reboot lock file if running another script # os.remove(rebootlockfile) @@ -174,33 +179,29 @@ def manualRerun(dated_dir, rmscfg = '~/source/RMS/.config'): return rmsExternal(cap_dir, arch_dir, config) -def main(args): - if len(args) < 2: +if __name__ == '__main__': + if len(sys.argv) < 2: print('usage: python ukmonPostProc.py arc_dir_name') print('eg python ukmonPostProc.py UK0006_20210312_183741_206154') - print('\n nb: script must be run from RMS source folder') - return False + exit(0) - arch_dir = args[1] + arch_dir = sys.argv[1] + if 'ConfirmedFiles' in arch_dir or 'ArchivedFiles' in arch_dir or 'CapturedFiles' in arch_dir: + _, arch_dir = os.path.split(arch_dir) + stationid = arch_dir.split('_')[0] myloc = os.path.split(os.path.abspath(__file__))[0] - inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) - if inifvals is None: - print('unable to open ukmon ini file') - return 'unable to open ukmon ini file' + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) + if not inifvals or inifvals['LOCATION']=='NOTCONFIGURED': + print('ukmon ini file invalid - check LOCATION') + exit(1) try: rmscfg = inifvals['RMSCFG'] except Exception: rmscfg='~/source/RMS/.config' try: - if 'ConfirmedFiles' in arch_dir or 'ArchivedFiles' in arch_dir or 'CapturedFiles' in arch_dir: - _, arch_dir = os.path.split(arch_dir) print('RMS config read from {}'.format(rmscfg)) ret = manualRerun(arch_dir, rmscfg) - return ret + exit(0) except Exception: print('unable to call manualRerun') - return False - - -if __name__ == '__main__': - main(sys.argv) + exit(1) diff --git a/uploadToArchive.py b/uploadToArchive.py index 8da3bda..27cfb81 100644 --- a/uploadToArchive.py +++ b/uploadToArchive.py @@ -23,9 +23,62 @@ warnings.filterwarnings('ignore', category=CryptographyDeprecationWarning) import paramiko import tempfile +import configparser + from RMS.Formats.FTPdetectinfo import readFTPdetectinfo + log = logging.getLogger("ukmonlogger") +logging.getLogger("paramiko").setLevel(logging.WARNING) + + +def getLatestKeys(homedir, stationid, remoteinifname='ukmon.ini'): + """ + Retrieve the latest ini and key files from the ukmon server. + If the ini file contains a new server IP or new location, the local copy of the + ini file is updated accordingly. + """ + homedir = os.path.expanduser(os.path.normpath(homedir)) + inifvals = readIniFile(os.path.join(homedir, 'ukmon.ini'), stationid) + if not inifvals or inifvals['LOCATION']=='NOTCONFIGURED': + return False + if not os.path.isfile(os.path.expanduser(inifvals['UKMONKEY'])): + return False + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + pkey = paramiko.RSAKey.from_private_key_file(os.path.expanduser(inifvals['UKMONKEY'])) + try: + ssh_client.connect(inifvals['UKMONHELPER'], username=inifvals['LOCATION'], pkey=pkey, look_for_keys=False) + ftp_client = ssh_client.open_sftp() + except Exception: + return False + + # get the aws key file + ftp_client.get('live.key', os.path.join(homedir, 'live.key')) + os.chmod(os.path.join(homedir, 'live.key'), 0o600) + + # get the new ini and check for changes + currinif = os.path.join(homedir, 'ukmon.ini') + newinif = os.path.join(homedir, '.ukmon.new') + ftp_client.put(currinif,'ukmon.ini.client') + ftp_client.get(remoteinifname, newinif) + ftp_client.close() + iniflines = open(newinif,'r').readlines() + for li in iniflines: + li = li.strip() + if 'UKMONHELPER' in li: + newhelper = li.split('=')[1] + if newhelper != inifvals['UKMONHELPER']: + updateHelperIp(homedir, newhelper) + print('server address updated') + if 'LOCATION' in li: + newloc = li.split('=')[1] + if newloc != inifvals['LOCATION']: + updateLocation(homedir, newloc) + print('location updated') + os.remove(newinif) + ssh_client.close() + return True def readKeyFile(filename, inifvals): @@ -78,6 +131,36 @@ def readKeyFile(filename, inifvals): return vals +def updateHelperIp(homedir, helperip): + """ + Update the ukmon.ini file with a new IP address if neeeded. + """ + homedir = os.path.normpath(homedir) + lis = open(os.path.join(homedir, 'ukmon.ini'), 'r').readlines() + with open(os.path.join(homedir, 'ukmon.ini'), 'w') as outf: + for li in lis: + if 'UKMONHELPER' in li: + outf.write("export UKMONHELPER={}\n".format(helperip)) + else: + outf.write('{}'.format(li)) + return + + +def updateLocation(homedir, newloc): + """ + Update the ukmon-specific location, if a new one was supplied. Allows us to move cameras to new sites. + """ + homedir = os.path.normpath(homedir) + lis = open(os.path.join(homedir, 'ukmon.ini'), 'r').readlines() + with open(os.path.join(homedir, 'ukmon.ini'), 'w') as outf: + for li in lis: + if 'LOCATION' in li: + outf.write("export LOCATION={}\n".format(newloc)) + else: + outf.write('{}'.format(li)) + return + + def getAWSKey(inifvals): ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -106,18 +189,29 @@ def getAWSKey(inifvals): log.info(e, exc_info=True) ssh_client.close() if key: - log.info('retrieved key details') return key.strip(), sec.strip() else: return False, False -def readIniFile(filename): +def readIniFile(filename, stationid): + myloc = os.path.dirname(filename) + camerafile = os.path.join(myloc,'cameras.ini') + if not os.path.isfile(camerafile): + location = None + else: + stations = getListOfStations(myloc) + thiscam = [x for x in stations if stationid.lower() in x[0]] + if len(thiscam)==0: + log.error('camera {} not in cameras.ini, cannot continue'.format(stationid)) + print('missing camera file') + return None + location = thiscam[0][1] if not os.path.isfile(filename): log.error('{} missing, cannot continue'.format(filename)) - return False - with open(filename, 'r') as fin: - lis = fin.readlines() + print('missing file') + return None + lis = open(filename, 'r').readlines() vals = {} for li in lis: if li[0]=='#': @@ -127,6 +221,15 @@ def readIniFile(filename): data = valstr.split('=') val = data[1].strip().strip('"') vals[data[0]] = val + if location: + vals['LOCATION'] = location.strip() + if stationid: + if os.path.isfile(os.path.expanduser('~/.ssh/ukmon_' + stationid.upper())): + vals['UKMONKEY'] = '~/.ssh/ukmon_' + stationid.upper() + if os.path.isfile(os.path.expanduser('~/source/Stations/' + stationid + '/.config')): + vals['RMSCFG'] = os.path.expanduser('~/source/Stations/' + stationid + '/.config') + #if vals['LOCATION'] == 'NOTCONFIGURED': + # return None return vals @@ -284,12 +387,12 @@ def checkMags(dir_path, ftpfile_name, min_mag): return ff_names -def uploadToArchive(arch_dir, sciencefiles=False, keys=False): +def uploadToArchive(arch_dir, stationid, sciencefiles=False, keys=False): # Upload all relevant files from *arch_dir* to ukmon's S3 Archive myloc = os.path.split(os.path.abspath(__file__))[0] - inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) - if not inifvals: + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) + if inifvals['LOCATION']=='NOTCONFIGURED': return False if not keys: keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) @@ -373,7 +476,17 @@ def uploadToArchive(arch_dir, sciencefiles=False, keys=False): return keys -def manualUpload(targ_dir, sciencefiles=False): +def getListOfStations(srcdir): + camfile = os.path.join(srcdir, 'cameras.ini') + if not os.path.isfile(camfile): + return [(None,'')] + camcfg = configparser.ConfigParser() + camcfg.read(camfile) + return camcfg['cameras'].items() + + + +def manualUpload(targ_dir, stationid, sciencefiles=False): """ Manually send the target folder to ukmon archive. Args: @@ -385,14 +498,22 @@ def manualUpload(targ_dir, sciencefiles=False): If the argument is 'test' then a test file is uploaded and the status reported back. """ if targ_dir == 'test': - try: - myloc = os.path.split(os.path.abspath(__file__))[0] - inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini')) - if not inifvals: - return False + myloc = os.path.split(os.path.abspath(__file__))[0] + stations = getListOfStations(myloc) + for cam in stations: + stationid = cam[0] + inifvals = readIniFile(os.path.join(myloc, 'ukmon.ini'), stationid) + if inifvals['LOCATION']=='NOTCONFIGURED': + continue + if stationid is not None: + stationid = stationid.upper() + if not os.path.isfile(os.path.join(myloc, 'live.key')): + if not getLatestKeys(myloc, stationid): + print('unable to get key for', inifvals['LOCATION']) + continue keys = readKeyFile(os.path.join(myloc, 'live.key'), inifvals) if not keys: - return False + continue with open('/tmp/test.txt', 'w') as f: f.write('{}'.format(inifvals['LOCATION'])) @@ -401,13 +522,8 @@ def manualUpload(targ_dir, sciencefiles=False): conn = boto3.Session(aws_access_key_id=keys['AWS_ACCESS_KEY_ID'], aws_secret_access_key=keys['AWS_SECRET_ACCESS_KEY']) s3 = conn.resource('s3', region_name=reg) s3.meta.client.upload_file('/tmp/test.txt', target, 'tmp/{}.txt'.format(keys['CAMLOC'])) - #key = {'Objects': []} - #key['Objects'] = [{'Key': 'test.txt'}] - #s3.meta.client.delete_objects(Bucket=target, Delete=key) - print('test successful') - except Exception: - print('unable to upload to archive - check key information') - return False + stationid = '' if stationid is None else stationid + print('test successful for', inifvals['LOCATION'], stationid) try: os.remove('/tmp/test.txt') except Exception: @@ -418,14 +534,20 @@ def manualUpload(targ_dir, sciencefiles=False): if not os.path.isdir(arch_dir): print('folder {} not found'.format(arch_dir)) return False - return uploadToArchive(arch_dir, sciencefiles, keys=None) + return uploadToArchive(arch_dir, stationid, sciencefiles, keys=None) if __name__ == '__main__': if len(sys.argv) < 2: - print('usage: python uploadToArchive.py ~/RMS_data/ArchivedFiles/dated_dir') + print('usage: python uploadToArchive.py dated_dir ') + print(' eg: python uploadToArchive.py UK001L_20260104_171228_956526') + print('Optionally include the full path otherwise RMSs ArchivedFiles folder is assumed') exit(0) - targdir = sys.argv[1] - manualUpload(targdir, True) + targdir = os.path.normpath(os.path.expanduser(sys.argv[1])) if targdir != 'test': - manualUpload(targdir) + nightdir = os.path.split(targdir)[1] + stationid = nightdir.split('_')[0] + manualUpload(targdir, stationid, sciencefiles=True) + manualUpload(targdir, stationid, sciencefiles=False) + else: + manualUpload(targdir, None, sciencefiles=True)