Merge branch 'release'

Author: Luis Fabregas

.github/workflows/ci.yml

@@ -0,0 +1,76 @@
+name: Continuous Integration
+
+on: 
+   push:
+      branches:
+      - master
+      - release
+
+jobs:
+  build:
+    name: CI
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout trigger commit
+      uses: actions/checkout@v1
+    - name: Install dependencies
+      run: |
+           sudo apt-get install -y openvpn openconnect
+    - name: Prepare VM
+      if: success()
+      run: |
+           rm -f -r ./.git
+           echo "${{ secrets.GATEWAY_KEY }}" > ~/sshkey
+           sudo chmod 600 ~/sshkey
+    - name: Establish VPN connection
+      if: success()
+      run: |
+           # Force returning packets to be routed over public interface and not over the VPN
+           sudo ip rule add from $(ip route get 1 | grep -Po '(?<=src )(\S+)') table 128
+           sudo ip route add table 128 to $(ip route get 1 | grep -Po '(?<=src )(\S+)')/32 dev $(ip -4 route ls | grep default | grep -Po '(?<=dev )(\S+)')
+           sudo ip route add table 128 default via $(ip -4 route ls | grep default | grep -Po '(?<=via )(\S+)')
+           # Start network tunnel (TUN) device
+           sudo openvpn --mktun --dev tun1
+           sudo ifconfig tun1 up
+           # Launch VPN connection
+           echo "${{ secrets.VPN_PASSWD }}" | sudo openconnect ${{ secrets.VPN_SERVER }} --user=${{ secrets.VPN_USER }} --background --passwd-on-stdin
+    - name: Transfer source files to CI-server
+      if: success()
+      run: |
+           scp -o StrictHostKeyChecking=no -i ~/sshkey -P ${{ secrets.GATEWAY_PORT }} -r ../DeerAnalysis2 ${{ secrets.GATEWAY_HOST }}:/home/lufa/scp_traffic/
+           ssh -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }} scp -r /home/lufa/scp_traffic/DeerAnalysis2 ${{ secrets.CISERVER_HOST }}:/home/deeranalysis/
+           ssh -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }} "rm -f -r /home/lufa/scp_traffic/DeerAnalysis2"
+    - name: Run test suite connected to CI-Server
+      if: success()
+      run: |
+           ssh -tt -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} -t ${{ secrets.GATEWAY_HOST }} ssh ${{ secrets.CISERVER_HOST }} << EOF
+               #Run test suite
+               cd DeerAnalysis2/build
+               matlab -nodesktop -r "datestsuite;exit"
+               python3 uploadS3.py --keyfile ~/.ssh/aws_access_keys.txt --file "coverage_badge.json" --bucket deershields
+               python3 uploadS3.py --keyfile ~/.ssh/aws_access_keys.txt --file "testsuite_badge.json" --bucket deershields
+               #Transfer log file to the gateway
+               scp -o StrictHostKeyChecking=no -i ~/.ssh/gateway_key -P ${{ secrets.GATEWAY_PORT }} ./datestsuite.error ${{ secrets.GATEWAY_HOST }}:/home/lufa/scp_traffic/ || true
+               #Delete the source files
+               cd /home/deeranalysis/
+               rm -f -r DeerAnalysis2
+           EOF
+    - name: Disconnect CI-server
+      if: success()
+      run: |
+           #Transfer log file to GitHub VM
+           scp -o StrictHostKeyChecking=no -i ~/sshkey -P ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }}:/home/lufa/scp_traffic/datestsuite.error ~/ || true
+           ssh -o StrictHostKeyChecking=no -i ~/sshkey -p ${{ secrets.GATEWAY_PORT }} ${{ secrets.GATEWAY_HOST }} "rm -f /home/lufa/scp_traffic/datestsuite.error" || true
+           rm -f ~/sshkey
+    - name: All tests passed successfully
+      if: success()
+      run: |
+           ! [ -f ~/datestsuite.error ] && echo "Test suite passed" || (echo "Test suite failed" && deerfailed)
+    - name: Disconnect VPN connection
+      if: always()
+      run: |
+           #Ensure the VPN is disconnected by killing openconnect
+           sudo pkill openconnect
+           #Remove TUN device
+           sudo ifconfig tun1 down
+           sudo openvpn --rmtun --dev tun1

.github/workflows/docbuilder.yml

@@ -36,5 +36,5 @@ jobs:
            sphinx-build -E -b html ./source ../docs
     - name: Connect and update DeerAnalysis AWS S3 Bucket
       run: |
-           cd docsrc
-           python -m websynch -k ${{ secrets.S3_ACCESS }} -s ${{ secrets.S3_SECRETACCESS }}
+           cd build
+           python -m websynch -k ${{ secrets.S3_ACCESS }} -s ${{ secrets.S3_SECRETACCESS }} -d "../docs" --bucket deeranalysis.org

.gitignore

@@ -3,9 +3,11 @@ cvx/
 docs/docsrc/build/
 docs/docsrc/venv/
 docsrc/venv/
+docsrc/source/.doctrees
 docs/
 docsrc/cache/
 docsrc/source/__pycache__/
 docsrc/aws_access_keys.txt
 docsrc/__pycache__/
 build/.ssh/
+tutorials/**/*.pdf

README.md

@@ -9,6 +9,8 @@
  <img src="https://img.shields.io/badge/MATLAB-R2016b--R2019b-brightgreen?style=flat"></img>
  <img src="https://img.shields.io/github/downloads/luisfabib/DeerAnalysis2/total?style=flat"></img>
  <img src="https://github.com/luisfabib/DeerAnalysis2/workflows/Webpage%20update/badge.svg?style=flat-square"></img>
+  <img src="https://img.shields.io/endpoint?url=https%3A%2F%2Fdeershields.s3.eu-central-1.amazonaws.com%2Fcoverage_badge.json"></img>
+  <img src="https://img.shields.io/endpoint?url=https%3A%2F%2Fdeershields.s3.eu-central-1.amazonaws.com%2Ftestsuite_badge.json"></img>
 </p>
 
 ### About

build/CI_testing.m

@@ -0,0 +1,54 @@
+
+clc
+
+files = dir('../tutorials/**/*.mlx');
+sourcenames = {files.name};
+pdfnames = cellfun(@(S)[S(1:end-3) 'pdf'],sourcenames,'UniformOutput',false);
+htmlnames = cellfun(@(S)[S(1:end-3) 'html'],sourcenames,'UniformOutput',false);
+mnames = cellfun(@(S)[S(1:end-3) 'm'],sourcenames,'UniformOutput',false);
+
+paths = {files.folder};
+sourcefiles = cellfun(@(x,y)fullfile(x,y),paths,sourcenames,'UniformOutput',false);
+pdffiles = cellfun(@(x,y)fullfile(x,y),paths,pdfnames,'UniformOutput',false);
+htmlfiles = cellfun(@(x,y)fullfile(x,y),paths,htmlnames,'UniformOutput',false);
+mfiles = cellfun(@(x,y)fullfile(x,y),paths,mnames,'UniformOutput',false);
+
+%Convert live script into other formats
+for i=1:numel(pdfnames)
+    fprintf('Conversion: mlx -> pdf (file %i of %i)...',i,numel(sourcefiles))
+    matlab.internal.liveeditor.openAndConvert(sourcefiles{i},pdffiles{i});
+    fprintf(' complete \n')
+end
+
+for i=1:numel(htmlnames)
+    fprintf('Conversion: mlx -> html (file %i of %i)...',i,numel(sourcefiles))
+    matlab.internal.liveeditor.openAndConvert(sourcefiles{i},htmlfiles{i});
+    fprintf(' complete \n')
+end
+
+for i=1:numel(mnames)
+    fprintf('Conversion: mlx -> m (file %i of %i)...',i,numel(sourcefiles))
+    matlab.internal.liveeditor.openAndConvert(sourcefiles{i},mfiles{i});
+    fprintf(' complete \n')
+end
+
+
+if ispc
+    
+    %Synchronize the AWS bucket
+    system('python synchS3.py --keyfile %USERPROFILE%\.ssh\aws_access_keys.txt --directory "../tutorials" --bucket deertutorials');
+    
+    %Delete files
+    for i=1:numel(pdfnames)
+        fprintf('Deleting: pdf (file %i of %i)...',i,numel(sourcefiles))
+        delete(pdffiles{i});
+        fprintf(' complete \n')
+    end
+    
+    for i=1:numel(mnames)
+        fprintf('Deleting: m (file %i of %i)...',i,numel(sourcefiles))
+        delete(mfiles{i});
+        fprintf(' complete \n')
+    end
+    
+end

build/build_ssh_euler.bat

@@ -0,0 +1,29 @@
+%Add paths to dependencies
+addpath('../functions')
+addpath('../functions/models')
+addpath('../tests')
+addpath('../tests/comparison')
+
+%Run full test suite 
+Out = datest('--time','--perform','--coverage','--badges');
+
+if Out.Errors>0
+    logname = 'datestsuite.error';
+    save(logname)
+end
+
+%Load the badges JSON endpoints to AWS
+if ispc
+    system('python uploadS3.py --keyfile %USERPROFILE%\.ssh\aws_access_keys.txt --file "coverage_badge.json" --bucket deershields')
+    system('python uploadS3.py --keyfile %USERPROFILE%\.ssh\aws_access_keys.txt --file "testsuite_badge.json" --bucket deershields')
+
+    %Delete files once uploaded
+    delete('coverage_badge.json')
+    delete('testsuite_badge.json')
+
+elseif isunix
+    %system('python3 uploadS3.py --keyfile ~/.ssh/aws_access_keys.txt --file "coverage_badge.json" --bucket deershields')
+    %system('python3 uploadS3.py --keyfile ~/.ssh/aws_access_keys.txt --file "testsuite_badge.json" --bucket deershields')
+elseif ismac
+    error('macOS is not supported')
+end

build/compiletutorials.m

@@ -13,6 +13,9 @@
 parser = ArgumentParser()
 parser.add_argument("-k", "--key",dest="accesskey",default=[],help="AWS Access Key")
 parser.add_argument("-s", "--secretkey",dest="secretkey", default=[],help="AWS Secret Access Key")
+parser.add_argument("-d", "--directory",dest="base_dir",default=[],help="Directory to upload")
+parser.add_argument("-b", "--bucket",dest="bucketname", default=[],help="Name of S3 bucket")
+parser.add_argument("-i", "--keyfile",dest="keyfile",default=[],help="AWS Access Key")
 args = parser.parse_args()
 
 def getListOfFiles(dirName):
@@ -55,13 +58,21 @@ def metadataType(file):
 
 #If access codes have not been passed as inputs then check if file is there
 if not args.accesskey or not args.secretkey:
+	
+	if not args.keyfile:
+		keypath = 'aws_access_keys.txt'
+	else:
+		keypath = args.keyfile
+	
+	#Set absolute path to key file
+	#cwd = os.getcwd()
+	#keypath =  os.path.join(cwd,keypath)
 
 	#Load and read the secure access keys to connect to the AWS S3 client
-	if os.path.isfile(".\aws_access_keys.txt"):
+	if not os.path.isfile(keypath):
 		print("AWS access keys not found. You may not have rights to request this action.")
 		exit()
-
-	with open('aws_access_keys.txt', 'r') as file:
+	with open(keypath, 'r') as file:
 		AccessKeys = [line.rstrip('\n') for line in file]
 else:
 	 AccessKeys = [args.accesskey, args.secretkey]
@@ -78,24 +89,27 @@ def metadataType(file):
 		 aws_access_key_id = AccessKeys[0],
 		 aws_secret_access_key = AccessKeys[1],
 		 region_name='eu-west-1')
-bucket = s3.Bucket('deeranalysis.org')
 
-base_dir = "../docs"
+bucketname = str(args.bucketname)
+bucket = s3.Bucket(bucketname)
+
+base_dir = str(args.base_dir)
 #Get full list of local files
 localFiles = getListOfFiles(base_dir)
+base_dir +=  '/'
 
 for key in bucket.objects.all():
 
 		#Get full local path of current file in bucket 
 		file = '/'.join([base_dir, key.key])
-		
+
 		#Remove bucket file from list of local files
 		if file in localFiles: localFiles.remove(file)
 
 		#If file has been removed from the local source, remove it from the bucket
 		if not os.path.isfile(file):
 			print("Removing ",file," from bucket, not found in local source...")
-			s3_client.delete_object(Bucket='deeranalysis.org',Key = key.key)
+			s3_client.delete_object(Bucket=bucketname,Key = key.key)
 			continue
 
 		#Get last modified date of local files
@@ -107,13 +121,14 @@ def metadataType(file):
 
 		#Update the file if the local source file is newer than the version in the S3 bucket
 		if modifyDate > key.last_modified:
-			print("Updating", file, "in web bucket... ")
-			s3.meta.client.upload_file(file, 'deeranalysis.org', key.key, ExtraArgs={'ContentType': metadataType(file)} )
+			print("Updating", file, "in ",bucketname," bucket... ")
+			s3.meta.client.upload_file(file, bucketname, key.key, ExtraArgs={'ContentType': metadataType(file)} )
 
 #Add the remaining local files which are still not on the we bucket
 for files in localFiles:
-	key = files.replace("../docs/","")
-	print("Adding", key, "to web bucket... ")
-	s3.meta.client.upload_file(file, 'deeranalysis.org', key, ExtraArgs={'ContentType': metadataType(file)} )
+	file = files;
+	key = files.replace(base_dir,"")
+	print("Adding", key, "to",bucketname,"bucket... ")
+	s3.meta.client.upload_file(file, bucketname, key, ExtraArgs={'ContentType': metadataType(str(key))} )
 
-print("Finished: AWS S3 DeerAnalysis.org bucket is up to date.")
+print("Finished: AWS S3 ",bucketname," bucket is up to date.")