Merge pull request #235 from galpHub/UpdateMaster

Update master

Author: galpHub

.cirrus.yml

@@ -0,0 +1,10 @@
+continous_integration_task:
+  container:
+    dockerfile: divi/DockerfileForCI
+  test_script:
+    - pushd divi/qa/rpc-tests && split --numeric-suffixes --lines=60 test_list.txt test_list && rm test_list.txt && popd
+    - pushd divi/src/GMock && autoreconf -fvi && popd
+    - pushd divi/ && { make distclean || true ; } && ./autogen.sh && ./configure --without-gui NO_QT=1 && popd
+    - pushd divi/ && make && make -j check && popd
+    - pushd divi/qa/rpc-tests && ./test_runner.py --test_suite_file test_list00 --exclude proxy_test.py,rpcbind_test.py --jobs=1 --cli_timeout=60.0 && popd
+    - pushd divi/qa/rpc-tests && ./test_runner.py --test_suite_file test_list01 --exclude proxy_test.py,rpcbind_test.py --jobs=1 --cli_timeout=60.0 && popd

.gitignore

@@ -6,3 +6,4 @@ doxygen/*
 developer_stuff
 documents/*
 doxygen/
+*/divi_regtest/*

DeploymentScripts/LastBuiltDevelopment

@@ -1,25 +0,0 @@
-b064ff156dad8c5e60e5b836bbe79ad0094faf4a
-e3f9a6592f18bded5b4355880707a8be9488f0ee
-f7e41254d2357b23184367a0f4d305feb9709702
-0e85c951d20c61298a295717ffed3795072435d2
-fb7c7142cbeddc3a07b2182a40cd2ccdfcafd851
-2d996d92af6f698c847daa2192876624afdc8108
-c486ba3ced3f9840bb5b8b7906d6f7c73032eddf
-13d3eb96e89e329e8f42d21cb9e3821af7266ad1
-44bc2d0f1560fd3b44c2e24ba338a058744e642b
-5b15ae43174cb2d884726ce7002557940298e01a
-1a9c10df4119a56a5038d472fad79b1788883f8a
-320f3c57f4ebf8485d6021db121631741c021eb6
-06db2f697a30bc960402faee048c1fb3549285cc
-0adb94a83698c9c4b787509ce2a3db91533221f3
-1b424747ceeb744362ec18b6dc6a1ca829921d73
-1b2613983a029fd7e332fdb57338050c71d338ab
-515ee8b9d0ed7c2dc246def30882c31ac913ee60
-119fe55587f5593bccac1f0573c8d22a4a9731bb
-987d866a1573a29c3bc0314a7f3b38489674b325
-d501f3dbb061306940b792454a5a6852d3e1428c
-5eb5b40d08214af2545408eea1e7158ddb1368dc
-8b10201f2f07e93ea4582e5b8c465beab764e6cb
-ee1328d04749b0ca93b1bd8be79ea50679568410
-23b4e8afb8771f51db9446b845f430545f78749a
-f5cc2c1f383531571858270ccc78578382fdc254

DeploymentScripts/README.txt

@@ -1,19 +1,19 @@
-# This tool was designed to be used in an Ubuntu 18.04 environment
-# It provides a quick way of building divi binaries without all the
-# maneuvering that's otherwise required. A few dependencies like the MacOSX10.9 sdk
-# as well as the raspberrypi-tools. These are downloaded as part of the tidy_build.sh
-# but require care as these tools have not been thoroughly tested against careless
-# usage.
-# First time usage should be './continuous_build.sh clean'
-# Later usages that wish to skip the download of the 3rd party dependencies
-# can ommit the call to clean
-# Note that these deployment scripts point to a specific remote repo and branch
-# namely the development branch being managed by @galpHub. These should be updated
-# whenever these change
+Release Process Procedure (For Ubuntu 20.04 OS base machine - e.g. Digital Ocean)
+* sudo apt-get update
+* sudo apt-get -y upgrade
+* sudo apt-get install -y ruby rename
+* git clone --depth=1 <development-repo> Divi
+* cp Divi/DeploymentScripts DeploymentScripts && cd DeploymentScripts
+* ./install_docker_20-04.sh
+* sudo chmod +x download_dependencies.sh continuous_build.sh tidy_build.sh build.sh
+* ./download_dependencies.sh
+If development binaries are desired:
+* rm LastBuiltDevelopment && nohup ./continuous_build.sh dev > build_process.log 2>&1 &
+If production binaries are desired:
+* rm LastBuiltDevelopment && nohup ./continuous_build.sh > build_process.log 2>&1 &
 
-## Pre-requisites to be run in ~/build_dependencies
-# curl -L https://github.com/phracker/MacOSX-SDKs/releases/download/10.13/MacOSX10.9.sdk.tar.xz | tar -xJf - &&
-# tar -cJf MacOSX10.9.sdk.tar.xz ./ &&
-# rm -r MacOSX10.9.sdk &&
-# git clone -q https://github.com/raspberrypi/tools.git && tar -czf raspberrypi-tools.tar.gz ./tools &&
-# rm -rf ./tools
+Additonal notes:
+* Should have at least 4 ram and 2 cpus of hardware for building
+* Built binaries are stored under /divi_binaries/
+* LastBuiltDevelopment is a list of recently built commit hashes. The build process ignores the last commit hash in this file.
+* It's important to run the build process in the background as it can slow down the reponsiveness of the shell on which it executes potentially cancelling the build in the event of disconnection

DeploymentScripts/build.sh

@@ -29,39 +29,31 @@ last_failed_build=""
 if ! $MayBuild; then
 	echo "Cannot build current configuration"
 fi
-all_builds_succeeded=false
+all_builds_succeeded=true
+builds=(linux win osx rpi2)
 if $MayBuild; then
 	echo "Working from commit: $1 -> $COMMIT & source: $2"
 	rm -rf /blddv/upload/ &&
-	mkdir -p /blddv/upload/ &&
-	mkdir -p /blddv/upload/ubuntu &&
-	mkdir -p /blddv/upload/windows &&
-	mkdir -p /blddv/upload/osx &&
-	mkdir -p /blddv/upload/rpi2
+	mkdir -p /blddv/upload/
 
 	cd /blddv/gitian-builder/ &&
 	mkdir -p /blddv/gitian-builder/inputs/ &&
 	cp /blddv/build_dependencies/* /blddv/gitian-builder/inputs/
 
 	[ -d "/blddv/gitian-builder/inputs/divi/" ] && rm -rf /blddv/gitian-builder/inputs/divi/
 
-	all_builds_succeeded=false &&
 	cd /blddv/gitian-builder/ &&
-	/blddv/gitian-builder/bin/make-base-vm --docker --suite trusty && wait &&
-	last_failed_build="ubuntu" &&
-	/blddv/gitian-builder/bin/gbuild --commit divi=$COMMIT --url divi=/blddv/Divi/ /blddv/Divi/divi/contrib/gitian-descriptors/gitian-linux.yml &&
-	cp -r /blddv/gitian-builder/build/out/* /blddv/upload/ubuntu/ &&
-	last_failed_build="windows" &&
-	/blddv/gitian-builder/bin/gbuild --commit divi=$COMMIT --url divi=/blddv/Divi/ /blddv/Divi/divi/contrib/gitian-descriptors/gitian-win.yml &&
-	cp -r /blddv/gitian-builder/build/out/* /blddv/upload/windows/ &&
-	last_failed_build="mac" &&
-	/blddv/gitian-builder/bin/gbuild --commit divi=$COMMIT --url divi=/blddv/Divi/ /blddv/Divi/divi/contrib/gitian-descriptors/gitian-osx.yml &&
-	cp -r /blddv/gitian-builder/build/out/* /blddv/upload/osx/ &&
-	last_failed_build="rpi" &&
-	/blddv/gitian-builder/bin/make-base-vm --docker --suite trusty && wait &&
-	/blddv/gitian-builder/bin/gbuild --commit divi=$COMMIT --url divi=/blddv/Divi/ /blddv/Divi/divi/contrib/gitian-descriptors/gitian-rpi2.yml &&
-	cp -r /blddv/gitian-builder/build/out/* /blddv/upload/rpi2/ &&
-	all_builds_succeeded=true
+	for build_id in ${builds[@]}
+	do
+		if [ "${last_failed_build}" = "${build_id}" ]; then all_builds_succeeded=false && break ; fi
+		last_failed_build="${build_id}" &&
+		/blddv/gitian-builder/bin/make-base-vm --docker --suite bionic && wait &&
+		/blddv/gitian-builder/bin/gbuild -j 1 --commit divi=$COMMIT --url divi=/blddv/Divi/ /blddv/Divi/divi/contrib/gitian-descriptors/gitian-${build_id}.yml &&
+		mkdir -p /blddv/upload/${build_id} &&
+		cp -r /blddv/gitian-builder/build/out/* /blddv/upload/${build_id}/ &&
+		last_failed_build="" &&
+		 docker container stop $( docker ps -a -q )
+	done
 fi
 
 mkdir -p /divi_binaries/

DeploymentScripts/continuous_build.sh

@@ -7,8 +7,20 @@ oldHash=$(tac LastBuiltDevelopment | head -n 1)
 remoteDevelopment="https://github.com/galpHub/Divi.git"
 remoteMaster="https://github.com/DiviProject/Divi.git"
 
-branch_name="Development"
-remoteRepo=$remoteDevelopment
+clean_mode=false
+if [[ -z $1 ]] ; then
+   branch_name="master"
+   remoteRepo=$remoteMaster
+   echo "Building master..."
+elif [[ ! -z $1 ]] && [[ $1 == "dev" ]] ; then
+   branch_name="Development"
+   remoteRepo=$remoteDevelopment   
+   echo "Building Development..."
+fi
+
+if [[ ! -z $2   ]] ; then
+    clean_mode=true
+fi
 
 git clone --depth=1 $remoteRepo --branch $branch_name /tmp/Divi/
 pushd /tmp/Divi/
@@ -17,7 +29,11 @@ popd
 rm -r /tmp/Divi/
 zero=0
 if [[ $oldHash != $newestHash ]]; then
-	./tidy_build.sh $branch_name $remoteRepo $1
+	if $clean_mode ; then
+		./tidy_build.sh $branch_name $remoteRepo clean
+	else
+		./tidy_build.sh $branch_name $remoteRepo
+	fi
 	if [[ $? ]]; then
 		if [[ -d "/divi_binaries/$newestHash" ]]; then
 			echo "Build complete!"
@@ -37,4 +53,4 @@ else
 fi
 
 }
-run $1
+run $1 $2

DeploymentScripts/derp_build.sh

@@ -9,7 +9,7 @@ pushd MacOSX10.9.sdk &&
 find . -type f ! -iname "sha256checksums" -print0 | sort -z | xargs -r0 sha256sum > sha256checksums &&
 echo "5c4e5b294f196c7f88f755c35e8ecc688f80da5a33f00d8078ed4c6f9ff4d3d8  sha256checksums" | sha256sum -c  &&
 rm sha256checksums && popd &&
-tar -cJf MacOSX10.9.sdk.tar.xz ./ &&
+{ tar -cJf MacOSX10.9.sdk.tar.xz ./ || true ; } &&
 rm -r MacOSX10.9.sdk &&
 git clone -q https://github.com/raspberrypi/tools.git &&
 tar -czf raspberrypi-tools.tar.gz ./tools &&
@@ -22,4 +22,4 @@ git config --global user.name "DevBot" &&
 git commit -m "Remove proxy"
 }
 
-run
+run

DeploymentScripts/download_dependencies.sh

@@ -0,0 +1,8 @@
+apt-get update && 
+apt-get -y install sudo &&
+sudo apt install -y apt-transport-https ca-certificates curl software-properties-common &&
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - &&
+sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" &&
+apt-cache policy docker-ce &&
+apt-cache search docker-ce &&
+sudo apt install -y docker-ce