From a083bbb3451fdbb464364fc872b97259911d12e3 Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Fri, 9 Nov 2018 21:07:07 +0100 Subject: [PATCH 01/14] vault integration changes --- .travis.yml | 62 +++++++++---- README.md | 31 ++++++- Vagrantfile | 54 ++++++----- nomad/__init__.py | 19 +++- nomad/api/base.py | 9 +- start_daemons.sh | 159 ++++++++++++++++++++++++++++++++ stop_daemons.sh | 15 +++ tests/common.py | 23 +++++ tests/conftest.py | 10 ++ tests/test_acl.py | 3 + tests/test_vault_integration.py | 95 +++++++++++++++++++ vault.hcl | 38 ++++++++ vault.json | 121 ++++++++++++++++++++++++ 13 files changed, 593 insertions(+), 46 deletions(-) create mode 100755 start_daemons.sh create mode 100755 stop_daemons.sh create mode 100644 tests/test_vault_integration.py create mode 100755 vault.hcl create mode 100755 vault.json diff --git a/.travis.yml b/.travis.yml index 7d1b295..384effc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,27 +22,55 @@ env: global: - NOMAD_IP="127.0.0.1" - NOMAD_PORT="4646" + - NOMAD_INTEGRATION_VAULT="0.6.2" + - VAULT_ADDR="http://127.0.0.1:8200" + - VAULT_TEST="true" matrix: - - NOMAD_VERSION="0.3.2" - - NOMAD_VERSION="0.4.1" - - NOMAD_VERSION="0.5.6" - - NOMAD_VERSION="0.6.0" - - NOMAD_VERSION="0.7.1" - - NOMAD_VERSION="0.8.1" - - NOMAD_VERSION="0.8.3" -before_install: -- curl -L -o /tmp/nomad_${NOMAD_VERSION}_linux_amd64.zip https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip -- yes | unzip -d /tmp /tmp/nomad_${NOMAD_VERSION}_linux_amd64.zip -- MAJOR_VERSION=`echo ${NOMAD_VERSION} | cut -d "." -f 2` -- if [[ ${MAJOR_VERSION} -gt 6 ]]; then echo "Nomad version $NOMAD_VERSION supports acls";export ACL_ENABLED="--acl-enabled"; else echo "Nomad version $NOMAD_VERSION";export ACL_ENABLED=""; fi -- /tmp/nomad agent -dev -bind ${NOMAD_IP} -node pynomad1 ${ACL_ENABLED} > /dev/null 2>&1 & -- sleep 30 + - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.6.2" + - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.6.2" + - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.6.2" + - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.6.2" + - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.6.2" + - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.6.2" + - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.6.2" + - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.6.2" + - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.6.2" + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.6.2" + - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.7.0" + - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.7.0" + - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.7.0" + - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.7.0" + - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.7.0" + - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.7.0" + - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.7.0" + - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.7.0" + - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.7.0" + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.7.0" + - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.8.0" + - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.8.0" + - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.8.0" + - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.8.0" + - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.8.0" + - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.8.0" + - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.8.0" + - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.8.0" + - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.8.0" + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.8.0" + - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.9.0" + - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.9.0" + - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.9.0" + - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.9.0" + - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.9.0" + - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.9.0" + - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.9.0" + - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.9.0" + - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.9.0" + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.9.0" install: -- pip install -r requirements.txt -r requirements-dev.txt +- pip install -r requirements-dev.txt - pip install codecov before_script: - - /tmp/nomad init - - /tmp/nomad run -output example.nomad > example.json + - sudo ./start_daemons.sh script: - py.test --cov=nomad --cov-report=term-missing --runxfail tests/ after_success: diff --git a/README.md b/README.md index 91497aa..93e730c 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,23 @@ NOMAD_TOKEN=xxxx-xxxx-xxxx-xxxx NOMAD_REGION=us-east-1a ``` +## With Vault integration. +if you have configured a [Vault Integration ](https://www.nomadproject.io/docs/configuration/vault.html) to store your secrets. + +And you have configured: [`allow_unantenticated = false`](https://www.nomadproject.io/docs/configuration/vault.html#allow_unauthenticated) +see you must to export and send a valid `VAULT_TOKEN`. + +```bash +VAULT_TOKEN=xxxx-xxxx-xxxx-xxxx +``` + +## Skipt vault tests +if you want to test vault integration please export the variable: + +```bash +VAULT_TEST=true +``` + ## Class Dunders | Class | contains | len | getitem | iter | @@ -98,13 +115,23 @@ pip install -r requirements-dev.txt ## Testing with vagrant and virtualbox ``` vagrant up --provider virtualbox +source /tmp/environment.vars.sh py.test --cov=nomad --cov-report=term-missing --runxfail tests/ ``` ## Testing with nomad binary ``` -./nomad agent -dev -node pynomad1 --acl-enabled -NOMAD_IP=127.0.0.1 NOMAD_VERSION= py.test --cov=nomad --cov-report=term-missing --runxfail tests/ +export NOMAD_IP=127.0.0.1 +export NOMAD_VERSION= +export VAULT_VERSION= # should be higher than 0.6.2 +export VAULT_TEST=true # if you select no, vault integration will not be tested +./start_daemons.sh +py.test --cov=nomad --cov-report=term-missing --runxfail tests/ +``` + +after make your tests, you can stop the necesary daemons with +``` +./stop_daemons.sh ``` - Examples diff --git a/Vagrantfile b/Vagrantfile index fafbb8c..0715caa 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,18 +1,26 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -IP = "192.168.33.10" -NOMAD_VERSION = "0.8.3" -NOMAD_PORT_GUEST = 4646 -NOMAD_PORT_HOST = 4646 +NOMAD_IP="192.168.33.10" +NOMAD_VERSION="0.8.6" +NOMAD_PORT_GUEST=4646 +NOMAD_PORT_HOST=4646 +VAULT_VERSION="0.9.0" +VAULT_PORT_GUEST=8200 +VAULT_PORT_HOST=8200 +VAULT_ADDR="http://127.0.0.1:8200" +VAULT_TEST="true" +NOMAD_INTEGRATION_VAULT="0.6.2" + Vagrant.configure(2) do |config| config.vm.box = "centos/7" config.vm.network "forwarded_port", guest: NOMAD_PORT_GUEST, host: NOMAD_PORT_HOST +config.vm.network "forwarded_port", guest: VAULT_PORT_GUEST, host: VAULT_PORT_HOST -config.vm.network "private_network", ip: "#{IP}" +config.vm.network "private_network", ip: "#{NOMAD_IP}" config.vm.provider "virtualbox" do |vb| vb.name = "python-nomad" @@ -22,8 +30,7 @@ end config.vm.provision "shell", inline: <<-SHELL -if [ ! -e /etc/yum.repos.d/docker.repo ] - then +if [ ! -e /etc/yum.repos.d/docker.repo ]; then tee /etc/yum.repos.d/docker.repo <<-EOF [dockerrepo] name=Docker Repository @@ -38,22 +45,27 @@ yum -y install docker-engine unzip wget net-tools usermod -aG docker vagrant systemctl enable docker; systemctl start docker -wget -q -P /tmp/ https://releases.hashicorp.com/nomad/#{NOMAD_VERSION}/nomad_#{NOMAD_VERSION}_linux_amd64.zip -yes | unzip -d /tmp /tmp/nomad_#{NOMAD_VERSION}_linux_amd64.zip +echo "pip for test inside the vagrant" +curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py" +python get-pip.py +pip install -r /vagrant/requirements-dev.txt -if [ ! -f /usr/bin/nomad ] - then - cp /tmp/nomad /usr/bin/. -fi +cat << EOF > /tmp/environment.vars.sh +export NOMAD_IP="#{NOMAD_IP}" +export NOMAD_VERSION="#{NOMAD_VERSION}" +export NOMAD_PORT_GUEST="#{NOMAD_PORT_GUEST}" +export NOMAD_PORT_HOST="#{NOMAD_PORT_HOST}" +export VAULT_VERSION="#{VAULT_VERSION}" +export VAULT_ADDR="#{VAULT_ADDR}" +export VAULT_TEST="#{VAULT_TEST}" +export NOMAD_INTEGRATION_VAULT="#{NOMAD_INTEGRATION_VAULT}" +EOF +chmod +x /tmp/environment.vars.sh +source /tmp/environment.vars.sh +cd /vagrant +./start_daemons.sh -if [ $(pgrep nomad) ] - then - echo "Nomad running" - else - echo "Starting Nomad" - nohup nomad agent -dev -bind #{IP} -node pynomad1 --acl-enabled > /dev/null 2>&1 & - sleep 30 -fi +py.test -s -v --cov=nomad --cov-report=term-missing --runxfail tests/ SHELL diff --git a/nomad/__init__.py b/nomad/__init__.py index 0eb04bf..0ef9834 100644 --- a/nomad/__init__.py +++ b/nomad/__init__.py @@ -10,11 +10,12 @@ def __init__(self, port=4646, address=os.getenv('NOMAD_ADDR', None), namespace=os.getenv('NOMAD_NAMESPACE', None), - token=os.getenv('NOMAD_TOKEN', None), - timeout=5, - region=os.getenv('NOMAD_REGION', None), - version='v1', - verify=False, + token=os.getenv('NOMAD_TOKEN', None), + vaulttoken=os.getenv('VAULT_TOKEN', None), + timeout=5, + region=os.getenv('NOMAD_REGION', None), + version='v1', + verify=False, cert=()): """ Nomad api client @@ -35,6 +36,9 @@ def __init__(self, be use to deploy or to ask info to nomad. - token (defaults to None), Specifies to append ACL token to the headers to make authentication on secured based nomad environemnts. + - vaulttoken (defaults to None), Specifies to append ACL token to the job and + make authentication on environemnts with allow_unantenticated = false, where + you must to send a valid vault token for policies. returns: Nomad api client object raises: @@ -50,6 +54,7 @@ def __init__(self, self.timeout = timeout self.version = version self.token = token + self.vaulttoken = vaulttoken self.verify = verify self.cert = cert self.__namespace = namespace @@ -60,6 +65,7 @@ def __init__(self, "port": self.port, "namespace": self.__namespace, "token": self.token, + "vaulttoken": self.vaulttoken, "timeout": self.timeout, "version": self.version, "verify": self.verify, @@ -103,6 +109,9 @@ def get_namespace(self): def get_token(self): return self.token + def get_vaulttoken(self): + return self.vaulttoken + @property def jobs(self): return self._jobs diff --git a/nomad/api/base.py b/nomad/api/base.py index cca3a09..5b88a8d 100644 --- a/nomad/api/base.py +++ b/nomad/api/base.py @@ -10,11 +10,12 @@ class Requester(object): ENDPOINT = "" - def __init__(self, address=None, uri='http://127.0.0.1', port=4646, namespace=None, token=None, timeout=5, version='v1', verify=False, cert=(), region=None, **kwargs): + def __init__(self, address=None, uri='http://127.0.0.1', port=4646, namespace=None, token=None, vaulttoken=None, timeout=5, version='v1', verify=False, cert=(), region=None, **kwargs): self.uri = uri self.port = port self.namespace = namespace self.token = token + self.vaulttoken = vaulttoken self.timeout = timeout self.version = version self.verify = verify @@ -99,6 +100,12 @@ def _request(self, method, endpoint, params=None, data=None, json=None, headers= except TypeError: headers = {"X-Nomad-Token": self.token} + if method == "post": + if json: + if self.vaulttoken: + if "Job" in json: + json["Job"]["VaultToken"] = self.vaulttoken + response = None try: diff --git a/start_daemons.sh b/start_daemons.sh new file mode 100755 index 0000000..8a43d60 --- /dev/null +++ b/start_daemons.sh @@ -0,0 +1,159 @@ +echo "Start daemons to test" + +if [ -z "${NOMAD_VERSION}" ]; then + echo "you should export NOMAD_VERSION" + exit 1 +fi + +if [ -z "${NOMAD_INTEGRATION_VAULT}" ]; then + NOMAD_INTEGRATION_VAULT="0.6.2" +fi + +if [ -z "${NOMAD_PORT_GUEST}" ]; then + NOMAD_PORT_GUEST="4646" +fi + +if [ -z "${NOMAD_IP}" ]; then + NOMAD_IP=127.0.0.1 +fi + +if [ -z "${VAULT_VERSION}" ]; then + VAULT_VERSION="0.6.2" +fi + +if [ ! -f /tmp/nomad ]; then + rm -rf /tmp/nomad +fi +echo "NOMAD: Get Binary Files" +wget -q -P /tmp/ https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip +yes | unzip -o -d /tmp /tmp/nomad_${NOMAD_VERSION}_linux_amd64.zip + + +if [ ! -f /tmp/vault ]; then + rm -rf /tmp/vault +fi +echo "VAULT: Get Binary Files" +wget -q -P /tmp/ https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip +yes | unzip -o -d /tmp /tmp/vault_${VAULT_VERSION}_linux_amd64.zip + + +VAULT_ADDR="http://localhost:8200" + +MAJOR_VERSION=`echo ${NOMAD_VERSION} | cut -d "." -f 2` +MAJOR_VERSION_VAULT_INTEGRATION=`echo ${NOMAD_VERSION} | tr -d "."|sed "s/^0*//"` +NOMAD_REQUIRED_TO_INEGRATE_WITH_VAULT=`echo ${NOMAD_INTEGRATION_VAULT}|tr -d "."|sed "s/^0*//"` + +echo "Nomad: Create config folder" +rm -rf /tmp/nomad.d +mkdir -p /tmp/nomad.d + +if [ "${VAULT_TEST}" = "true" ]; then +if [ ${MAJOR_VERSION_VAULT_INTEGRATION} -gt ${NOMAD_REQUIRED_TO_INEGRATE_WITH_VAULT} ]; then + echo "Vault: Create policy file" +cat << EOF > /tmp/policy-demo.hcl +path "secret/demo" { + capabilities = ["read"] +} +EOF + + echo "Vault: Start Daemon Version: ${VAULT_VERSION}" + /tmp/vault server -dev -dev-listen-address=0.0.0.0:8200 -dev-root-token-id="root" > /dev/null 2>&1 & + sleep 5 + + echo "Vault: Write Vault Policies with API" + curl -s --data '{"rules":"path \"secret/demo\" {capabilities = [\"read\",\"list\"]}"}' --request PUT --header "X-Vault-Token: root" ${VAULT_ADDR}/v1/sys/policy/policy-demo + + echo "Vault: Write Vault Secret" + curl -s --data '{"value":"python_nomad"}' --request PUT --header "X-Vault-Token: root" ${VAULT_ADDR}/v1/secret/demo + + echo "Nomad: Enable Config Vault" +cat << EOF > /tmp/nomad.d/vault.hcl +vault +{ + enabled = true + address = "${VAULT_ADDR}" + token = "root" + allow_unauthenticated = false +} +EOF + fi +fi + +echo "Nomad: Config base" +cat << EOF > /tmp/nomad.d/base_config.hcl +datacenter = "dc1" +name = "pynomad1" +bind_addr = "${NOMAD_IP}" +client +{ + enabled = true + node_class = "default" +} +ports +{ + http = ${NOMAD_PORT_GUEST} + rpc = 4647 +} +addresses +{ + http = "${NOMAD_IP}" + rpc = "${NOMAD_IP}" +} +advertise +{ + http = "${NOMAD_IP}:${NOMAD_PORT_GUEST}" + rpc = "${NOMAD_IP}:4647" +} +log_level = "INFO" +enable_debug = false +EOF + +echo "Nomad: Config Server" +cat << EOF > /tmp/nomad.d/server.hcl +server +{ + enabled = true + bootstrap_expect = 1 +} +EOF + +if [ ${MAJOR_VERSION} -gt 6 ]; then +echo "Nomad: Version $NOMAD_VERSION supports acls" +echo "Nomad: Config ACL" +cat << EOF > /tmp/nomad.d/acl.hcl +acl +{ + enabled = true + token_ttl = "30s" + policy_ttl = "60s" +} +EOF +else + echo "Nomad: version $NOMAD_VERSION" +fi + +echo "Nomad: Create test job samples" +/tmp/nomad init +/tmp/nomad run -output example.nomad > example.json +chmod 777 example* + +echo "Nomad: Starting Nomad" +nohup /tmp/nomad agent -server -dev -config=/tmp/nomad.d > /dev/null 2>&1 & + + +PID=`ps -eaf | grep "vault server -dev" | grep -v grep | awk '{print $2}'` +if [ "" != "$PID" ]; then + echo "Vault: service is RUNNING" +else + echo "Vault: service is STOPED (could be not necessary)" +fi + +PID=`ps -eaf | grep "nomad agent -server" | grep -v grep | awk '{print $2}'` +if [ "" != "$PID" ]; then + echo "Nomad: service is RUNNING" +else + echo "Nomad: service is STOPED" +fi +sleep 15 + +echo "You can execute your test! ENJOY!" diff --git a/stop_daemons.sh b/stop_daemons.sh new file mode 100755 index 0000000..eeb28ef --- /dev/null +++ b/stop_daemons.sh @@ -0,0 +1,15 @@ + + +echo "VAULT: stoping" +PID=`ps -eaf | grep "vault server -dev" | grep -v grep | awk '{print $2}'` +if [ "" != "$PID" ]; then + echo "killing $PID" + kill -9 $PID +fi + +echo "NOMAD: stoping" +PID=`ps -eaf | grep "nomad agent -server" | grep -v grep | awk '{print $2}'` +if [ "" != "$PID" ]; then + echo "killing $PID" + kill -9 $PID +fi diff --git a/tests/common.py b/tests/common.py index e8de589..3d04a4e 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,4 +1,5 @@ import os +import requests # internal ip of docker IP = os.environ.get("NOMAD_IP", "192.168.33.10") @@ -12,3 +13,25 @@ # Test namespace NOMAD_NAMESPACE = "admin" + +# Security token +VAULT_TOKEN = os.environ.get("VAULT_TOKEN", "root") +VAULT_ADDR = os.environ.get("VAULT_ADDR", "http://" + IP + ":8200") +NOMAD_INTEGRATION_VAULT = os.environ.get("NOMAD_INTEGRATION_VAULT", "0.6.2") +VAULT_TEST = os.environ.get("VAULT_TEST", "false") +VAULT_POLICY_INVALID_TOKEN = '1a77d23a-01f9-d848-8457-08bcec267c65' +NOMAD_VERSION = os.environ.get("NOMAD_VERSION", "3.2.0") + + +NOMAD_INTEGRATION_VAULT_NUMBER = int(NOMAD_INTEGRATION_VAULT.replace(".","")) +NOMAD_VERSION_NUMBER = int(NOMAD_VERSION.replace(".","")) + +if VAULT_TEST == "true": + if NOMAD_VERSION_NUMBER >= NOMAD_INTEGRATION_VAULT_NUMBER: + print ("\n Vault integration") + # create token based on policy "policy-demo" + headers = {'X-Vault-Token': 'root'} + payload = '{"policies": ["policy-demo"],"ttl": "3h","renewable": true}' + r = requests.post(VAULT_ADDR + "/v1" + "/auth/token/create", headers=headers, data=payload) + VAULT_POLICY_TOKEN=r.json()["auth"]["client_token"] + print("\n SecurityVaultAcl: {}\n".format(VAULT_POLICY_TOKEN)) diff --git a/tests/conftest.py b/tests/conftest.py index a871f3b..aa8c27e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,3 +11,13 @@ def nomad_setup(): def nomad_setup_with_namespace(): n = nomad.Nomad(host=common.IP, port=common.NOMAD_PORT, verify=False, token=common.NOMAD_TOKEN, namespace=common.NOMAD_NAMESPACE) return n + +@pytest.fixture +def nomad_setup_vault_valid_token(): + n = nomad.Nomad(host=common.IP, port=common.NOMAD_PORT, verify=False, token=common.NOMAD_TOKEN, vaulttoken=common.VAULT_POLICY_TOKEN) + return n + +@pytest.fixture +def nomad_setup_vault_invalid_token(): + n = nomad.Nomad(host=common.IP, port=common.NOMAD_PORT, verify=False, token=common.NOMAD_TOKEN, vaulttoken=common.VAULT_POLICY_INVALID_TOKEN) + return n diff --git a/tests/test_acl.py b/tests/test_acl.py index ec7c151..ae84ddf 100644 --- a/tests/test_acl.py +++ b/tests/test_acl.py @@ -13,6 +13,9 @@ def test_create_bootstrap(nomad_setup): bootstrap = nomad_setup.acl.generate_bootstrap() assert "SecretID" in bootstrap common.NOMAD_TOKEN = bootstrap["SecretID"] + # For debug at vagrant you should use -s at test to view the token + # py.test -s --cov=nomad --cov-report=term-missing --runxfail tests/ + print("\n SecurityNomadRootToken: {}\n".format(common.NOMAD_TOKEN)) @pytest.mark.skipif(tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) < (0, 7, 0), reason="Nomad dispatch not supported") diff --git a/tests/test_vault_integration.py b/tests/test_vault_integration.py new file mode 100644 index 0000000..ae7fd81 --- /dev/null +++ b/tests/test_vault_integration.py @@ -0,0 +1,95 @@ +import pytest +import nomad +import json +import os +import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry +from nomad.api import exceptions + +def requests_retry_session(retries=3, backoff_factor=0.3, + status_forcelist=(500, 502, 504), session=None,): + session = session or requests.Session() + retry = Retry(total=retries, + read=retries, + connect=retries, + backoff_factor=backoff_factor, + status_forcelist=status_forcelist, + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + return session + +# # integration tests requires nomad Vagrant VM or Binary running +# Specific token for this policy +# Register Job +@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) < (0, 6, 2) + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) + or os.environ.get("VAULT_TEST") != "true", + reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") +@pytest.mark.run(order=-1) +def test_register_vault_job_valid(nomad_setup_vault_valid_token): + with open("vault.json") as fh: + job = json.loads(fh.read()) + nomad_setup_vault_valid_token.job.register_job("vault", job) + assert "vault" in nomad_setup_vault_valid_token.job + + +# Specific token for this policy +# Get Job +@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) < (0, 6, 2) + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) + or os.environ.get("VAULT_TEST") != "true", + reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") +@pytest.mark.run(order=-2) +def test_get_vault_job_valid(nomad_setup_vault_valid_token): + assert isinstance(nomad_setup_vault_valid_token.job.get_job("vault"), dict) == True + + +# Specific token for this policy +# Validate Job +@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) < (0, 6, 2) + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) + or os.environ.get("VAULT_TEST") != "true", + reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") +@pytest.mark.run(order=-3) +def test_get_vault_job_valid(nomad_setup_vault_valid_token): + assert isinstance(nomad_setup_vault_valid_token.job.get_job("vault"), dict) == True + +# Specific token for this policy +# Validate secret from vault +# deploy a container that run and http server +# and shows the secret stored at vault +@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) < (0, 6, 2) + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) + or os.environ.get("VAULT_TEST") != "true", + reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") +@pytest.mark.run(order=-4) +def test_get_secret_from_vault_job_valid(): + url="http://localhost:8080" + response = requests_retry_session(retries=20).get(url,timeout=60) + assert "python_nomad" in response.text + + +@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) < (0, 6, 2) + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) + or os.environ.get("VAULT_TEST") != "true", + reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") +@pytest.mark.run(order=-5) +def test_delete_vault_job(nomad_setup_vault_valid_token): + assert "EvalID" in nomad_setup_vault_valid_token.job.deregister_job("vault") + test_register_vault_job_valid(nomad_setup_vault_valid_token) + +# Specific BAD token for this policy +# test non valid token for deploy +@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) < (0, 6, 2) + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) + or os.environ.get("VAULT_TEST") != "true", + reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") +@pytest.mark.run(order=-6) +def test_register_vault_job_invalid(nomad_setup_vault_invalid_token): + with open("vault.json") as fh: + job = json.loads(fh.read()) + with pytest.raises(nomad.api.exceptions.BaseNomadException): + nomad_setup_vault_invalid_token.job.register_job("vault", job) diff --git a/vault.hcl b/vault.hcl new file mode 100755 index 0000000..626d221 --- /dev/null +++ b/vault.hcl @@ -0,0 +1,38 @@ +job "vault" { + datacenters = ["dc1"] + type = "service" + vault { policies = ["policy-demo"]} + group "web" { + count = 1 + restart { + attempts = 2 + interval = "1m" + delay = "15s" + mode = "fail" + } + task "vatul-sercret" { + driver = "docker" + config { + image = "i4spserrano/nomadvaultsecret:latest" + port_map { + http = 8080 + } + } + template { + data = < Date: Sun, 11 Nov 2018 01:37:33 +0100 Subject: [PATCH 02/14] correction of test and start stop files --- .travis.yml | 20 +++ README.md | 1 - Vagrantfile | 4 +- example_batch_parameterized.json | 0 start_daemons.sh | 70 ++++++---- stop_daemons.sh | 6 +- tests/common.py | 18 +-- ...tegration.py => test_integration_vault.py} | 75 +++++++---- vault.hcl | 0 vault.json | 0 vault_kv.hcl | 38 ++++++ vault_kv.json | 121 ++++++++++++++++++ 12 files changed, 288 insertions(+), 65 deletions(-) mode change 100644 => 100755 example_batch_parameterized.json rename tests/{test_vault_integration.py => test_integration_vault.py} (65%) mode change 100755 => 100644 vault.hcl mode change 100755 => 100644 vault.json create mode 100644 vault_kv.hcl create mode 100644 vault_kv.json diff --git a/.travis.yml b/.travis.yml index 384effc..2c1d166 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,6 +66,26 @@ env: - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.9.0" - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.9.0" - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.9.0" + - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.11.0" install: - pip install -r requirements-dev.txt - pip install codecov diff --git a/README.md b/README.md index 93e730c..9a01da9 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,6 @@ py.test --cov=nomad --cov-report=term-missing --runxfail tests/ export NOMAD_IP=127.0.0.1 export NOMAD_VERSION= export VAULT_VERSION= # should be higher than 0.6.2 -export VAULT_TEST=true # if you select no, vault integration will not be tested ./start_daemons.sh py.test --cov=nomad --cov-report=term-missing --runxfail tests/ ``` diff --git a/Vagrantfile b/Vagrantfile index 0715caa..be6ab68 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,12 +5,11 @@ NOMAD_IP="192.168.33.10" NOMAD_VERSION="0.8.6" NOMAD_PORT_GUEST=4646 NOMAD_PORT_HOST=4646 -VAULT_VERSION="0.9.0" +VAULT_VERSION="0.11.4" VAULT_PORT_GUEST=8200 VAULT_PORT_HOST=8200 VAULT_ADDR="http://127.0.0.1:8200" VAULT_TEST="true" -NOMAD_INTEGRATION_VAULT="0.6.2" Vagrant.configure(2) do |config| @@ -58,7 +57,6 @@ export NOMAD_PORT_HOST="#{NOMAD_PORT_HOST}" export VAULT_VERSION="#{VAULT_VERSION}" export VAULT_ADDR="#{VAULT_ADDR}" export VAULT_TEST="#{VAULT_TEST}" -export NOMAD_INTEGRATION_VAULT="#{NOMAD_INTEGRATION_VAULT}" EOF chmod +x /tmp/environment.vars.sh source /tmp/environment.vars.sh diff --git a/example_batch_parameterized.json b/example_batch_parameterized.json old mode 100644 new mode 100755 diff --git a/start_daemons.sh b/start_daemons.sh index 8a43d60..2f1590f 100755 --- a/start_daemons.sh +++ b/start_daemons.sh @@ -5,10 +5,6 @@ if [ -z "${NOMAD_VERSION}" ]; then exit 1 fi -if [ -z "${NOMAD_INTEGRATION_VAULT}" ]; then - NOMAD_INTEGRATION_VAULT="0.6.2" -fi - if [ -z "${NOMAD_PORT_GUEST}" ]; then NOMAD_PORT_GUEST="4646" fi @@ -21,10 +17,17 @@ if [ -z "${VAULT_VERSION}" ]; then VAULT_VERSION="0.6.2" fi +NOMAD_MAJOR_VERSION=`echo ${NOMAD_VERSION} | tr -d "."|sed "s/^0*//"` +VAULT_MAJOR_VERSION=`echo ${VAULT_VERSION} | tr -d "."|sed "s/^0*//"` + +if [ ${VAULT_MAJOR_VERSION} -lt 62 ]; then + echo "ATTENTION: Nomad Vault integration require Vault version >= 0.6.2. See https://www.nomadproject.io/guides/operations/vault-integration/index.html" +fi + if [ ! -f /tmp/nomad ]; then rm -rf /tmp/nomad fi -echo "NOMAD: Get Binary Files" +echo "Nomad: Get Binary Files" wget -q -P /tmp/ https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip yes | unzip -o -d /tmp /tmp/nomad_${NOMAD_VERSION}_linux_amd64.zip @@ -32,41 +35,61 @@ yes | unzip -o -d /tmp /tmp/nomad_${NOMAD_VERSION}_linux_amd64.zip if [ ! -f /tmp/vault ]; then rm -rf /tmp/vault fi -echo "VAULT: Get Binary Files" +echo "Vault: Get Binary Files" wget -q -P /tmp/ https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip yes | unzip -o -d /tmp /tmp/vault_${VAULT_VERSION}_linux_amd64.zip VAULT_ADDR="http://localhost:8200" -MAJOR_VERSION=`echo ${NOMAD_VERSION} | cut -d "." -f 2` -MAJOR_VERSION_VAULT_INTEGRATION=`echo ${NOMAD_VERSION} | tr -d "."|sed "s/^0*//"` -NOMAD_REQUIRED_TO_INEGRATE_WITH_VAULT=`echo ${NOMAD_INTEGRATION_VAULT}|tr -d "."|sed "s/^0*//"` echo "Nomad: Create config folder" rm -rf /tmp/nomad.d mkdir -p /tmp/nomad.d -if [ "${VAULT_TEST}" = "true" ]; then -if [ ${MAJOR_VERSION_VAULT_INTEGRATION} -gt ${NOMAD_REQUIRED_TO_INEGRATE_WITH_VAULT} ]; then - echo "Vault: Create policy file" + +if [ ${VAULT_MAJOR_VERSION} -lt 62 ]; then + echo "Vault: this version is not supported" +else +echo "Vault: Create policy file" cat << EOF > /tmp/policy-demo.hcl path "secret/demo" { capabilities = ["read"] } EOF - echo "Vault: Start Daemon Version: ${VAULT_VERSION}" - /tmp/vault server -dev -dev-listen-address=0.0.0.0:8200 -dev-root-token-id="root" > /dev/null 2>&1 & - sleep 5 + echo "Vault: Start Daemon Version: ${VAULT_VERSION}" + /tmp/vault server -dev -dev-listen-address=0.0.0.0:8200 -dev-root-token-id="root" > /dev/null 2>&1 & + sleep 5 + + echo "Vault: Write Vault Policies with API" + # + # For version > 0.9.0 deprecated use of policies. Rules vs Policy + # + if [ ${VAULT_MAJOR_VERSION} -lt 90 ]; then + curl -s --data '{"rules":"path \"secret/demo\" {capabilities = [\"read\",\"list\"]}"}' --request PUT --header "X-Vault-Token: root" ${VAULT_ADDR}/v1/sys/policy/policy-demo + else + curl -s --data '{"policy":"path \"secret/data/demo\" {capabilities = [\"read\",\"list\"]}"}' --request PUT --header "X-Vault-Token: root" ${VAULT_ADDR}/v1/sys/policy/policy-demo + fi + + echo "Vault: Write Vault Secret" + # + # Vault version >= 0.9.0 require versioned secrets + # + if [ ${VAULT_MAJOR_VERSION} -lt 90 ]; then + curl -s --data '{"value":"python_nomad"}' --request PUT --header "X-Vault-Token: root" ${VAULT_ADDR}/v1/secret/demo + else + curl -s --data '{"options": {"cas": 0},"data": {"value": "python_nomad"}}' --request PUT --header "X-Vault-Token: root" ${VAULT_ADDR}/v1/secret/data/demo + fi + + +fi + - echo "Vault: Write Vault Policies with API" - curl -s --data '{"rules":"path \"secret/demo\" {capabilities = [\"read\",\"list\"]}"}' --request PUT --header "X-Vault-Token: root" ${VAULT_ADDR}/v1/sys/policy/policy-demo - echo "Vault: Write Vault Secret" - curl -s --data '{"value":"python_nomad"}' --request PUT --header "X-Vault-Token: root" ${VAULT_ADDR}/v1/secret/demo - echo "Nomad: Enable Config Vault" +if [ ${NOMAD_MAJOR_VERSION} -ge 50 ]; then +echo "Nomad: Enable Config Vault" cat << EOF > /tmp/nomad.d/vault.hcl vault { @@ -76,7 +99,6 @@ vault allow_unauthenticated = false } EOF - fi fi echo "Nomad: Config base" @@ -117,7 +139,7 @@ server } EOF -if [ ${MAJOR_VERSION} -gt 6 ]; then +if [ ${NOMAD_MAJOR_VERSION} -gt 60 ]; then echo "Nomad: Version $NOMAD_VERSION supports acls" echo "Nomad: Config ACL" cat << EOF > /tmp/nomad.d/acl.hcl @@ -133,6 +155,8 @@ else fi echo "Nomad: Create test job samples" +rm -rf example.nomad +rm -rf example.json /tmp/nomad init /tmp/nomad run -output example.nomad > example.json chmod 777 example* @@ -154,6 +178,6 @@ if [ "" != "$PID" ]; then else echo "Nomad: service is STOPED" fi -sleep 15 +sleep 10 echo "You can execute your test! ENJOY!" diff --git a/stop_daemons.sh b/stop_daemons.sh index eeb28ef..4152b5a 100755 --- a/stop_daemons.sh +++ b/stop_daemons.sh @@ -1,15 +1,17 @@ -echo "VAULT: stoping" +echo "Vault: stoping" PID=`ps -eaf | grep "vault server -dev" | grep -v grep | awk '{print $2}'` if [ "" != "$PID" ]; then echo "killing $PID" kill -9 $PID fi -echo "NOMAD: stoping" +echo "Nomas: stoping" PID=`ps -eaf | grep "nomad agent -server" | grep -v grep | awk '{print $2}'` if [ "" != "$PID" ]; then echo "killing $PID" kill -9 $PID fi + +rm -rf example.json example.nomad diff --git a/tests/common.py b/tests/common.py index 3d04a4e..b253f9d 100644 --- a/tests/common.py +++ b/tests/common.py @@ -15,23 +15,19 @@ NOMAD_NAMESPACE = "admin" # Security token +VAULT_POLICY_INVALID_TOKEN = '1a77d23a-01f9-d848-8457-08bcec267c65' VAULT_TOKEN = os.environ.get("VAULT_TOKEN", "root") VAULT_ADDR = os.environ.get("VAULT_ADDR", "http://" + IP + ":8200") -NOMAD_INTEGRATION_VAULT = os.environ.get("NOMAD_INTEGRATION_VAULT", "0.6.2") -VAULT_TEST = os.environ.get("VAULT_TEST", "false") -VAULT_POLICY_INVALID_TOKEN = '1a77d23a-01f9-d848-8457-08bcec267c65' -NOMAD_VERSION = os.environ.get("NOMAD_VERSION", "3.2.0") - - -NOMAD_INTEGRATION_VAULT_NUMBER = int(NOMAD_INTEGRATION_VAULT.replace(".","")) -NOMAD_VERSION_NUMBER = int(NOMAD_VERSION.replace(".","")) +VAULT_TEST = os.environ.get("VAULT_TEST", "true") +VAULT_VERSION = os.environ.get("VAULT_VERSION", "0.6.0") -if VAULT_TEST == "true": - if NOMAD_VERSION_NUMBER >= NOMAD_INTEGRATION_VAULT_NUMBER: +if VAULT_TEST != "false": + if tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) >= (0, 6, 2): print ("\n Vault integration") # create token based on policy "policy-demo" headers = {'X-Vault-Token': 'root'} payload = '{"policies": ["policy-demo"],"ttl": "3h","renewable": true}' r = requests.post(VAULT_ADDR + "/v1" + "/auth/token/create", headers=headers, data=payload) VAULT_POLICY_TOKEN=r.json()["auth"]["client_token"] - print("\n SecurityVaultAcl: {}\n".format(VAULT_POLICY_TOKEN)) + print("\n SecurityVaultAclForPolicy: {}\n".format(VAULT_POLICY_TOKEN)) + print("\n SecurityVaultRootToken: root") diff --git a/tests/test_vault_integration.py b/tests/test_integration_vault.py similarity index 65% rename from tests/test_vault_integration.py rename to tests/test_integration_vault.py index ae7fd81..c0df962 100644 --- a/tests/test_vault_integration.py +++ b/tests/test_integration_vault.py @@ -21,75 +21,100 @@ def requests_retry_session(retries=3, backoff_factor=0.3, session.mount('https://', adapter) return session -# # integration tests requires nomad Vagrant VM or Binary running +# VAULT INTEGRATION # Specific token for this policy # Register Job -@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) < (0, 6, 2) +# Depens on version uses versioned secrets +# +@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 6, 2) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) - or os.environ.get("VAULT_TEST") != "true", + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) <= (0,5,0) + or os.environ.get("VAULT_TEST") == "false", reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") -@pytest.mark.run(order=-1) def test_register_vault_job_valid(nomad_setup_vault_valid_token): - with open("vault.json") as fh: + if tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 9, 0): + vault_job = "vault.json" + else: + vault_job = "vault_kv.json" + with open(vault_job) as fh: job = json.loads(fh.read()) nomad_setup_vault_valid_token.job.register_job("vault", job) assert "vault" in nomad_setup_vault_valid_token.job +# VAULT INTEGRATION # Specific token for this policy # Get Job -@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) < (0, 6, 2) +# Depens on version uses versioned secrets +# +@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 6, 2) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) - or os.environ.get("VAULT_TEST") != "true", + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) <= (0,5,0) + or os.environ.get("VAULT_TEST") == "false", reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") -@pytest.mark.run(order=-2) def test_get_vault_job_valid(nomad_setup_vault_valid_token): assert isinstance(nomad_setup_vault_valid_token.job.get_job("vault"), dict) == True +# VAULT INTEGRATION # Specific token for this policy # Validate Job -@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) < (0, 6, 2) +# Depens on version uses versioned secrets +# +@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 6, 2) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) - or os.environ.get("VAULT_TEST") != "true", + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) <= (0,5,0) + or os.environ.get("VAULT_TEST") == "false", reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") -@pytest.mark.run(order=-3) def test_get_vault_job_valid(nomad_setup_vault_valid_token): assert isinstance(nomad_setup_vault_valid_token.job.get_job("vault"), dict) == True +# VAULT INTEGRATION # Specific token for this policy # Validate secret from vault -# deploy a container that run and http server -# and shows the secret stored at vault -@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) < (0, 6, 2) +# Depens on version uses versioned secrets +# +@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 6, 2) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) - or os.environ.get("VAULT_TEST") != "true", + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) <= (0,5,0) + or os.environ.get("VAULT_TEST") == "false", reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") -@pytest.mark.run(order=-4) def test_get_secret_from_vault_job_valid(): url="http://localhost:8080" response = requests_retry_session(retries=20).get(url,timeout=60) assert "python_nomad" in response.text -@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) < (0, 6, 2) +# VAULT INTEGRATION +# Specific token for this policy +# De-Register Job +# Depens on version uses versioned secrets +# +@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 6, 2) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) - or os.environ.get("VAULT_TEST") != "true", + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) <= (0,5,0) + or os.environ.get("VAULT_TEST") == "false", reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") -@pytest.mark.run(order=-5) def test_delete_vault_job(nomad_setup_vault_valid_token): assert "EvalID" in nomad_setup_vault_valid_token.job.deregister_job("vault") test_register_vault_job_valid(nomad_setup_vault_valid_token) -# Specific BAD token for this policy -# test non valid token for deploy -@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) < (0, 6, 2) +# VAULT INTEGRATION +# Specific token for this policy +# Register Job with Bad Vault Token. It will report not authorized +# Depens on version uses versioned secrets +# +@pytest.mark.skipif(tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 6, 2) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) - or os.environ.get("VAULT_TEST") != "true", + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) <= (0,5,0) + or os.environ.get("VAULT_TEST") == "false", reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") -@pytest.mark.run(order=-6) def test_register_vault_job_invalid(nomad_setup_vault_invalid_token): - with open("vault.json") as fh: + if tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 9, 0): + vault_job = "vault.json" + else: + vault_job = "vault_kv.json" + with open(vault_job) as fh: job = json.loads(fh.read()) with pytest.raises(nomad.api.exceptions.BaseNomadException): nomad_setup_vault_invalid_token.job.register_job("vault", job) diff --git a/vault.hcl b/vault.hcl old mode 100755 new mode 100644 diff --git a/vault.json b/vault.json old mode 100755 new mode 100644 diff --git a/vault_kv.hcl b/vault_kv.hcl new file mode 100644 index 0000000..e6dbe5d --- /dev/null +++ b/vault_kv.hcl @@ -0,0 +1,38 @@ +job "vault" { + datacenters = ["dc1"] + type = "service" + vault { policies = ["policy-demo"]} + group "web" { + count = 1 + restart { + attempts = 2 + interval = "1m" + delay = "15s" + mode = "fail" + } + task "vatul-sercret" { + driver = "docker" + config { + image = "i4spserrano/nomadvaultsecret:latest" + port_map { + http = 8080 + } + } + template { + data = < Date: Sun, 11 Nov 2018 13:15:03 +0100 Subject: [PATCH 03/14] add documentation and minor changes --- .travis.yml | 1 - README.md | 19 +++++++------- Vagrantfile | 2 +- example_batch_parameterized.json | 0 start_daemons.sh | 8 +++--- vaultintegration.md | 43 ++++++++++++++++++++++++++++++++ 6 files changed, 58 insertions(+), 15 deletions(-) mode change 100755 => 100644 example_batch_parameterized.json create mode 100644 vaultintegration.md diff --git a/.travis.yml b/.travis.yml index 2c1d166..8df978f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,6 @@ env: global: - NOMAD_IP="127.0.0.1" - NOMAD_PORT="4646" - - NOMAD_INTEGRATION_VAULT="0.6.2" - VAULT_ADDR="http://127.0.0.1:8200" - VAULT_TEST="true" matrix: diff --git a/README.md b/README.md index 9a01da9..331c7bf 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,9 @@ NOMAD_REGION=us-east-1a ``` ## With Vault integration. + +Vault info [:link:](vaultintegration.md) + if you have configured a [Vault Integration ](https://www.nomadproject.io/docs/configuration/vault.html) to store your secrets. And you have configured: [`allow_unantenticated = false`](https://www.nomadproject.io/docs/configuration/vault.html#allow_unauthenticated) @@ -68,13 +71,6 @@ see you must to export and send a valid `VAULT_TOKEN`. VAULT_TOKEN=xxxx-xxxx-xxxx-xxxx ``` -## Skipt vault tests -if you want to test vault integration please export the variable: - -```bash -VAULT_TEST=true -``` - ## Class Dunders | Class | contains | len | getitem | iter | @@ -113,10 +109,15 @@ pip install -r requirements-dev.txt ``` ## Testing with vagrant and virtualbox + +- Define versions at vagrant file +- Execute tests ``` vagrant up --provider virtualbox -source /tmp/environment.vars.sh -py.test --cov=nomad --cov-report=term-missing --runxfail tests/ +``` +- Destroy Vagrant +``` +vagrant destroy ``` ## Testing with nomad binary diff --git a/Vagrantfile b/Vagrantfile index be6ab68..b73c7da 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -63,7 +63,7 @@ source /tmp/environment.vars.sh cd /vagrant ./start_daemons.sh -py.test -s -v --cov=nomad --cov-report=term-missing --runxfail tests/ +py.test --cov=nomad --cov-report=term-missing --runxfail tests/ SHELL diff --git a/example_batch_parameterized.json b/example_batch_parameterized.json old mode 100755 new mode 100644 diff --git a/start_daemons.sh b/start_daemons.sh index 2f1590f..3b547e0 100755 --- a/start_daemons.sh +++ b/start_daemons.sh @@ -167,16 +167,16 @@ nohup /tmp/nomad agent -server -dev -config=/tmp/nomad.d > /dev/null 2>&1 & PID=`ps -eaf | grep "vault server -dev" | grep -v grep | awk '{print $2}'` if [ "" != "$PID" ]; then - echo "Vault: service is RUNNING" + echo "Vault: service is Running" else - echo "Vault: service is STOPED (could be not necessary)" + echo "Vault: service is Stoped (could be not necessary)" fi PID=`ps -eaf | grep "nomad agent -server" | grep -v grep | awk '{print $2}'` if [ "" != "$PID" ]; then - echo "Nomad: service is RUNNING" + echo "Nomad: service is Running" else - echo "Nomad: service is STOPED" + echo "Nomad: service is Stoped. This make problems to make test!" fi sleep 10 diff --git a/vaultintegration.md b/vaultintegration.md new file mode 100644 index 0000000..b053489 --- /dev/null +++ b/vaultintegration.md @@ -0,0 +1,43 @@ +# Nomad & Vault + +## Vault + +Vault is a system to store secrets. This system could be integrated as a backend +of secrets of your Nomad cluster. + +## Documentation +https://www.hashicorp.com/products/vault/ + +## Nomad Stanza Job with Vault +https://www.nomadproject.io/docs/job-specification/vault.html + +## Security recomendaion using Vault backend +**You should enable** at nomad vault integration [`allow_unantenticated = false`](https://www.nomadproject.io/docs/configuration/vault.html#allow_unauthenticated) + +## Nomad Stanza with Vault API v1 +https://www.nomadproject.io/docs/job-specification/template.html#vault-kv-api-v1 + +- **Simple secrets backend** + +Ex: +``` +template { + data = < Date: Sun, 11 Nov 2018 13:35:05 +0100 Subject: [PATCH 04/14] travis did not start. try with less test --- .travis.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8df978f..e6f30cd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -75,18 +75,9 @@ env: - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.10.0" - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.10.0" - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.11.0" + install: -- pip install -r requirements-dev.txt +- pip install -r requirements.txt -r requirements-dev.txt - pip install codecov before_script: - sudo ./start_daemons.sh From 0497626f3a0ef09b206f8569e55b064fb0135f27 Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Sun, 11 Nov 2018 13:35:38 +0100 Subject: [PATCH 05/14] travis did not start. try with less test --- .travis.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index e6f30cd..e29819a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,17 +65,6 @@ env: - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.9.0" - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.9.0" - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.9.0" - - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.10.0" - install: - pip install -r requirements.txt -r requirements-dev.txt - pip install codecov From 4be2646645392cbfb9f449e77b5b95eaeca3eb7f Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Sun, 11 Nov 2018 14:02:25 +0100 Subject: [PATCH 06/14] review fix at nomad and vault integration to skip non valid test 0.8.5 and 0.5.6. From nomad changelog.md --- .travis.yml | 20 ++++++++++++++++++++ example_batch_parameterized.json | 0 tests/test_integration_vault.py | 30 ++++++++++++++++++++++++------ 3 files changed, 44 insertions(+), 6 deletions(-) mode change 100644 => 100755 example_batch_parameterized.json diff --git a/.travis.yml b/.travis.yml index e29819a..090a10a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,6 +65,26 @@ env: - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.9.0" - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.9.0" - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.9.0" + - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.11.0" install: - pip install -r requirements.txt -r requirements-dev.txt - pip install codecov diff --git a/example_batch_parameterized.json b/example_batch_parameterized.json old mode 100644 new mode 100755 diff --git a/tests/test_integration_vault.py b/tests/test_integration_vault.py index c0df962..9d96ad2 100644 --- a/tests/test_integration_vault.py +++ b/tests/test_integration_vault.py @@ -26,11 +26,14 @@ def requests_retry_session(retries=3, backoff_factor=0.3, # Register Job # Depens on version uses versioned secrets # +# NOMAD v 0.8.5 --> vault: Fix a regression in which Nomad was only compatible with Vault versions greater than 0.10.0 [GH-4698] +# NOMAD v 0.5.6 --> server/vault: Fix Vault Client panic when given nonexistent role [GH-2648] @pytest.mark.skipif(tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 6, 2) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) <= (0,5,0) + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,5,6) or os.environ.get("VAULT_TEST") == "false", - reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") + reason="0.8.5 vault: Fix [GH-4698]. 0.5.6 server/vault: [GH-2648]. Review at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md") def test_register_vault_job_valid(nomad_setup_vault_valid_token): if tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 9, 0): vault_job = "vault.json" @@ -47,11 +50,14 @@ def test_register_vault_job_valid(nomad_setup_vault_valid_token): # Get Job # Depens on version uses versioned secrets # +# NOMAD v 0.8.5 --> vault: Fix a regression in which Nomad was only compatible with Vault versions greater than 0.10.0 [GH-4698] +# NOMAD v 0.5.6 --> server/vault: Fix Vault Client panic when given nonexistent role [GH-2648] @pytest.mark.skipif(tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 6, 2) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) <= (0,5,0) + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,5,6) or os.environ.get("VAULT_TEST") == "false", - reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") + reason="0.8.5 vault: Fix [GH-4698]. 0.5.6 server/vault: [GH-2648]. Review at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md") def test_get_vault_job_valid(nomad_setup_vault_valid_token): assert isinstance(nomad_setup_vault_valid_token.job.get_job("vault"), dict) == True @@ -61,11 +67,14 @@ def test_get_vault_job_valid(nomad_setup_vault_valid_token): # Validate Job # Depens on version uses versioned secrets # +# NOMAD v 0.8.5 --> vault: Fix a regression in which Nomad was only compatible with Vault versions greater than 0.10.0 [GH-4698] +# NOMAD v 0.5.6 --> server/vault: Fix Vault Client panic when given nonexistent role [GH-2648] @pytest.mark.skipif(tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 6, 2) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) <= (0,5,0) + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,5,6) or os.environ.get("VAULT_TEST") == "false", - reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") + reason="0.8.5 vault: Fix [GH-4698]. 0.5.6 server/vault: [GH-2648]. Review at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md") def test_get_vault_job_valid(nomad_setup_vault_valid_token): assert isinstance(nomad_setup_vault_valid_token.job.get_job("vault"), dict) == True @@ -74,11 +83,14 @@ def test_get_vault_job_valid(nomad_setup_vault_valid_token): # Validate secret from vault # Depens on version uses versioned secrets # +# NOMAD v 0.8.5 --> vault: Fix a regression in which Nomad was only compatible with Vault versions greater than 0.10.0 [GH-4698] +# NOMAD v 0.5.6 --> server/vault: Fix Vault Client panic when given nonexistent role [GH-2648] @pytest.mark.skipif(tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 6, 2) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) <= (0,5,0) + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,5,6) or os.environ.get("VAULT_TEST") == "false", - reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") + reason="0.8.5 vault: Fix [GH-4698]. 0.5.6 server/vault: [GH-2648]. Review at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md") def test_get_secret_from_vault_job_valid(): url="http://localhost:8080" response = requests_retry_session(retries=20).get(url,timeout=60) @@ -90,11 +102,14 @@ def test_get_secret_from_vault_job_valid(): # De-Register Job # Depens on version uses versioned secrets # +# NOMAD v 0.8.5 --> vault: Fix a regression in which Nomad was only compatible with Vault versions greater than 0.10.0 [GH-4698] +# NOMAD v 0.5.6 --> server/vault: Fix Vault Client panic when given nonexistent role [GH-2648] @pytest.mark.skipif(tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 6, 2) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) <= (0,5,0) + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,5,6) or os.environ.get("VAULT_TEST") == "false", - reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") + reason="0.8.5 vault: Fix [GH-4698]. 0.5.6 server/vault: [GH-2648]. Review at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md") def test_delete_vault_job(nomad_setup_vault_valid_token): assert "EvalID" in nomad_setup_vault_valid_token.job.deregister_job("vault") test_register_vault_job_valid(nomad_setup_vault_valid_token) @@ -104,11 +119,14 @@ def test_delete_vault_job(nomad_setup_vault_valid_token): # Register Job with Bad Vault Token. It will report not authorized # Depens on version uses versioned secrets # +# NOMAD v 0.8.5 --> vault: Fix a regression in which Nomad was only compatible with Vault versions greater than 0.10.0 [GH-4698] +# NOMAD v 0.5.6 --> server/vault: Fix Vault Client panic when given nonexistent role [GH-2648] @pytest.mark.skipif(tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 6, 2) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,8,5) or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) <= (0,5,0) + or tuple(int(i) for i in os.environ.get("NOMAD_VERSION").split(".")) == (0,5,6) or os.environ.get("VAULT_TEST") == "false", - reason="Not supported in version. At version 0.8.5 see regresion of 8.5.6 at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md, or you have configured a false VAULT_TEST") + reason="0.8.5 vault: Fix [GH-4698]. 0.5.6 server/vault: [GH-2648]. Review at https://github.com/hashicorp/nomad/blob/master/CHANGELOG.md") def test_register_vault_job_invalid(nomad_setup_vault_invalid_token): if tuple(int(i) for i in os.environ.get("VAULT_VERSION").split(".")) < (0, 9, 0): vault_job = "vault.json" From 036f3dec43dac9606732451f6aa677997c93df90 Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Sun, 11 Nov 2018 16:09:59 +0100 Subject: [PATCH 07/14] review fix at nomad and vault integration to skip non valid test 0.8.5 and 0.5.6. From nomad changelog.md --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 090a10a..d95ca75 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,6 +85,7 @@ env: - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.11.0" - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.11.0" - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.11.0" + install: - pip install -r requirements.txt -r requirements-dev.txt - pip install codecov From 09c7873586b038662ce87bde1da47999cafe0534 Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Sun, 11 Nov 2018 16:12:21 +0100 Subject: [PATCH 08/14] test until vault 10 --- .travis.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index d95ca75..888fc6d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -75,17 +75,7 @@ env: - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.10.0" - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.10.0" - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.11.0" - + install: - pip install -r requirements.txt -r requirements-dev.txt - pip install codecov From c90103ff9ec15bd0830497320a9578690bdc0cac Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Sun, 11 Nov 2018 16:13:47 +0100 Subject: [PATCH 09/14] test until vault 10 --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 888fc6d..c3f419f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,8 +74,7 @@ env: - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.10.0" - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.10.0" - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.10.0" - + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.10.0" install: - pip install -r requirements.txt -r requirements-dev.txt - pip install codecov From 4345340d8784454f2b97cc7214315b7f3a817c12 Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Sun, 11 Nov 2018 16:15:01 +0100 Subject: [PATCH 10/14] test until vault 9 --- .travis.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index c3f419f..9666c94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,17 +64,7 @@ env: - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.9.0" - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.9.0" - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.9.0" - - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.9.0" - - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.9.0" install: - pip install -r requirements.txt -r requirements-dev.txt - pip install codecov From fa714f78942034aa2789c371a54707c5a9516da8 Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Sun, 11 Nov 2018 16:36:59 +0100 Subject: [PATCH 11/14] test until vault 10 --- .travis.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9666c94..daf4e21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,17 @@ env: - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.9.0" - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.9.0" - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.9.0" - - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.9.0" + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.9.0" + - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.10.0" install: - pip install -r requirements.txt -r requirements-dev.txt - pip install codecov From 2cec75f68c5c36f5499d1b6ed1c85ec7dcd57594 Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Sun, 11 Nov 2018 19:09:12 +0100 Subject: [PATCH 12/14] test until vault 11 --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.travis.yml b/.travis.yml index daf4e21..090a10a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -75,6 +75,16 @@ env: - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.10.0" - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.10.0" - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.10.0" + - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.11.0" install: - pip install -r requirements.txt -r requirements-dev.txt - pip install codecov From 5ff9ed257d0cd2dd9e5a638a79df5f62af3de2ee Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Sun, 11 Nov 2018 19:27:21 +0100 Subject: [PATCH 13/14] reduce to avoid the max jobs: Build matrix exceeds 200 jobs. --- .travis.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 090a10a..1b346c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,26 +25,6 @@ env: - VAULT_ADDR="http://127.0.0.1:8200" - VAULT_TEST="true" matrix: - - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.6.2" - - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.6.2" - - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.6.2" - - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.6.2" - - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.6.2" - - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.6.2" - - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.6.2" - - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.6.2" - - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.6.2" - - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.6.2" - - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.7.0" - - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.7.0" - - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.7.0" - - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.7.0" - - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.7.0" - - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.7.0" - - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.7.0" - - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.7.0" - - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.7.0" - - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.7.0" - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.8.0" - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.8.0" - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.8.0" From 0f8dd9dfa1d448465be490f0acf9f5df96cd893f Mon Sep 17 00:00:00 2001 From: Pedro Serrano Date: Sun, 11 Nov 2018 20:22:24 +0100 Subject: [PATCH 14/14] remove test with vault version 0.11.0 that make crash nomad at all versions --- .travis.yml | 20 ++++++++++---------- vaultintegration.md | 6 ++++++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b346c0..9b1db4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,16 +55,16 @@ env: - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.10.0" - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.10.0" - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.10.0" - - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.11.0" - - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.11.0" + - NOMAD_VERSION="0.3.2";VAULT_VERSION="0.11.4" + - NOMAD_VERSION="0.4.1";VAULT_VERSION="0.11.4" + - NOMAD_VERSION="0.5.6";VAULT_VERSION="0.11.4" + - NOMAD_VERSION="0.6.0";VAULT_VERSION="0.11.4" + - NOMAD_VERSION="0.7.1";VAULT_VERSION="0.11.4" + - NOMAD_VERSION="0.8.1";VAULT_VERSION="0.11.4" + - NOMAD_VERSION="0.8.3";VAULT_VERSION="0.11.4" + - NOMAD_VERSION="0.8.4";VAULT_VERSION="0.11.4" + - NOMAD_VERSION="0.8.5";VAULT_VERSION="0.11.4" + - NOMAD_VERSION="0.8.6";VAULT_VERSION="0.11.4" install: - pip install -r requirements.txt -r requirements-dev.txt - pip install codecov diff --git a/vaultintegration.md b/vaultintegration.md index b053489..54752bd 100644 --- a/vaultintegration.md +++ b/vaultintegration.md @@ -41,3 +41,9 @@ template { EOF } ``` + +## Non working versions + +- NOMAD v 0.5.6 --> server/vault: Fix Vault Client panic when given nonexistent role [GH-2648] +- NOMAD v 0.8.5 --> vault: Fix a regression in which Nomad was only compatible with Vault versions greater than 0.10.0 [GH-4698] +- NOMAD (all version) vs Vault 0.11.0 --> Make crash nomad agent.