From 1ec72f8633e4f59de566e7104c8e3cbbc02b0f52 Mon Sep 17 00:00:00 2001 From: Ilia Feldgun Date: Tue, 6 Apr 2021 19:17:31 +0300 Subject: [PATCH 1/3] Added setup.py PEP-440 versioning with git cli Readme dev-requirements.txt Lint for python3 (incomplete) --- .gitignore | 1 + Makefile | 2 +- README.md | 17 ++++++++++++++ dev-requirements.txt | 2 ++ py/rackattack/playaround.py | 20 ++++++++-------- setup.py | 41 +++++++++++++++++++++++++++++++++ test/simpleusecase.py | 10 ++++---- test/test.py | 46 ++++++++++++++++++------------------- 8 files changed, 100 insertions(+), 39 deletions(-) create mode 100644 README.md create mode 100644 dev-requirements.txt create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 9ad8b7b..0823aac 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ images.fortests .coverage attic .idea/ +.vscode/ diff --git a/Makefile b/Makefile index 76beaab..2af6624 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ unittest: python -m coverage report --show-missing --include=py/*.py --omit=py/rackattack/tests/*.py,*/__init__.py check_convention: - pep8 py test* --max-line-length=109 + python -m pep8 py test* --max-line-length=109 #this can not run in dirbalak clean build, as solvent can not yet be run at this point (still running in #rootfs-build-nostrato, this project is a dependency of rootfs-buid). This is why this is actually being diff --git a/README.md b/README.md new file mode 100644 index 0000000..e095c28 --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +# Rackattack API +API for provisioning rackattack hosts + + +```python +from rackattack.ssh import connection + +def get_connection(ip, username, password, timeout=5 * 60): + node = connection.Connection(ip, username, password) + node.waitForTCPServer(timeout=timeout, interval=60) + node.connect() + return node + +node = get_connection("192.168.1.1", "root", "password") +date = node.run.script("date") + +``` \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..fd3cee0 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,2 @@ +pep8 +coverage \ No newline at end of file diff --git a/py/rackattack/playaround.py b/py/rackattack/playaround.py index 9c12430..9f90734 100644 --- a/py/rackattack/playaround.py +++ b/py/rackattack/playaround.py @@ -14,13 +14,13 @@ parser.add_argument("--disk1SizeGB", type=int, default=None) args = parser.parse_args() -print "Connecting" +print("Connecting") client = clientfactory.factory() -print "Allocating" +print("Allocating") hardwareConstraints = dict() if args.disk1SizeGB is not None: hardwareConstraints['disk1SizeGB'] = args.disk1SizeGB -print "Hardware Constraints:", hardwareConstraints +print("Hardware Constraints:"), hardwareConstraints requirement = api.Requirement( imageLabel=args.label, imageHint="playaround", @@ -30,13 +30,13 @@ allocationInfo=api.AllocationInfo(user=args.user, purpose="playaround", nice=args.nice)) allocation.wait(timeout=8 * 60) assert allocation.done(), "Allocation failed" -print "Done allocating, Waiting for boot to finish" +print("Done allocating, Waiting for boot to finish") try: node = allocation.nodes()['node'] credentials = dict(port=22) credentials.update(node.rootSSHCredentials()) - print "ROOT SSH Credentials:" - print credentials + print("ROOT SSH Credentials:") + print(credentials) if not args.noSSH: ssh = connection.Connection(**credentials) ssh.waitForTCPServer() @@ -51,17 +51,17 @@ try: log = node.fetchSerialLog() open("/tmp/serial.log", "w").write(log) - print "serial log stored in /tmp/serial.log" + print("serial log stored in /tmp/serial.log") except Exception as e: - print "Unable to fetch serial log: %s" % e + print("Unable to fetch serial log: %s" % e) raise - print "Opening ssh session. Close it to free up allocation" + print("Opening ssh session. Close it to free up allocation") os.system( "sshpass -p %(password)s ssh -o ServerAliveInterval=5 -o ServerAliveCountMax=1 " "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p %(port)d " "%(username)s@%(hostname)s" % credentials) else: - print "Not connecting to machine via SSH. Hit Ctrl-C to close the machine." + print("Not connecting to machine via SSH. Hit Ctrl-C to close the machine.") time.sleep(1000000000) finally: allocation.free() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d77966e --- /dev/null +++ b/setup.py @@ -0,0 +1,41 @@ +import setuptools +import subprocess +import re + + +def version(): + git_describe_cmd = ["git", "describe", "--tags", "HEAD"] + git_head_tag = subprocess.check_output(git_describe_cmd).strip().decode("utf-8") + python_version = git_head_tag + unofficial_tag = re.search(r"(?=.*)-(\d+)-", python_version) + if unofficial_tag: + pep_440_suffix = ".dev{}+".format(unofficial_tag.group(1)) + python_version = python_version.replace(unofficial_tag.group(0), pep_440_suffix) + return python_version + + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="rackattack", + packages=setuptools.find_packages(where="py"), + package_dir={"": "py"}, + version=version(), + python_requires="<=2.7.18", + install_requires=["twisted", "greenlet"], + author="Stratoscale", + author_email="zcompute@zadarastorage.com", + description="API for provisioning rackattack hosts", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/Stratoscale/rackattack-api", + project_urls={ + "Bug Tracker": "https://github.com/Stratoscale/rackattack-api/issues", + }, + classifiers=[ + "Programming Language :: Python :: 2", + "License :: Apache License 2.0", + "Operating System :: OS Independent", + ], +) diff --git a/test/simpleusecase.py b/test/simpleusecase.py index 5108730..09f2959 100644 --- a/test/simpleusecase.py +++ b/test/simpleusecase.py @@ -2,7 +2,7 @@ from rackattack import clientfactory from rackattack.ssh import connection from rackattack import api -import pdb +import ipdb hint = 'rootfs-basic' @@ -15,9 +15,9 @@ requirement = api.Requirement(imageLabel=label, imageHint=hint) info = api.AllocationInfo(user='rackattack-api test', purpose='integration test') allocation = client.allocate(dict(it=requirement), info) - print "Created allocation, waiting for node inauguration" + print("Created allocation, waiting for node inauguration") allocation.wait(timeout=7 * 60) - print "Allocation successfull, waiting for ssh" + print("Allocation successfull, waiting for ssh") try: nodes = allocation.nodes() assert len(nodes) == 1, nodes @@ -25,8 +25,8 @@ ssh = connection.Connection(**it.rootSSHCredentials()) ssh.waitForTCPServer() ssh.connect() - print "SSH connected" - pdb.set_trace() + print("SSH connected") + ipdb.set_trace() echo = ssh.run.script("echo hello") assert 'hello' in echo, echo finally: diff --git a/test/test.py b/test/test.py index eb585f9..e073e96 100644 --- a/test/test.py +++ b/test/test.py @@ -1,18 +1,18 @@ -import logging -import unittest -import subprocess -import os -import test -import time -import contextlib -import socket -import threading -import thread -from rackattack import clientfactory from rackattack.ssh import connection -from rackattack import api +from rackattack import clientfactory from rackattack.tcp import transport from rackattack.tcp import subscribe +from rackattack import api +import contextlib +import subprocess +import threading +import unittest +import logging +import thread +import socket +import time +import test +import os SIMPLE_USE_CASE = os.path.join(os.path.dirname(test.__file__), "simpleusecase.py") @@ -39,9 +39,9 @@ def test_singleNodeAllocation(self): def test_singleNodeAllocation_PDBDoesNotCauseAllocationToDie(self): popen = subprocess.Popen(["python", SIMPLE_USE_CASE], stdin=subprocess.PIPE) - print "Sleeping for 180 seconds, to make sure heartbeat timeout occours, if pdb stops" + print("Sleeping for 180 seconds, to make sure heartbeat timeout occours, if pdb stops") time.sleep(180) - print "Done Sleeping for 180 seconds, resuming PDB" + print("Done Sleeping for 180 seconds, resuming PDB") popen.stdin.write("c\n") popen.stdin.close() result = popen.wait() @@ -65,18 +65,18 @@ def test_LocalForwardingTunnel(self): ssh.run.backgroundScript("python /tmp/server.py 7789 > /tmp/output") self._send(port2, "wassup") self.assertIn('wassup', ssh.ftp.getContents("/tmp/output")) - print "Closing just one tunnel server" + print("Closing just one tunnel server") ssh.tunnel.stopLocalForward(7787) time.sleep(1) - print "Stopping all tunnels" + print("Stopping all tunnels") ssh.tunnel.stopAll() with self.assertRaises(Exception): self._receiveAll(port) - print "Closing tunnel" + print("Closing tunnel") ssh.tunnel.close() - print "Sleeping for few seconds, watch for exceptions from other threads" + print("Sleeping for few seconds, watch for exceptions from other threads") time.sleep(3) - print "Done Sleeping for few seconds" + print("Done Sleeping for few seconds") def _flood_server(self, port, message): s = socket.socket() @@ -114,20 +114,20 @@ def _allocateOne(self): requirement = api.Requirement(imageLabel=LABEL, imageHint=HINT) info = api.AllocationInfo(user='rackattack-api test', purpose='integration test') allocation = client.allocate(dict(it=requirement), info) - print "Created allocation, waiting for node inauguration" + print("Created allocation, waiting for node inauguration") try: allocation.wait(timeout=7 * 60) - print "Allocation successfull, waiting for ssh" + print("Allocation successfull, waiting for ssh") nodes = allocation.nodes() assert len(nodes) == 1, nodes it = nodes['it'] it.fetchSerialLog() allocation.fetchPostMortemPack() - print "SSH credentials:", it.rootSSHCredentials() + print("SSH credentials:", it.rootSSHCredentials()) ssh = connection.Connection(**it.rootSSHCredentials()) ssh.waitForTCPServer() ssh.connect() - print "SSH connected" + print("SSH connected") yield it, ssh, allocation finally: allocation.free() From 5fd84f04ebb1106826ec4b9200d6f8f0f682eab0 Mon Sep 17 00:00:00 2001 From: IliaFeldgun Date: Thu, 7 Dec 2023 22:24:37 +0200 Subject: [PATCH 2/3] squash! Added setup.py PEP-440 versioning with git cli Readme dev-requirements.txt Lint for python3 (incomplete) --- dev-requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index fd3cee0..dcd28be 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,2 +1,3 @@ pep8 -coverage \ No newline at end of file +coverage +ipdb \ No newline at end of file From ad2940df39f2bea1ef7bb915c14e9490c0d159c3 Mon Sep 17 00:00:00 2001 From: IliaFeldgun Date: Tue, 16 Jul 2024 11:44:42 +0300 Subject: [PATCH 3/3] package-wip --- Dockerfile.rackattack-api-build | 8 +++++ Makefile | 36 +++++++++++++++---- artifacts.yaml | 2 ++ dev-requirements.txt | 4 ++- py/rackattack/__init__.py | 2 -- py/rackattack/packagesvalidation.py | 4 +-- .../pikapatchwakeupfromanotherthread.py | 2 +- py/rackattack/ssh/run.py | 3 +- py/rackattack/tcp/allocation.py | 2 +- py/rackattack/tcp/publish.py | 2 +- py/rackattack/tests/fake_pika.py | 2 +- py/rackattack/tests/test_publish.py | 2 +- pyproject.toml | 3 ++ requirements.txt | 1 + setup.py | 23 +++--------- skipper.yaml | 5 +++ 16 files changed, 66 insertions(+), 35 deletions(-) create mode 100644 Dockerfile.rackattack-api-build create mode 100644 artifacts.yaml create mode 100644 pyproject.toml create mode 100644 skipper.yaml diff --git a/Dockerfile.rackattack-api-build b/Dockerfile.rackattack-api-build new file mode 100644 index 0000000..763d394 --- /dev/null +++ b/Dockerfile.rackattack-api-build @@ -0,0 +1,8 @@ +FROM rackattack-nas.dc1:5000/ubuntu-dev-base:379308aa77bd0a19ebf12d868e7d19ee35a19d6c +RUN apt-get install -y python2.7 python2.7-dev +RUN curl 'https://bootstrap.pypa.io/pip/2.7/get-pip.py' | python2.7 +COPY requirements.txt /tmp/requirements.txt +COPY dev-requirements.txt /tmp/dev-requirements.txt +RUN python2.7 -m pip install -r /tmp/requirements.txt -r /tmp/dev-requirements.txt +RUN python3 -m pip install -r /tmp/requirements.txt -r /tmp/dev-requirements.txt +RUN rm /tmp/requirements.txt /tmp/dev-requirements.txt diff --git a/Makefile b/Makefile index 2af6624..23e66ae 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,41 @@ all: check_convention unittest +.PHONY: all test build test_py2 test_py3 lint_py2 lint_py3 +ARTIFACT=dist/racattack_api-*-py2.py3-none-any.whl +all: lint test build -unittest: - UPSETO_JOIN_PYTHON_NAMESPACES=Yes PYTHONPATH=py python -m coverage run py/rackattack/tests/runner.py - python -m coverage report --show-missing --include=py/*.py --omit=py/rackattack/tests/*.py,*/__init__.py +test: test_py2 test_py3 -check_convention: - python -m pep8 py test* --max-line-length=109 +test_py3: + PYTHONPATH=py python3 -m coverage run py/rackattack/tests/runner.py + python3 -m coverage report --show-missing --include=py/*.py --omit=py/rackattack/tests/*.py,*/__init__.py + +test_py2: + PYTHONPATH=py python2.7 -m coverage run py/rackattack/tests/runner.py + python2.7 -m coverage report --show-missing --include=py/*.py --omit=py/rackattack/tests/*.py,*/__init__.py + +lint: lint_py2 lint_py3 + +lint_py3: + python3 -m pep8 py --max-line-length=120 + +lint_py2: + python2.7 -m pep8 py --max-line-length=120 + +build: $(ARTIFACT) + +clean: + find . -name *.pyc -delete + find . -name __pycache__ -delete + rm -rf dist */*.egg-info build *.stratolog + +$(ARTIFACT): + python3 -m build --wheel #this can not run in dirbalak clean build, as solvent can not yet be run at this point (still running in #rootfs-build-nostrato, this project is a dependency of rootfs-buid). This is why this is actually being #run in pyracktest delayed_racktest: - UPSETO_JOIN_PYTHON_NAMESPACES=yes PYTHONPATH=$(PWD):$(PWD)/py python test/test.py $(TESTS) + PYTHONPATH=$(PWD):$(PWD)/py python test/test.py $(TESTS) virttest: RACKATTACK_PROVIDER=tcp://localhost:1014@@amqp://guest:guest@localhost:1013/%2F@@http://localhost:1016 $(MAKE) delayed_racktest phystest: diff --git a/artifacts.yaml b/artifacts.yaml new file mode 100644 index 0000000..54936e1 --- /dev/null +++ b/artifacts.yaml @@ -0,0 +1,2 @@ +pips: + - dist/rackattack-api-*-py2.py3-none-any.whl diff --git a/dev-requirements.txt b/dev-requirements.txt index dcd28be..20d50c8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,5 @@ pep8 coverage -ipdb \ No newline at end of file +ipdb +build +mock \ No newline at end of file diff --git a/py/rackattack/__init__.py b/py/rackattack/__init__.py index 48e952a..e69de29 100644 --- a/py/rackattack/__init__.py +++ b/py/rackattack/__init__.py @@ -1,2 +0,0 @@ -import upseto.pythonnamespacejoin -__path__.extend(upseto.pythonnamespacejoin.join(globals())) diff --git a/py/rackattack/packagesvalidation.py b/py/rackattack/packagesvalidation.py index b2f648d..bc7dd49 100644 --- a/py/rackattack/packagesvalidation.py +++ b/py/rackattack/packagesvalidation.py @@ -18,7 +18,7 @@ def _validateMinimumVersion(packageName, minVersion): package = __import__(packageName) actualVersion = package.__version__ actualVersion = _normalizedVersionNumber(actualVersion) - for componentNr in xrange(len(minVersion)): + for componentNr in range(len(minVersion)): minimumComponent = minVersion[componentNr] actualComponent = actualVersion[componentNr] if actualComponent < minimumComponent: @@ -32,5 +32,5 @@ def _validateMinimumVersion(packageName, minVersion): def validateMinimumVersions(**packagesVersions): - for packageName, minVersion in packagesVersions.iteritems(): + for packageName, minVersion in packagesVersions.items(): _validateMinimumVersion(packageName, minVersion) diff --git a/py/rackattack/pikapatchwakeupfromanotherthread.py b/py/rackattack/pikapatchwakeupfromanotherthread.py index fcd3f34..40194a1 100644 --- a/py/rackattack/pikapatchwakeupfromanotherthread.py +++ b/py/rackattack/pikapatchwakeupfromanotherthread.py @@ -1,5 +1,5 @@ import os -import Queue +from queue import Queue import select import logging import signal diff --git a/py/rackattack/ssh/run.py b/py/rackattack/ssh/run.py index 0730f4f..0b5adad 100644 --- a/py/rackattack/ssh/run.py +++ b/py/rackattack/ssh/run.py @@ -21,7 +21,8 @@ def execute(self, command, outputTimeout=20 * 60, wrapCmd=True, verbose=True, co try: if verbose: name = getattr(self._sshClient, 'name', '') - self._logger.debug("Running bash script: %(cmd)s %(name)s" % dict(cmd=command.strip(), name='on ' + name if name else name)) + message = "Running bash script: %s %s" % (command.strip(), "on " + name if name else "") + self._logger.debug(message) chan.exec_command(commandToExecute) chan.settimeout(outputTimeout) stdin = chan.makefile('wb', -1) diff --git a/py/rackattack/tcp/allocation.py b/py/rackattack/tcp/allocation.py index a5d7eb0..1ad4a6a 100644 --- a/py/rackattack/tcp/allocation.py +++ b/py/rackattack/tcp/allocation.py @@ -177,4 +177,4 @@ def _refetchInauguratorIDs(self): def renameUser(self, user): assert isinstance(user, str) - self._ipcClient.call('rename_allocation', id=self._id, user=user) \ No newline at end of file + self._ipcClient.call('rename_allocation', id=self._id, user=user) diff --git a/py/rackattack/tcp/publish.py b/py/rackattack/tcp/publish.py index 4efecbc..a73a798 100644 --- a/py/rackattack/tcp/publish.py +++ b/py/rackattack/tcp/publish.py @@ -4,7 +4,7 @@ import pika import simplejson import threading -import Queue +from queue import Queue class PublishSpooler(threading.Thread): diff --git a/py/rackattack/tests/fake_pika.py b/py/rackattack/tests/fake_pika.py index 753806f..2b26840 100644 --- a/py/rackattack/tests/fake_pika.py +++ b/py/rackattack/tests/fake_pika.py @@ -1,5 +1,5 @@ import pika -import Queue +from queue import Queue _rabbitMQBrokers = dict() diff --git a/py/rackattack/tests/test_publish.py b/py/rackattack/tests/test_publish.py index 5ee6c55..89d91d7 100644 --- a/py/rackattack/tests/test_publish.py +++ b/py/rackattack/tests/test_publish.py @@ -2,7 +2,7 @@ import mock import json import pika -import Queue +from queue import Queue import logging import greenlet import unittest diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..38b9357 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "python-setuptools-plugins", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 9cba1d5..cc43291 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ +python-setuptools-plugins==0.0.2+g865898c twisted greenlet diff --git a/setup.py b/setup.py index d77966e..e61b75e 100644 --- a/setup.py +++ b/setup.py @@ -1,29 +1,15 @@ +from setuptools_plugins.version import get_version import setuptools -import subprocess -import re - - -def version(): - git_describe_cmd = ["git", "describe", "--tags", "HEAD"] - git_head_tag = subprocess.check_output(git_describe_cmd).strip().decode("utf-8") - python_version = git_head_tag - unofficial_tag = re.search(r"(?=.*)-(\d+)-", python_version) - if unofficial_tag: - pep_440_suffix = ".dev{}+".format(unofficial_tag.group(1)) - python_version = python_version.replace(unofficial_tag.group(0), pep_440_suffix) - return python_version - with open("README.md", "r") as fh: long_description = fh.read() setuptools.setup( - name="rackattack", + name="rackattack_api", packages=setuptools.find_packages(where="py"), package_dir={"": "py"}, - version=version(), - python_requires="<=2.7.18", - install_requires=["twisted", "greenlet"], + setup_requires=['python-setuptools-plugins'], + version=get_version(), author="Stratoscale", author_email="zcompute@zadarastorage.com", description="API for provisioning rackattack hosts", @@ -38,4 +24,5 @@ def version(): "License :: Apache License 2.0", "Operating System :: OS Independent", ], + options={'bdist_wheel': {'universal': True}} ) diff --git a/skipper.yaml b/skipper.yaml new file mode 100644 index 0000000..e86f9a7 --- /dev/null +++ b/skipper.yaml @@ -0,0 +1,5 @@ +registry: rackattack-nas.dc1:5000 +build-container-image: rackattack-api-build + +make: + makefile: Makefile