From f8e5267ae103c21a48a6c4603ba25755d808894e Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Wed, 9 Mar 2022 19:25:56 +0100 Subject: [PATCH 1/8] Implement the report using the Tern This commit implements the report using Tern The reports use the ``concurrent.futures``, but instead of building all the implementation for it, we use flask-executer that makes the implementation on top of Flask. All the report requests go to the ``tern-tasks`` (Futures) and are handled such as a Celery implementation. https://docs.celeryproject.org/en/stable/reference/celery.states.html#misc All tasks start with the status 'PENDING', which means that the task has no defined status until the ``tern-tasks`` knows it. Once the task is in picked up by the ``tern-tasks`` it is moved to 'RUNNING'. If the processing finishes, the status is moved to 'SUCCESS' independently of the task content/result. Here the status is for the task itself. The 'FAILURE' status is given when the task cannot finish its routine. The Tern, for now, is called using ``subprocess``, and the output is handled by the stdout and stderr. The initial API specification was changed a bit for more clearness. How to run as Docker container was added to the ``README.md``. Signed-off-by: Kairo de Araujo --- .dockerignore | 21 +++ .gitignore | 3 + Dockerfile | 29 ++++ Makefile | 6 + Pipfile | 2 + Pipfile.lock | 253 +++++++++++++++---------------- README.md | 11 +- app.py | 9 +- docker_start.sh | 7 + requirements-dev.txt | 16 +- requirements.txt | 8 +- tern_api/__init__.py | 10 +- tern_api/api/v1/common_models.py | 42 +++-- tern_api/api/v1/reports.py | 22 ++- tern_api/constants.py | 15 ++ tern_api/reports.py | 181 ++++++++++++++++++++++ 16 files changed, 467 insertions(+), 168 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 docker_start.sh create mode 100644 tern_api/constants.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3e0dcff --- /dev/null +++ b/.dockerignore @@ -0,0 +1,21 @@ +tests/ +docs/ +cached/ +.github +.gitignore +Dockerfile +Makefile +Pipfile +Pipfile.lock +README.md +requirements-dev.txt +setup.py +tox.ini +.coverage +.dockerignore +.git +.mypy_cache +.pytest_cache +.tox +__pycache__ +tern_rest_api.egg-info \ No newline at end of file diff --git a/.gitignore b/.gitignore index b6e4761..b1aa18c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# tern-rest-api specifics +cached + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..24db03f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# Copyright (c) 2022 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause + +FROM python:3.9-slim-buster as base + +RUN echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.list.d/bullseye.list \ + && echo "Package: *\nPin: release n=bullseye\nPin-Priority: 50" > /etc/apt/preferences.d/bullseye \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + attr \ + findutils \ + fuse-overlayfs/bullseye \ + fuse3/bullseye \ + git \ + jq \ + skopeo \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir /opt/tern-rest-api + +ADD . /opt/tern-rest-api +WORKDIR /opt/tern-rest-api +RUN pip install --no-cache -r ./requirements.txt + +ENV TERN_API_CACHE_DIR=/var/opt/tern-rest-api/cached +ENV TERN_DEFAULT_REGISTRY="registry.hub.docker.com" + +ENV FLASK_APP=/opt/tern-rest-api/app.py +CMD ["bash", "docker_start.sh"] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a2d939b --- /dev/null +++ b/Makefile @@ -0,0 +1,6 @@ +build-dev: + docker build -t tern-rest-api:dev . + + +serve-dev: build-dev + docker run --rm --name tern-rest-api -e ENVIRONMENT=DEVELOPMENT --privileged -v /var/run/docker.sock:/var/run/docker.sock -v $(PWD):/opt/tern-rest-api -p 5001:80 tern-rest-api:dev diff --git a/Pipfile b/Pipfile index 4d5e929..825d21f 100644 --- a/Pipfile +++ b/Pipfile @@ -7,6 +7,8 @@ name = "pypi" flask = "*" flask-restx = "*" tern = "*" +flask-executor = "*" +gunicorn = "*" [dev-packages] black = "*" diff --git a/Pipfile.lock b/Pipfile.lock index a6a397f..c8fd3f5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6d5cd08ed1e881cbdf1b0f6d2b85741bf13aea25b22687cb9606168c27a686e3" + "sha256": "bc709a78553a8ae95dd6dc4ba0fc0d8f4a07fe1a4eb0668baf4e3fa1856d99ed" }, "pipfile-spec": 6, "requires": { @@ -57,11 +57,11 @@ }, "click": { "hashes": [ - "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3", - "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b" + "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", + "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" ], "markers": "python_version >= '3.6'", - "version": "==8.0.3" + "version": "==8.0.4" }, "debian-inspector": { "hashes": [ @@ -94,6 +94,14 @@ "index": "pypi", "version": "==2.0.3" }, + "flask-executor": { + "hashes": [ + "sha256:074885fc6d04764c86ab7f070818ea87dd08ee54767a216a8e7a00c39550bea2", + "sha256:ddb51b9e10f0fbfcff6c2386a5d92957402b7c81b7614eb1f3c77be64bcfd684" + ], + "index": "pypi", + "version": "==0.10.0" + }, "flask-restx": { "hashes": [ "sha256:63c69a61999a34f1774eaccc6fc8c7f504b1aad7d56a8ec672264e52d9ac05f4", @@ -118,6 +126,14 @@ "markers": "python_version >= '3.7'", "version": "==3.1.26" }, + "gunicorn": { + "hashes": [ + "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e", + "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8" + ], + "index": "pypi", + "version": "==20.1.0" + }, "idna": { "hashes": [ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", @@ -128,11 +144,11 @@ }, "itsdangerous": { "hashes": [ - "sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c", - "sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0" + "sha256:29285842166554469a56d427addc0843914172343784cb909695fdbe90a3e129", + "sha256:d848fcb8bc7d507c4546b448574e8a44fc4ea2ba84ebf8d783290d53e81992f5" ], - "markers": "python_version >= '3.6'", - "version": "==2.0.1" + "markers": "python_version >= '3.7'", + "version": "==2.1.0" }, "jinja2": { "hashes": [ @@ -152,78 +168,49 @@ }, "markupsafe": { "hashes": [ - "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298", - "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64", - "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b", - "sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194", - "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567", - "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff", - "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724", - "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74", - "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646", - "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35", - "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6", - "sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a", - "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6", - "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad", - "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26", - "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38", - "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac", - "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7", - "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6", - "sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047", - "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75", - "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f", - "sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b", - "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135", - "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8", - "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a", - "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a", - "sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1", - "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9", - "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864", - "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914", - "sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee", - "sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f", - "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18", - "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8", - "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2", - "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d", - "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b", - "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b", - "sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86", - "sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6", - "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f", - "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb", - "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833", - "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28", - "sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e", - "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415", - "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902", - "sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f", - "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d", - "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9", - "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d", - "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145", - "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066", - "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c", - "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1", - "sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a", - "sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207", - "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f", - "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53", - "sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd", - "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134", - "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85", - "sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9", - "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5", - "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94", - "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509", - "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51", - "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872" + "sha256:023af8c54fe63530545f70dd2a2a7eed18d07a9a77b94e8bf1e2ff7f252db9a3", + "sha256:09c86c9643cceb1d87ca08cdc30160d1b7ab49a8a21564868921959bd16441b8", + "sha256:142119fb14a1ef6d758912b25c4e803c3ff66920635c44078666fe7cc3f8f759", + "sha256:1d1fb9b2eec3c9714dd936860850300b51dbaa37404209c8d4cb66547884b7ed", + "sha256:204730fd5fe2fe3b1e9ccadb2bd18ba8712b111dcabce185af0b3b5285a7c989", + "sha256:24c3be29abb6b34052fd26fc7a8e0a49b1ee9d282e3665e8ad09a0a68faee5b3", + "sha256:290b02bab3c9e216da57c1d11d2ba73a9f73a614bbdcc027d299a60cdfabb11a", + "sha256:3028252424c72b2602a323f70fbf50aa80a5d3aa616ea6add4ba21ae9cc9da4c", + "sha256:30c653fde75a6e5eb814d2a0a89378f83d1d3f502ab710904ee585c38888816c", + "sha256:3cace1837bc84e63b3fd2dfce37f08f8c18aeb81ef5cf6bb9b51f625cb4e6cd8", + "sha256:4056f752015dfa9828dce3140dbadd543b555afb3252507348c493def166d454", + "sha256:454ffc1cbb75227d15667c09f164a0099159da0c1f3d2636aa648f12675491ad", + "sha256:598b65d74615c021423bd45c2bc5e9b59539c875a9bdb7e5f2a6b92dfcfc268d", + "sha256:599941da468f2cf22bf90a84f6e2a65524e87be2fce844f96f2dd9a6c9d1e635", + "sha256:5ddea4c352a488b5e1069069f2f501006b1a4362cb906bee9a193ef1245a7a61", + "sha256:62c0285e91414f5c8f621a17b69fc0088394ccdaa961ef469e833dbff64bd5ea", + "sha256:679cbb78914ab212c49c67ba2c7396dc599a8479de51b9a87b174700abd9ea49", + "sha256:6e104c0c2b4cd765b4e83909cde7ec61a1e313f8a75775897db321450e928cce", + "sha256:736895a020e31b428b3382a7887bfea96102c529530299f426bf2e636aacec9e", + "sha256:75bb36f134883fdbe13d8e63b8675f5f12b80bb6627f7714c7d6c5becf22719f", + "sha256:7d2f5d97fcbd004c03df8d8fe2b973fe2b14e7bfeb2cfa012eaa8759ce9a762f", + "sha256:80beaf63ddfbc64a0452b841d8036ca0611e049650e20afcb882f5d3c266d65f", + "sha256:84ad5e29bf8bab3ad70fd707d3c05524862bddc54dc040982b0dbcff36481de7", + "sha256:8da5924cb1f9064589767b0f3fc39d03e3d0fb5aa29e0cb21d43106519bd624a", + "sha256:961eb86e5be7d0973789f30ebcf6caab60b844203f4396ece27310295a6082c7", + "sha256:96de1932237abe0a13ba68b63e94113678c379dca45afa040a17b6e1ad7ed076", + "sha256:a0a0abef2ca47b33fb615b491ce31b055ef2430de52c5b3fb19a4042dbc5cadb", + "sha256:b2a5a856019d2833c56a3dcac1b80fe795c95f401818ea963594b345929dffa7", + "sha256:b8811d48078d1cf2a6863dafb896e68406c5f513048451cd2ded0473133473c7", + "sha256:c532d5ab79be0199fa2658e24a02fce8542df196e60665dd322409a03db6a52c", + "sha256:d3b64c65328cb4cd252c94f83e66e3d7acf8891e60ebf588d7b493a55a1dbf26", + "sha256:d4e702eea4a2903441f2735799d217f4ac1b55f7d8ad96ab7d4e25417cb0827c", + "sha256:d5653619b3eb5cbd35bfba3c12d575db2a74d15e0e1c08bf1db788069d410ce8", + "sha256:d66624f04de4af8bbf1c7f21cc06649c1c69a7f84109179add573ce35e46d448", + "sha256:e67ec74fada3841b8c5f4c4f197bea916025cb9aa3fe5abf7d52b655d042f956", + "sha256:e6f7f3f41faffaea6596da86ecc2389672fa949bd035251eab26dc6697451d05", + "sha256:f02cf7221d5cd915d7fa58ab64f7ee6dd0f6cddbb48683debf5d04ae9b1c2cc1", + "sha256:f0eddfcabd6936558ec020130f932d479930581171368fd728efcfb6ef0dd357", + "sha256:fabbe18087c3d33c5824cb145ffca52eccd053061df1d79d4b66dafa5ad2a5ea", + "sha256:fc3150f85e2dbcf99e65238c842d1cfe69d3e7649b19864c1cc043213d9cd730" ], - "markers": "python_version >= '3.6'", - "version": "==2.0.1" + "markers": "python_version >= '3.7'", + "version": "==2.1.0" }, "packageurl-python": { "hashes": [ @@ -513,58 +500,58 @@ }, "click": { "hashes": [ - "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3", - "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b" + "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", + "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" ], "markers": "python_version >= '3.6'", - "version": "==8.0.3" + "version": "==8.0.4" }, "coverage": { "hashes": [ - "sha256:1245ab82e8554fa88c4b2ab1e098ae051faac5af829efdcf2ce6b34dccd5567c", - "sha256:1bc6d709939ff262fd1432f03f080c5042dc6508b6e0d3d20e61dd045456a1a0", - "sha256:25e73d4c81efa8ea3785274a2f7f3bfbbeccb6fcba2a0bdd3be9223371c37554", - "sha256:276b13cc085474e482566c477c25ed66a097b44c6e77132f3304ac0b039f83eb", - "sha256:2aed4761809640f02e44e16b8b32c1a5dee5e80ea30a0ff0912158bde9c501f2", - "sha256:2dd70a167843b4b4b2630c0c56f1b586fe965b4f8ac5da05b6690344fd065c6b", - "sha256:352c68e233409c31048a3725c446a9e48bbff36e39db92774d4f2380d630d8f8", - "sha256:3f2b05757c92ad96b33dbf8e8ec8d4ccb9af6ae3c9e9bd141c7cc44d20c6bcba", - "sha256:448d7bde7ceb6c69e08474c2ddbc5b4cd13c9e4aa4a717467f716b5fc938a734", - "sha256:463e52616ea687fd323888e86bf25e864a3cc6335a043fad6bbb037dbf49bbe2", - "sha256:482fb42eea6164894ff82abbcf33d526362de5d1a7ed25af7ecbdddd28fc124f", - "sha256:56c4a409381ddd7bbff134e9756077860d4e8a583d310a6f38a2315b9ce301d0", - "sha256:56d296cbc8254a7dffdd7bcc2eb70be5a233aae7c01856d2d936f5ac4e8ac1f1", - "sha256:5e15d424b8153756b7c903bde6d4610be0c3daca3986173c18dd5c1a1625e4cd", - "sha256:618eeba986cea7f621d8607ee378ecc8c2504b98b3fdc4952b30fe3578304687", - "sha256:61d47a897c1e91f33f177c21de897267b38fbb45f2cd8e22a710bcef1df09ac1", - "sha256:621f6ea7260ea2ffdaec64fe5cb521669984f567b66f62f81445221d4754df4c", - "sha256:6a5cdc3adb4f8bb8d8f5e64c2e9e282bc12980ef055ec6da59db562ee9bdfefa", - "sha256:6c3f6158b02ac403868eea390930ae64e9a9a2a5bbfafefbb920d29258d9f2f8", - "sha256:704f89b87c4f4737da2860695a18c852b78ec7279b24eedacab10b29067d3a38", - "sha256:72128176fea72012063200b7b395ed8a57849282b207321124d7ff14e26988e8", - "sha256:78fbb2be068a13a5d99dce9e1e7d168db880870f7bc73f876152130575bd6167", - "sha256:7bff3a98f63b47464480de1b5bdd80c8fade0ba2832c9381253c9b74c4153c27", - "sha256:84f2436d6742c01136dd940ee158bfc7cf5ced3da7e4c949662b8703b5cd8145", - "sha256:9976fb0a5709988778ac9bc44f3d50fccd989987876dfd7716dee28beed0a9fa", - "sha256:9ad0a117b8dc2061ce9461ea4c1b4799e55edceb236522c5b8f958ce9ed8fa9a", - "sha256:9e3dd806f34de38d4c01416344e98eab2437ac450b3ae39c62a0ede2f8b5e4ed", - "sha256:9eb494070aa060ceba6e4bbf44c1bc5fa97bfb883a0d9b0c9049415f9e944793", - "sha256:9fde6b90889522c220dd56a670102ceef24955d994ff7af2cb786b4ba8fe11e4", - "sha256:9fff3ff052922cb99f9e52f63f985d4f7a54f6b94287463bc66b7cdf3eb41217", - "sha256:a06c358f4aed05fa1099c39decc8022261bb07dfadc127c08cfbd1391b09689e", - "sha256:a4f923b9ab265136e57cc14794a15b9dcea07a9c578609cd5dbbfff28a0d15e6", - "sha256:c5b81fb37db76ebea79aa963b76d96ff854e7662921ce742293463635a87a78d", - "sha256:d5ed164af5c9078596cfc40b078c3b337911190d3faeac830c3f1274f26b8320", - "sha256:d651fde74a4d3122e5562705824507e2f5b2d3d57557f1916c4b27635f8fbe3f", - "sha256:de73fca6fb403dd72d4da517cfc49fcf791f74eee697d3219f6be29adf5af6ce", - "sha256:e647a0be741edbb529a72644e999acb09f2ad60465f80757da183528941ff975", - "sha256:e92c7a5f7d62edff50f60a045dc9542bf939758c95b2fcd686175dd10ce0ed10", - "sha256:eeffd96882d8c06d31b65dddcf51db7c612547babc1c4c5db6a011abe9798525", - "sha256:f5a4551dfd09c3bd12fca8144d47fe7745275adf3229b7223c2f9e29a975ebda", - "sha256:fac0bcc5b7e8169bffa87f0dcc24435446d329cbc2b5486d155c2e0f3b493ae1" + "sha256:03e2a7826086b91ef345ff18742ee9fc47a6839ccd517061ef8fa1976e652ce9", + "sha256:07e6db90cd9686c767dcc593dff16c8c09f9814f5e9c51034066cad3373b914d", + "sha256:18d520c6860515a771708937d2f78f63cc47ab3b80cb78e86573b0a760161faf", + "sha256:1ebf730d2381158ecf3dfd4453fbca0613e16eaa547b4170e2450c9707665ce7", + "sha256:21b7745788866028adeb1e0eca3bf1101109e2dc58456cb49d2d9b99a8c516e6", + "sha256:26e2deacd414fc2f97dd9f7676ee3eaecd299ca751412d89f40bc01557a6b1b4", + "sha256:2c6dbb42f3ad25760010c45191e9757e7dce981cbfb90e42feef301d71540059", + "sha256:2fea046bfb455510e05be95e879f0e768d45c10c11509e20e06d8fcaa31d9e39", + "sha256:34626a7eee2a3da12af0507780bb51eb52dca0e1751fd1471d0810539cefb536", + "sha256:37d1141ad6b2466a7b53a22e08fe76994c2d35a5b6b469590424a9953155afac", + "sha256:46191097ebc381fbf89bdce207a6c107ac4ec0890d8d20f3360345ff5976155c", + "sha256:4dd8bafa458b5c7d061540f1ee9f18025a68e2d8471b3e858a9dad47c8d41903", + "sha256:4e21876082ed887baed0146fe222f861b5815455ada3b33b890f4105d806128d", + "sha256:58303469e9a272b4abdb9e302a780072c0633cdcc0165db7eec0f9e32f901e05", + "sha256:5ca5aeb4344b30d0bec47481536b8ba1181d50dbe783b0e4ad03c95dc1296684", + "sha256:68353fe7cdf91f109fc7d474461b46e7f1f14e533e911a2a2cbb8b0fc8613cf1", + "sha256:6f89d05e028d274ce4fa1a86887b071ae1755082ef94a6740238cd7a8178804f", + "sha256:7a15dc0a14008f1da3d1ebd44bdda3e357dbabdf5a0b5034d38fcde0b5c234b7", + "sha256:8bdde1177f2311ee552f47ae6e5aa7750c0e3291ca6b75f71f7ffe1f1dab3dca", + "sha256:8ce257cac556cb03be4a248d92ed36904a59a4a5ff55a994e92214cde15c5bad", + "sha256:8cf5cfcb1521dc3255d845d9dca3ff204b3229401994ef8d1984b32746bb45ca", + "sha256:8fbbdc8d55990eac1b0919ca69eb5a988a802b854488c34b8f37f3e2025fa90d", + "sha256:9548f10d8be799551eb3a9c74bbf2b4934ddb330e08a73320123c07f95cc2d92", + "sha256:96f8a1cb43ca1422f36492bebe63312d396491a9165ed3b9231e778d43a7fca4", + "sha256:9b27d894748475fa858f9597c0ee1d4829f44683f3813633aaf94b19cb5453cf", + "sha256:9baff2a45ae1f17c8078452e9e5962e518eab705e50a0aa8083733ea7d45f3a6", + "sha256:a2a8b8bcc399edb4347a5ca8b9b87e7524c0967b335fbb08a83c8421489ddee1", + "sha256:acf53bc2cf7282ab9b8ba346746afe703474004d9e566ad164c91a7a59f188a4", + "sha256:b0be84e5a6209858a1d3e8d1806c46214e867ce1b0fd32e4ea03f4bd8b2e3359", + "sha256:b31651d018b23ec463e95cf10070d0b2c548aa950a03d0b559eaa11c7e5a6fa3", + "sha256:b78e5afb39941572209f71866aa0b206c12f0109835aa0d601e41552f9b3e620", + "sha256:c76aeef1b95aff3905fb2ae2d96e319caca5b76fa41d3470b19d4e4a3a313512", + "sha256:dd035edafefee4d573140a76fdc785dc38829fe5a455c4bb12bac8c20cfc3d69", + "sha256:dd6fe30bd519694b356cbfcaca9bd5c1737cddd20778c6a581ae20dc8c04def2", + "sha256:e5f4e1edcf57ce94e5475fe09e5afa3e3145081318e5fd1a43a6b4539a97e518", + "sha256:ec6bc7fe73a938933d4178c9b23c4e0568e43e220aef9472c4f6044bfc6dd0f0", + "sha256:f1555ea6d6da108e1999b2463ea1003fe03f29213e459145e70edbaf3e004aaa", + "sha256:f5fa5803f47e095d7ad8443d28b01d48c0359484fec1b9d8606d0e3282084bc4", + "sha256:f7331dbf301b7289013175087636bbaf5b2405e57259dd2c42fdcc9fcc47325e", + "sha256:f9987b0354b06d4df0f4d3e0ec1ae76d7ce7cbca9a2f98c25041eb79eec766f1", + "sha256:fd9e830e9d8d89b20ab1e5af09b32d33e1a08ef4c4e14411e559556fd788e6b2" ], "index": "pypi", - "version": "==6.3.1" + "version": "==6.3.2" }, "distlib": { "hashes": [ @@ -575,11 +562,11 @@ }, "filelock": { "hashes": [ - "sha256:7b23620a293cf3e19924e469cb96672dc72b36c26e8f80f85668310117fcbe4e", - "sha256:d1eccb164ed020bc84edd9e45bf6cdb177f64749f6b8fe066648832d2e98726d" + "sha256:9cd540a9352e432c7246a48fe4e8712b10acb1df2ad1f30e8c070b82ae1fed85", + "sha256:f8314284bfffbdcfa0ff3d7992b023d4c628ced6feb957351d4c48d059f56bc0" ], "markers": "python_version >= '3.7'", - "version": "==3.5.1" + "version": "==3.6.0" }, "flake8": { "hashes": [ @@ -635,11 +622,11 @@ }, "platformdirs": { "hashes": [ - "sha256:30671902352e97b1eafd74ade8e4a694782bd3471685e78c32d0fdfd3aa7e7bb", - "sha256:8ec11dfba28ecc0715eb5fb0147a87b1bf325f349f3da9aab2cd6b50b96b692b" + "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d", + "sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227" ], "markers": "python_version >= '3.7'", - "version": "==2.5.0" + "version": "==2.5.1" }, "pluggy": { "hashes": [ @@ -731,11 +718,11 @@ }, "virtualenv": { "hashes": [ - "sha256:45e1d053cad4cd453181ae877c4ffc053546ae99e7dd049b9ff1d9be7491abf7", - "sha256:e0621bcbf4160e4e1030f05065c8834b4e93f4fcc223255db2a823440aca9c14" + "sha256:dd448d1ded9f14d1a4bfa6bfc0c5b96ae3be3f2d6c6c159b23ddcfd701baa021", + "sha256:e9dd1a1359d70137559034c0f5433b34caf504af2dc756367be86a5a32967134" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==20.13.1" + "version": "==20.13.3" } } } diff --git a/README.md b/README.md index e578d0e..811d049 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,16 @@ $ pipenv install -d ### Running the development Tern REST API +## As a Docker Container +```shell +$ make serve-dev +``` +Open http://localhost/ in your browser. +Changing the source code will automatically reload the server inside the +container and makes the development easier. + +## On your local machine Runing the API locally ```shell @@ -99,4 +108,4 @@ Updating the ``requirements.txt`` and ``requirements-dev.txt`` ```shell $ pipenv lock -r > requirements.txt $ pipenv lock -r -d > requirements-dev.txt -``` \ No newline at end of file +``` diff --git a/app.py b/app.py index 3043291..81ec199 100644 --- a/app.py +++ b/app.py @@ -8,7 +8,7 @@ from flask_restx import Api -from tern_api import __version__, tern_api +from tern_api import __version__, tern_app from tern_api.api.v1.common_models import api_models_namespace from tern_api.api.v1.reports import ns as report_v1 from tern_api.api.v1.version import ns as version_v1 @@ -26,20 +26,21 @@ api = Api( - tern_api, + tern_app, version=__version__.version, title="Tern REST API", description="Tern Project REST API", ) + api.add_namespace(api_models_namespace) api.add_namespace(version_v1, path="/api/v1/version") api.add_namespace(report_v1, path="/api/v1/report") def export_swagger_json(filepath): - tern_api.config["SERVER_NAME"] = "localhost" - with tern_api.app_context().__enter__(): + tern_app.config["SERVER_NAME"] = "localhost" + with tern_app.app_context().__enter__(): with open(filepath, "w") as f: swagger_json = json.dumps(api.__schema__, indent=4) f.write(swagger_json) diff --git a/docker_start.sh b/docker_start.sh new file mode 100644 index 0000000..dfa4e06 --- /dev/null +++ b/docker_start.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [[ ${ENVIRONMENT^^} == "DEVELOPMENT" ]]; then + flask run --reload --debugger -h 0.0.0.0 -p 80 +else + gunicorn --workers=1 -b 0.0.0.0:80 app:tern_api +fi \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 6e786aa..06b400e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,32 +15,34 @@ black==22.1.0 certifi==2021.10.8 chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' charset-normalizer==2.0.10; python_version >= '3' -click==8.0.3; python_version >= '3.6' -coverage==6.3.1 +click==8.0.4; python_version >= '3.6' +coverage==6.3.2 debian-inspector==30.0.0; python_version >= '3.6' and python_version < '4' distlib==0.3.4 docker==5.0.3; python_version >= '3.6' dockerfile-parse==1.2.0 -filelock==3.5.1; python_version >= '3.7' +filelock==3.6.0; python_version >= '3.7' flake8==4.0.1 +flask-executor==0.10.0 flask-restx==0.5.1 flask==2.0.3 gitdb==4.0.9; python_version >= '3.6' gitpython==3.1.26; python_version >= '3.7' +gunicorn==20.1.0 idna==3.3; python_version >= '3' iniconfig==1.1.1 isort==5.10.1 -itsdangerous==2.0.1; python_version >= '3.6' +itsdangerous==2.1.0; python_version >= '3.7' jinja2==3.0.3; python_version >= '3.6' jsonschema==4.4.0; python_version >= '3.7' -markupsafe==2.0.1; python_version >= '3.6' +markupsafe==2.1.0; python_version >= '3.7' mccabe==0.6.1 mypy-extensions==0.4.3 packageurl-python==0.9.6; python_version >= '3.6' packaging==21.3; python_version >= '3.6' pathspec==0.9.0 pbr==5.8.0; python_version >= '2.6' -platformdirs==2.5.0; python_version >= '3.7' +platformdirs==2.5.1; python_version >= '3.7' pluggy==1.0.0; python_version >= '3.6' prettytable==3.0.0; python_version >= '3.7' py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' @@ -62,7 +64,7 @@ tomli==2.0.1; python_version >= '3.7' tox==3.24.5 typing-extensions==4.1.1; python_version < '3.10' urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4' -virtualenv==20.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +virtualenv==20.13.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' wcwidth==0.2.5 websocket-client==1.2.3; python_version >= '3.6' werkzeug==2.0.3; python_version >= '3.6' diff --git a/requirements.txt b/requirements.txt index eaccc09..49250d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,19 +11,21 @@ attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, certifi==2021.10.8 chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' charset-normalizer==2.0.10; python_version >= '3' -click==8.0.3; python_version >= '3.6' +click==8.0.4; python_version >= '3.6' debian-inspector==30.0.0; python_version >= '3.6' and python_version < '4' docker==5.0.3; python_version >= '3.6' dockerfile-parse==1.2.0 +flask-executor==0.10.0 flask-restx==0.5.1 flask==2.0.3 gitdb==4.0.9; python_version >= '3.6' gitpython==3.1.26; python_version >= '3.7' +gunicorn==20.1.0 idna==3.3; python_version >= '3' -itsdangerous==2.0.1; python_version >= '3.6' +itsdangerous==2.1.0; python_version >= '3.7' jinja2==3.0.3; python_version >= '3.6' jsonschema==4.4.0; python_version >= '3.7' -markupsafe==2.0.1; python_version >= '3.6' +markupsafe==2.1.0; python_version >= '3.7' packageurl-python==0.9.6; python_version >= '3.6' pbr==5.8.0; python_version >= '2.6' prettytable==3.0.0; python_version >= '3.7' diff --git a/tern_api/__init__.py b/tern_api/__init__.py index 6d1bd3b..5b3cc1e 100644 --- a/tern_api/__init__.py +++ b/tern_api/__init__.py @@ -1,7 +1,15 @@ +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright (c) 2022 VMware, Inc. All Rights Reserved. # SPDX-License-Identifier: BSD-2-Clause +import os + from flask import Flask +from flask_executor import Executor + +tern_app = Flask(__name__) +tern_app.config["TERN_API_CACHE_DIR"] = os.getenv("TERN_API_CACHE_DIR") +tern_app.config["TERN_DEFAULT_REGISTRY"] = os.getenv("TERN_DEFAULT_REGISTRY") -tern_api = Flask(__name__) +tern_tasks = Executor(tern_app) diff --git a/tern_api/api/v1/common_models.py b/tern_api/api/v1/common_models.py index 1170558..9adbbfc 100644 --- a/tern_api/api/v1/common_models.py +++ b/tern_api/api/v1/common_models.py @@ -18,21 +18,6 @@ }, ) -async_response_model = api_models_namespace.model( - "async_response_model", - { - "message": fields.String( - description="Status message", - requored=True, - example="Request submitted.", - ), - "id": fields.String( - description="Unique Identification for request", - required=True, - example="19f035a711644eab84ef5a38ceb5572e", - ), - }, -) image_report_data = api_models_namespace.model( "image_report_data", @@ -52,6 +37,12 @@ example="3.0", required=True, ), + "cache": fields.String( + description="Use cache if available?", + exampple=True, + required=True, + default=True, + ), }, ) image_report_model = api_models_namespace.model( @@ -60,3 +51,24 @@ report_model = api_models_namespace.model( "report_mode", {"images": fields.List(fields.Nested(image_report_model))} ) + +async_response_model = api_models_namespace.model( + "async_response_model", + { + "message": fields.String( + description="Status message", + requored=True, + example="Request submitted.", + ), + "id": fields.String( + description="Unique Identification for request", + required=True, + example="19f035a711644eab84ef5a38ceb5572e", + ), + "cache": fields.Boolean( + description="Request uses cache?", + required=True, + example=True, + ), + }, +) diff --git a/tern_api/api/v1/reports.py b/tern_api/api/v1/reports.py index a0f82bb..d70f614 100644 --- a/tern_api/api/v1/reports.py +++ b/tern_api/api/v1/reports.py @@ -3,8 +3,10 @@ # # Copyright (c) 2022 VMware, Inc. All Rights Reserved. # SPDX-License-Identifier: BSD-2-Clause +from flask import request from flask_restx import Namespace, Resource, fields +from tern_api import reports, tern_app from tern_api.api.v1.common_models import ( async_response_model, error_model, @@ -22,8 +24,8 @@ class Report(Resource): "registry": fields.String( description="Registry Server", required=False, - default="https://registry.hub.docker.com", - example="http://registry.example.com", + default=tern_app.config["TERN_DEFAULT_REGISTRY"], + example=tern_app.config["TERN_DEFAULT_REGISTRY"], ), "image": fields.String( description="Image name", @@ -35,6 +37,11 @@ class Report(Resource): required=True, example="3.0", ), + "cache": fields.Boolean( + description="Use cache data if available?", + required=True, + example=True, + ), }, ) report_response_request = ns.model( @@ -52,6 +59,9 @@ def post(self): **Note**: This request will be processed assynchronous. """ + payload = request.json + response = reports.request(payload) + return response.to_response() @ns.route("/status") @@ -72,8 +82,8 @@ class ReportStatus(Resource): "status": fields.String( description="Status of request", required=True, - example="DONE", - enum=["UNKNOWN", "FAILED", "DONE"], + example="FINISH", + enum=["UNKNOWN", "FINISH", "RUNNING", "FAIL"], ), "result": fields.Nested(report_model), }, @@ -90,3 +100,7 @@ class ReportStatus(Resource): @ns.expect(report_status_parameters) def post(self): """Request Tern BoM report status/result""" + + payload = request.json + response = reports.status(payload.get("id")) + return response.to_response() diff --git a/tern_api/constants.py b/tern_api/constants.py new file mode 100644 index 0000000..c2edf76 --- /dev/null +++ b/tern_api/constants.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause + +from enum import Enum + + +class task_status(Enum): + PENDING = "PENDING" + RECEIVED = "RECEIVED" + RUNNING = "RUNNING" + SUCCESS = "SUCCESS" + FAILURE = "FAILURE" diff --git a/tern_api/reports.py b/tern_api/reports.py index a4bb83b..9cfa4b9 100644 --- a/tern_api/reports.py +++ b/tern_api/reports.py @@ -3,3 +3,184 @@ # # Copyright (c) 2022 VMware, Inc. All Rights Reserved. # SPDX-License-Identifier: BSD-2-Clause + +import json +import logging +import os +import subprocess +import sys +from dataclasses import asdict, dataclass, field +from typing import Any, Dict, Optional +from uuid import uuid4 + +from tern_api import tern_app, tern_tasks +from tern_api.constants import task_status +from tern_api.utils import TernAPIResponse + + +class TernError(Exception): + """Failure on the Tern execution.""" + + +@dataclass +class DataResponse: + id: str + cache: bool = field(default=True) + message: str = field(default="") + status: str = field(default=task_status.PENDING.value) + report: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self): + """Returns the DataResponse as a dictionary.""" + return asdict(self) + + +def tern(command: list) -> Dict[str, Any]: + """ + Runs the tern CLI using the ``subprocess``. + + Args: + command: Command line as a list (required by subprocess). + + Returns: + report as a dictionary (JSON). + + Raises: + TernError: failure during running the command from tern CLI. + """ + logging.debug(command) + + tern_cmd = subprocess.run(command, capture_output=True) + if tern_cmd.stdout: + json_report = json.loads(tern_cmd.stdout) + return json_report + else: + if tern_cmd.stderr: + logging.debug(tern_cmd.stderr) + + # Is important to return a specific error if the error comes from + # tern CLI, it means the task FINISH, for example, invalid image. + logging.info(tern_cmd.stderr) + raise TernError(tern_cmd.stderr) + + +@tern_tasks.job +def tern_report( + command: list, cache: bool, cache_file: Optional[str] +) -> Dict[str, Any]: + """ + Tern Report as a task (background) and manages the cache + + Args: + command: Command line as a list (required by subprocess). + cache: Use caching + cache_file: If cache, inform the cache_file + + Return: + Report as Dictionary + """ + + if cache: + try: + with open(cache_file, "r") as f: + report = json.load(f) + return report + + except FileNotFoundError: + report = tern(command) + with open(cache_file, "w") as f: + json.dump(report, f, indent=2) + + else: + report = tern(command) + + return report + + +def request(payload: dict) -> TernAPIResponse: + """ + Get the Payload from API and prepare to request the report. + The request will be handled in the background as tern tasks. + + Args: + payload: API Payload + + Return: + API Response + """ + TERN_API_CACHE_DIR = tern_app.config["TERN_API_CACHE_DIR"] + TERN_DEFAULT_REGISTRY = tern_app.config["TERN_DEFAULT_REGISTRY"] + + task_id = uuid4().hex + registry = payload.get( + "registry", + ) + image = payload.get("image") + tag = payload.get("tag") + cache = payload.get("cache", True) + cache_file_dir = os.path.join(TERN_API_CACHE_DIR, registry, image) + cache_file = os.path.join(cache_file_dir, f"{tag}.json") + + os.makedirs(cache_file_dir, exist_ok=True) + + report_request_response = DataResponse(id=task_id, cache=cache) + if registry != TERN_DEFAULT_REGISTRY: + registry_image_tag = f"{registry}/{image}:{tag}" + else: + registry_image_tag = f"{image}:{tag}" + + command = ["tern", "report", "-i", registry_image_tag, "-f", "json"] + logging.info(command) + + tern_report.submit_stored( + task_id, command=command, cache=cache, cache_file=cache_file + ) + + report_request_response.message = "Request submitted." + return TernAPIResponse(report_request_response.to_dict()) + + +def status(task_id: str) -> TernAPIResponse: + """ + Request to the status/result from the tern tasks. + + The tern tasks can have basically the following status: + - UNKNOWN: Not known (yet) by the task manager (initial status). + - RUNNING: Task is running by the task manager. + - FINISH : Task has fineshed in the task manager. + - FAIL : Task has failed before finished. + + Args: + task_id: the unique task ID + """ + data_response = DataResponse(id=task_id, status=task_status.UNKNOWN.value) + + try: + if not tern_tasks.futures.done(task_id): + status = tern_tasks.futures._state(task_id) + if status: + data_response.status = status + + return TernAPIResponse(data_response.to_dict()) + + report = tern_tasks.futures.pop(task_id) + data_response.report = report.result() + data_response.status = task_status.SUCCESS.value + + # It means the task was finished by the task manager (SUCCESS), but the + # report has no data and the error is given to the API user. + except TernError as e: + data_response.status = task_status.SUCCESS.value + response = TernAPIResponse(data_response.to_dict()) + response.errors = {"message": str(e)} + return response + + # Any kind of not expected failure means that the task didn't finished as + # expected (FAIL). + except: # noqa + data_response.status = task_status.FAILURE.value + data_response.message = ( + f"Task couln't finish due: {str(sys.exc_info())}" + ) + + return TernAPIResponse(data_response.to_dict()) From a6452229cffbc630e878eef65f0543f965915408 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Thu, 10 Mar 2022 09:20:00 +0100 Subject: [PATCH 2/8] Document the cache logic in tern_report The logic is easy, but add some quick reference. Signed-off-by: Kairo de Araujo --- tern_api/reports.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tern_api/reports.py b/tern_api/reports.py index 9cfa4b9..8aadef3 100644 --- a/tern_api/reports.py +++ b/tern_api/reports.py @@ -81,16 +81,17 @@ def tern_report( """ if cache: + # If a API user is using the cache, first try to load the cached + # instead doing a new call to the tern. try: with open(cache_file, "r") as f: report = json.load(f) return report - except FileNotFoundError: + # call the tern and dump it to the cache report = tern(command) with open(cache_file, "w") as f: json.dump(report, f, indent=2) - else: report = tern(command) From cb2d554718087338fc35828052ca9c702e33c0a2 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Thu, 10 Mar 2022 09:38:47 +0100 Subject: [PATCH 3/8] Fix invalid task_status getting status reports Fix the invalid UNKNOWN status. Correct initial status is PENDING. This issue causes HTTP 500 when requested the status endpoint. Signed-off-by: Kairo de Araujo --- tern_api/reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tern_api/reports.py b/tern_api/reports.py index 8aadef3..b9ca665 100644 --- a/tern_api/reports.py +++ b/tern_api/reports.py @@ -154,7 +154,7 @@ def status(task_id: str) -> TernAPIResponse: Args: task_id: the unique task ID """ - data_response = DataResponse(id=task_id, status=task_status.UNKNOWN.value) + data_response = DataResponse(id=task_id, status=task_status.PENDING.value) try: if not tern_tasks.futures.done(task_id): From 9de90d0969dd51be654644cf5878b7984d10915c Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Thu, 10 Mar 2022 09:41:32 +0100 Subject: [PATCH 4/8] Add License header to the Makefile Missing License header in the Makefile was added. Signed-off-by: Kairo de Araujo --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a2d939b..662607b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,11 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause + build-dev: docker build -t tern-rest-api:dev . - serve-dev: build-dev docker run --rm --name tern-rest-api -e ENVIRONMENT=DEVELOPMENT --privileged -v /var/run/docker.sock:/var/run/docker.sock -v $(PWD):/opt/tern-rest-api -p 5001:80 tern-rest-api:dev From eb8e0b304adcae38c0843a2cdbdd6e7d27d7f4e3 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Tue, 15 Mar 2022 07:51:45 +0100 Subject: [PATCH 5/8] Included the unit tests to run in tox This commit adds to tox.ini the unit tests. It guarantee that the tests will run in the CI if new changes or tests are added. Signed-off-by: Kairo de Araujo --- tests/conftest.py | 6 +++--- tox.ini | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d044db9..8f53af9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,15 +6,15 @@ import pytest from werkzeug.test import TestResponse -from app import tern_api +from app import tern_app from tests.utils import RequestDataTest @pytest.fixture def api_request(): def _api_request(request_data: RequestDataTest) -> TestResponse: - with tern_api.test_client() as api_client: - with tern_api.app_context(): + with tern_app.test_client() as api_client: + with tern_app.app_context(): if request_data.method.lower() == "get": response = api_client.get( request_data.endpoint, json=request_data.payload diff --git a/tox.ini b/tox.ini index 3ba0654..9ee0db3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py39,py310,lint +envlist = py39,lint,test [flake8] exclude = ownca/__init__.py,venv,.venv,settings.py,.git,.tox,dist,docs,*lib/python*,*egg,build,tools From 53a9ef03d0fd1345827009eeb520129130fff5c6 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Fri, 18 Mar 2022 16:51:08 +0100 Subject: [PATCH 6/8] Add tests for tern-rest-api reports implementation This commit adds all the tern-rest-api reports implementation tests covering 100% Adjusts the repository to use Docker. Signed-off-by: Kairo de Araujo --- .coveragerc | 2 + .github/workflows/ci.yml | 2 +- Dockerfile | 9 +- Makefile | 14 +- README.md | 12 +- app.py | 4 +- docker-compose.yml | 21 ++ docker_start.sh | 5 +- tern_api/__init__.py | 1 + tern_api/api/v1/reports.py | 32 ++- tern_api/reports.py | 29 ++- tern_api/utils.py | 9 - tests/conftest.py | 26 ++- tests/tern_api/test_reports.py | 368 ++++++++++++++++++++++++++++++ tests/tern_api/test_utils.py | 55 +++++ tests/tern_api/v1/test_reports.py | 107 +++++++++ tests/tern_api/v1/test_version.py | 2 +- tox.ini | 2 +- 18 files changed, 654 insertions(+), 46 deletions(-) create mode 100644 .coveragerc create mode 100644 docker-compose.yml mode change 100644 => 100755 docker_start.sh create mode 100644 tests/tern_api/test_reports.py create mode 100644 tests/tern_api/test_utils.py create mode 100644 tests/tern_api/v1/test_reports.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..3d1e5c5 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +omit = *tests*,app.py \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb5cace..c99a05a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,4 +22,4 @@ jobs: run: pip install tox tox-gh-actions - name: Run Python tests - run: tox + run: tox -r diff --git a/Dockerfile b/Dockerfile index 24db03f..f44eb7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,14 +16,15 @@ RUN echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.lis skopeo \ && rm -rf /var/lib/apt/lists/* +COPY requirements.txt requirements.txt +RUN pip install --no-cache -r ./requirements.txt + RUN mkdir /opt/tern-rest-api ADD . /opt/tern-rest-api WORKDIR /opt/tern-rest-api -RUN pip install --no-cache -r ./requirements.txt -ENV TERN_API_CACHE_DIR=/var/opt/tern-rest-api/cached +ENV TERN_API_CACHE_DIR=/var/opt/tern-rest-api/cache ENV TERN_DEFAULT_REGISTRY="registry.hub.docker.com" -ENV FLASK_APP=/opt/tern-rest-api/app.py -CMD ["bash", "docker_start.sh"] \ No newline at end of file +ENTRYPOINT [ "bash", "docker_start.sh" ] \ No newline at end of file diff --git a/Makefile b/Makefile index 662607b..a77648b 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,17 @@ # SPDX-License-Identifier: BSD-2-Clause build-dev: - docker build -t tern-rest-api:dev . + docker-compose build --force-rm tern-rest-api serve-dev: build-dev - docker run --rm --name tern-rest-api -e ENVIRONMENT=DEVELOPMENT --privileged -v /var/run/docker.sock:/var/run/docker.sock -v $(PWD):/opt/tern-rest-api -p 5001:80 tern-rest-api:dev + docker-compose up --remove-orphans + +tests: build-dev + docker-compose run --rm --volume=$(PWD):/opt/tern-rest-api --entrypoint="/bin/sh" tern-rest-api -c 'pip install tox && tox' + +stop: + docker-compose down -v + +update-requirements: + pipenv lock -r > requirements.txt + pipenv lock -r -d > requirements-dev.txt \ No newline at end of file diff --git a/README.md b/README.md index 811d049..1167754 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ Open http://localhost/ in your browser. Changing the source code will automatically reload the server inside the container and makes the development easier. +You can stop the sever using ``Ctrl+C`` and running ``make stop`` + ## On your local machine Runing the API locally @@ -81,7 +83,12 @@ Open http://localhost:5000/ in your browser. We use [Tox](https://tox.wiki/en/latest/) to manage running the tests. -Running tests +## As a Docker Container +```shell +$ make tests +``` + +## On your local machine ```shell $ tox ``` @@ -106,6 +113,5 @@ $ pipenv update Updating the ``requirements.txt`` and ``requirements-dev.txt`` ```shell -$ pipenv lock -r > requirements.txt -$ pipenv lock -r -d > requirements-dev.txt +$ make update-requirements ``` diff --git a/app.py b/app.py index 81ec199..7a18284 100644 --- a/app.py +++ b/app.py @@ -10,7 +10,7 @@ from tern_api import __version__, tern_app from tern_api.api.v1.common_models import api_models_namespace -from tern_api.api.v1.reports import ns as report_v1 +from tern_api.api.v1.reports import ns as reports_v1 from tern_api.api.v1.version import ns as version_v1 logging.basicConfig( @@ -35,7 +35,7 @@ api.add_namespace(api_models_namespace) api.add_namespace(version_v1, path="/api/v1/version") -api.add_namespace(report_v1, path="/api/v1/report") +api.add_namespace(reports_v1, path="/api/v1/reports") def export_swagger_json(filepath): diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..41f051a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3' + +volumes: + tern-cache: + +services: + tern-rest-api: + build: + context: . + args: + DEVEL: "yes" + command: "bash bash_start.sh" + environment: + - ENVIRONMENT=development + - TERN_DEFAULT_REGISTRY=registry.hub.docker.com + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - tern-cache:/var/opt/tern-rest-api/cache:z + - ./tern_api:/opt/tern-rest-api/tern_api:z + ports: + - "5001:80" diff --git a/docker_start.sh b/docker_start.sh old mode 100644 new mode 100755 index dfa4e06..2204d26 --- a/docker_start.sh +++ b/docker_start.sh @@ -1,7 +1,8 @@ #!/bin/bash if [[ ${ENVIRONMENT^^} == "DEVELOPMENT" ]]; then - flask run --reload --debugger -h 0.0.0.0 -p 80 + echo "Starting tern-rest-api in development mode" + gunicorn --reload -b 0.0.0.0:80 app:tern_app else - gunicorn --workers=1 -b 0.0.0.0:80 app:tern_api + gunicorn -b 0.0.0.0:80 app:tern_app fi \ No newline at end of file diff --git a/tern_api/__init__.py b/tern_api/__init__.py index 5b3cc1e..5d378d7 100644 --- a/tern_api/__init__.py +++ b/tern_api/__init__.py @@ -13,3 +13,4 @@ tern_app.config["TERN_DEFAULT_REGISTRY"] = os.getenv("TERN_DEFAULT_REGISTRY") tern_tasks = Executor(tern_app) +tern_app.config["TERN_TASKS"] = tern_tasks diff --git a/tern_api/api/v1/reports.py b/tern_api/api/v1/reports.py index d70f614..e2161f4 100644 --- a/tern_api/api/v1/reports.py +++ b/tern_api/api/v1/reports.py @@ -6,12 +6,13 @@ from flask import request from flask_restx import Namespace, Resource, fields -from tern_api import reports, tern_app +from tern_api import constants, tern_app from tern_api.api.v1.common_models import ( async_response_model, error_model, report_model, ) +from tern_api.reports import status, submit ns = Namespace("/reports", description="Tern Bill of Materials Report") @@ -53,14 +54,14 @@ class Report(Resource): ) @ns.response(200, "OK", report_response_request) - @ns.expect(report_parameters) + @ns.expect(report_parameters, validate=True) def post(self): """Tern BoM report **Note**: This request will be processed assynchronous. """ payload = request.json - response = reports.request(payload) + response = submit(payload) return response.to_response() @@ -79,13 +80,28 @@ class ReportStatus(Resource): data_status_response = ns.model( "data_status_response", { + "cache": fields.Boolean( + description="Requested using cache?", + required=True, + example=True, + ), + "id": fields.String( + description="Unique Identification for request", + required=False, + example="19f035a711644eab84ef5a38ceb5572e", + ), + "message": fields.String( + description="Message", + required=False, + exampple="Request is running", + ), + "report": fields.Nested(report_model), "status": fields.String( description="Status of request", required=True, - example="FINISH", - enum=["UNKNOWN", "FINISH", "RUNNING", "FAIL"], + example=constants.task_status.SUCCESS.value, + enum=[s.value for s in constants.task_status], ), - "result": fields.Nested(report_model), }, ) report_status_response = ns.model( @@ -97,10 +113,10 @@ class ReportStatus(Resource): ) @ns.response(200, "OK", report_status_response) - @ns.expect(report_status_parameters) + @ns.expect(report_status_parameters, validate=True) def post(self): """Request Tern BoM report status/result""" payload = request.json - response = reports.status(payload.get("id")) + response = status(payload.get("id")) return response.to_response() diff --git a/tern_api/reports.py b/tern_api/reports.py index b9ca665..b166490 100644 --- a/tern_api/reports.py +++ b/tern_api/reports.py @@ -50,6 +50,9 @@ def tern(command: list) -> Dict[str, Any]: """ logging.debug(command) + if type(command) != list: + raise TypeError("command must be a list") + tern_cmd = subprocess.run(command, capture_output=True) if tern_cmd.stdout: json_report = json.loads(tern_cmd.stdout) @@ -61,7 +64,7 @@ def tern(command: list) -> Dict[str, Any]: # Is important to return a specific error if the error comes from # tern CLI, it means the task FINISH, for example, invalid image. logging.info(tern_cmd.stderr) - raise TernError(tern_cmd.stderr) + raise TernError(tern_cmd.stderr.decode()) @tern_tasks.job @@ -79,7 +82,6 @@ def tern_report( Return: Report as Dictionary """ - if cache: # If a API user is using the cache, first try to load the cached # instead doing a new call to the tern. @@ -88,6 +90,7 @@ def tern_report( report = json.load(f) return report except FileNotFoundError: + logging.debug(f"Cache file not found: {cache_file}") # call the tern and dump it to the cache report = tern(command) with open(cache_file, "w") as f: @@ -98,7 +101,7 @@ def tern_report( return report -def request(payload: dict) -> TernAPIResponse: +def submit(payload: dict) -> TernAPIResponse: """ Get the Payload from API and prepare to request the report. The request will be handled in the background as tern tasks. @@ -111,7 +114,6 @@ def request(payload: dict) -> TernAPIResponse: """ TERN_API_CACHE_DIR = tern_app.config["TERN_API_CACHE_DIR"] TERN_DEFAULT_REGISTRY = tern_app.config["TERN_DEFAULT_REGISTRY"] - task_id = uuid4().hex registry = payload.get( "registry", @@ -131,7 +133,7 @@ def request(payload: dict) -> TernAPIResponse: registry_image_tag = f"{image}:{tag}" command = ["tern", "report", "-i", registry_image_tag, "-f", "json"] - logging.info(command) + logging.debug(command) tern_report.submit_stored( task_id, command=command, cache=cache, cache_file=cache_file @@ -146,10 +148,10 @@ def status(task_id: str) -> TernAPIResponse: Request to the status/result from the tern tasks. The tern tasks can have basically the following status: - - UNKNOWN: Not known (yet) by the task manager (initial status). + - PENDING: Not known (yet) by the task manager (initial status). - RUNNING: Task is running by the task manager. - - FINISH : Task has fineshed in the task manager. - - FAIL : Task has failed before finished. + - SUCCESS: Task has fineshed in the task manager. + - FAILURE: Task has failed before finished. Args: task_id: the unique task ID @@ -159,9 +161,16 @@ def status(task_id: str) -> TernAPIResponse: try: if not tern_tasks.futures.done(task_id): status = tern_tasks.futures._state(task_id) - if status: + if status == task_status.RUNNING.value: data_response.status = status + else: + # if the task state is not done, then the task is still running + # we can't know the status, so we return the status as pending + # and remove the task from the task manager + data_response.status = task_status.PENDING.value + tern_tasks.futures.pop(task_id) + return TernAPIResponse(data_response.to_dict()) report = tern_tasks.futures.pop(task_id) @@ -181,7 +190,7 @@ def status(task_id: str) -> TernAPIResponse: except: # noqa data_response.status = task_status.FAILURE.value data_response.message = ( - f"Task couln't finish due: {str(sys.exc_info())}" + f"Task could not finish due: {str(sys.exc_info())}" ) return TernAPIResponse(data_response.to_dict()) diff --git a/tern_api/utils.py b/tern_api/utils.py index 9af4e6d..3dac5f1 100644 --- a/tern_api/utils.py +++ b/tern_api/utils.py @@ -6,7 +6,6 @@ from dataclasses import dataclass, field from typing import Any, Dict -from attr import asdict from flask import jsonify from flask.wrappers import Response @@ -17,14 +16,6 @@ class TernAPIResponse: status_code: int = 200 errors: Dict[str, Any] = field(default_factory=dict) - def to_dict(self) -> Dict[str, Any]: - """Converts the dataclass data to a dictionary. - - :return: Response data as a dictionary - :rtype: ``dict`` - """ - return asdict(self) - def to_response(self) -> Response: """Converts the dataclass data to a Flask jsonified format, building a consistent response format for the requests to the API. diff --git a/tests/conftest.py b/tests/conftest.py index 8f53af9..79e600b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,11 +10,21 @@ from tests.utils import RequestDataTest +@pytest.fixture(scope="module") +def test_tern_app(): + # Configuration + tern_app.config["TERN_API_CACHE_DIR"] = "FakeCacheDir" + tern_app.config["TERN_DEFAULT_REGISTRY"] = "registry_fake_tests" + + return tern_app + + @pytest.fixture -def api_request(): +def api_request(test_tern_app): def _api_request(request_data: RequestDataTest) -> TestResponse: - with tern_app.test_client() as api_client: - with tern_app.app_context(): + with test_tern_app.test_client() as api_client: + with test_tern_app.app_context(): + if request_data.method.lower() == "get": response = api_client.get( request_data.endpoint, json=request_data.payload @@ -41,3 +51,13 @@ def _api_request(request_data: RequestDataTest) -> TestResponse: return response return _api_request + + +@pytest.fixture +def fake_id(): + class FakeID: + @property + def hex(self): + return "fake-id" + + return FakeID diff --git a/tests/tern_api/test_reports.py b/tests/tern_api/test_reports.py new file mode 100644 index 0000000..b578173 --- /dev/null +++ b/tests/tern_api/test_reports.py @@ -0,0 +1,368 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause +import json +import time +from dataclasses import dataclass +from unittest import mock + +import pytest + +from tern_api import reports +from tern_api.constants import task_status +from tern_api.utils import TernAPIResponse + + +@dataclass +class FakeSubprocessResult: + stdout_: bytes + stderr_: bytes + returncode_: int + + @property + def stdout(self): + return self.stdout_ + + @property + def stderr(self): + return self.stderr_ + + @property + def returncode(self): + return self.returncode_ + + +TERN_REPORT = b'{"images": [{"image": "photon", "tag": "3.0"}]}' + + +class TestReports: + @mock.patch("tern_api.reports.subprocess.run") + def test_tern(self, mock_subprocess_run): + """Test tern report""" + + fake_subprocess_result = FakeSubprocessResult( + stdout_=b'{"images": [{"image": "photon", "tag": "3.0"}]}', + stderr_=b"", + returncode_=0, + ) + + mock_subprocess_run.return_value = fake_subprocess_result + test_response = reports.tern( + ["tern", "report", "-i", "phothon:3.0", "-f", "json"] + ) + + assert test_response == json.loads(fake_subprocess_result.stdout_) + + @mock.patch("tern_api.reports.subprocess.run") + def test_tern_command_failed(self, mock_subprocess_run): + """Test tern report with a failure running tern command""" + + fake_subprocess_result = FakeSubprocessResult( + stdout_=b"", + stderr_=b"Failed to run tern command", + returncode_=1, + ) + + mock_subprocess_run.return_value = fake_subprocess_result + + with pytest.raises(reports.TernError) as e: + reports.tern(["tern", "report", "-i", "phothon:3.0", "-f", "json"]) + + assert e.value.args[0] == "Failed to run tern command" + + def test_tern_with_invalid_command_type(self): + """Test tern report with invalid command type for subprocess""" + + with pytest.raises(TypeError) as e: + reports.tern("tern report -i phothon:3.0 -f json") + + assert e.value.args[0] == "command must be a list" + + @mock.patch("builtins.open", mock.mock_open(read_data=TERN_REPORT)) + def test_tern_report_from_cache(self, test_tern_app, fake_id): + """Test tern report using cache and cache is available""" + command = ["tern", "report", "-i", "phothon:3.0", "-f", "json"] + + task_id = fake_id().hex + + with test_tern_app.test_request_context(): + future = reports.tern_report.submit_stored( + task_id, + command=command, + cache=True, + cache_file="fake_cache_file", + ) + + time.sleep(1) + assert future._state == "FINISHED" # futures state + assert future.done() is True + assert future.result() == json.loads(TERN_REPORT) + + @mock.patch("builtins.open") + @mock.patch("tern_api.reports.tern") + @mock.patch("tern_api.reports.json") + def test_tern_report_cache_not_available( + self, mock_json, mock_tern, mock_open, test_tern_app, fake_id + ): + """Test tern report using cache and cache is not available""" + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + command = ["tern", "report", "-i", "phothon:3.0", "-f", "json"] + task_id = fake_id().hex + + mock_open.side_effect = [ + FileNotFoundError, + mock.mock_open().return_value, + ] + mock_tern.return_value = json.loads(TERN_REPORT) + mock_json.dump.return_value = None + with test_tern_app.test_request_context(): + future = reports.tern_report.submit_stored( + task_id, + command=command, + cache=True, + cache_file="fake_cache_file", + ) + + time.sleep(1) + assert future._state == "FINISHED" # futures state + assert future.done() is True + assert future.result() == json.loads(TERN_REPORT) + + @mock.patch("tern_api.reports.tern") + def test_tern_report_no_cache(self, mock_tern, test_tern_app, fake_id): + """Test tern report without using cache (cache=False)""" + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + command = ["tern", "report", "-i", "phothon:3.0", "-f", "json"] + task_id = fake_id().hex + + mock_tern.return_value = json.loads(TERN_REPORT) + with test_tern_app.test_request_context(): + future = reports.tern_report.submit_stored( + task_id, + command=command, + cache=False, + cache_file="fake_cache_file", + ) + + time.sleep(1) + assert future._state == "FINISHED" # futures state + assert future.done() is True + assert future.result() == json.loads(TERN_REPORT) + + @mock.patch("tern_api.reports.os") + @mock.patch("tern_api.reports.uuid4") + @mock.patch("tern_api.reports.tern_report") + def test_request( + self, mock_os, mock_uuid4, mock_tern_report, test_tern_app, fake_id + ): + """Test report request""" + + expected_response = TernAPIResponse( + data={ + "message": "Request submitted.", + "id": "fake-id", + "cache": True, + "status": task_status.PENDING.value, + "report": {}, + } + ) + + payload = { + "registry": "registry_fake_tests", + "image": "photon", + "tag": "3.0", + "cache": True, + } + + mock_os.return_value = None + mock_uuid4.return_value.hex = fake_id().hex + mock_tern_report.submit_stored.return_value = None + + with test_tern_app.app_context(): + test_response = reports.submit(payload=payload) + + assert test_response == expected_response + assert test_response.status_code == 200 + + @mock.patch("tern_api.reports.os") + @mock.patch("tern_api.reports.uuid4") + @mock.patch("tern_api.reports.tern_report") + def test_request_using_different_registry( + self, mock_os, mock_uuid4, mock_tern_report, test_tern_app, fake_id + ): + """Test report request using non-default registry""" + + expected_response = TernAPIResponse( + data={ + "message": "Request submitted.", + "id": "fake-id", + "cache": True, + "status": task_status.PENDING.value, + "report": {}, + } + ) + + payload = { + "registry": "another_registry", + "image": "photon", + "tag": "3.0", + "cache": True, + } + + mock_os.return_value = None + mock_uuid4.return_value.hex = fake_id().hex + mock_tern_report.submit_stored.return_value = None + + with test_tern_app.app_context(): + test_response = reports.submit(payload=payload) + + assert test_response == expected_response + + def test_status(self, test_tern_app, fake_id): + """Test report status + + The very basic status is a non existent task id (PENDING). + """ + + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + task_id = fake_id().hex + expected_response = TernAPIResponse( + reports.DataResponse(id=task_id).to_dict() + ) + + test_result = reports.status(task_id) + assert test_result == expected_response + + @mock.patch("tern_api.reports.tern_tasks") + def test_status_task_not_done_running( + self, mock_tern_tasks, test_tern_app, fake_id + ): + """Test report task status is not done and running""" + + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + task_id = fake_id().hex + expected_response = TernAPIResponse( + reports.DataResponse( + id=task_id, status=task_status.RUNNING.value + ).to_dict() + ) + + # Task didn't finish yet + mock_tern_tasks.futures.done.return_value = False + # Task return as running + mock_tern_tasks.futures._state.return_value = task_status.RUNNING.value + + test_result = reports.status(task_id) + assert test_result == expected_response + + @mock.patch("tern_api.reports.tern_tasks") + def test_status_task_not_done_state_not_running( + self, mock_tern_tasks, test_tern_app, fake_id + ): + """Test report task status is not done and state not running""" + + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + task_id = fake_id().hex + expected_response = TernAPIResponse( + reports.DataResponse( + id=task_id, status=task_status.PENDING.value + ).to_dict() + ) + + # Task didn't finish yet + mock_tern_tasks.futures.done.return_value = False + # Task return as running + mock_tern_tasks.futures._state.return_value = "NOT_RUNNING" + + test_result = reports.status(task_id) + assert test_result == expected_response + + @mock.patch("tern_api.reports.tern_tasks") + def test_status_task_success( + self, mock_tern_tasks, test_tern_app, fake_id + ): + """Test report task status is not done and state not running""" + + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + task_id = fake_id().hex + expected_response = TernAPIResponse( + reports.DataResponse( + id=task_id, + status=task_status.SUCCESS.value, + report=json.loads(TERN_REPORT), + ).to_dict() + ) + + # Task didn't finish yet + mock_tern_tasks.futures.done.return_value = True + + mocked_result = mock.Mock() + mocked_result.result.return_value = json.loads(TERN_REPORT) + mock_tern_tasks.futures.pop.return_value = mocked_result + + test_result = reports.status(task_id) + assert test_result == expected_response + + @mock.patch("tern_api.reports.tern_tasks") + def test_status_tern_error(self, mock_tern_tasks, test_tern_app, fake_id): + """Test report task status success but tern error (finished)""" + + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + task_id = fake_id().hex + expected_response = TernAPIResponse( + reports.DataResponse( + id=task_id, + status=task_status.SUCCESS.value, + ).to_dict() + ) + expected_response.errors = {"message": "fake error"} + + # Task raise a handled error from Tern, what means the task + # is success finished. + mock_tern_tasks.futures.done.side_effect = reports.TernError( + "fake error" + ) + + test_result = reports.status(task_id) + assert test_result == expected_response + + @mock.patch("tern_api.reports.tern_tasks") + def test_status_task_error(self, mock_tern_tasks, test_tern_app, fake_id): + """Test report task status failed during the process (unfinished)""" + + # clean test tern tasks + test_tern_app.config["TERN_TASKS"].futures.pop("fake-id") + + task_id = fake_id().hex + expected_response = TernAPIResponse( + reports.DataResponse( + id=task_id, + status=task_status.FAILURE.value, + message="Task could not finish due", + ).to_dict() + ) + # Task raise a error, what means the task is didn't finished. + mock_tern_tasks.futures.done.side_effect = TypeError( + "wrong type during process" + ) + + test_result = reports.status(task_id) + assert test_result.status_code == 200 + assert expected_response.data.get("message") in test_result.data.get( + "message" + ), test_result.data.get("message") diff --git a/tests/tern_api/test_utils.py b/tests/tern_api/test_utils.py new file mode 100644 index 0000000..153e0af --- /dev/null +++ b/tests/tern_api/test_utils.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause +from tern_api.utils import TernAPIResponse + + +class TestTernAPIResponse: + def test_ternapi_response(self, test_tern_app): + """Test ternapi_response() basic functionality""" + with test_tern_app.app_context(): + response = TernAPIResponse() + + assert response.status_code == 200 + assert response.data == {} + assert response.errors == {} + assert response.to_response().status_code == 200 + assert response.to_response().json == {"data": {}} + + def test_to_response_with_data(self, test_tern_app): + """Test to_response() functionality with data""" + + with test_tern_app.app_context(): + response = TernAPIResponse( + data={"key": "value"}, + ) + + assert response.to_response().status_code == 200 + assert response.to_response().json == {"data": {"key": "value"}} + + def test_to_response_with_errors(self, test_tern_app): + """Test to_response() with errors""" + with test_tern_app.app_context(): + response = TernAPIResponse( + errors={"message": "error"}, + ) + + assert response.to_response().status_code == 200 + assert response.to_response().json == { + "data": {}, + "error": {"message": "error"}, + } + + def test_to_response_not_200(self, test_tern_app): + """Test bto response when is not 200 HTTP code""" + with test_tern_app.app_context(): + response = TernAPIResponse( + errors={"message": "Bad Request"}, status_code=400 + ) + + assert response.to_response().status_code == 400 + assert response.to_response().json == { + "data": {}, + "error": {"message": "Bad Request"}, + } diff --git a/tests/tern_api/v1/test_reports.py b/tests/tern_api/v1/test_reports.py new file mode 100644 index 0000000..7f2cd5d --- /dev/null +++ b/tests/tern_api/v1/test_reports.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause +from unittest import mock + +from tern_api.utils import TernAPIResponse +from tests.utils import RequestDataTest + + +class TestAPIReports: + @mock.patch("tern_api.api.v1.reports.submit") + def test_post_reports(self, mock_reports_submit, api_request): + expected_response = { + "data": { + "message": "Request submitted.", + "id": "fake-id", + "cache": True, + } + } + + payload = { + "registry": "registry.hub.docker.com", + "image": "photon", + "tag": "3.0", + "cache": True, + } + + mock_reports_submit.return_value = TernAPIResponse(expected_response) + test_response = api_request( + RequestDataTest( + method="POST", endpoint="/api/v1/reports", payload=payload + ) + ) + assert test_response.status_code == 200 + assert test_response.json.get("data") == expected_response + + def test_post_reports_missing_required_payload(self, api_request): + expected_response = { + "errors": { + "cache": "'cache' is a required property", + "image": "'image' is a required property", + "tag": "'tag' is a required property", + }, + "message": "Input payload validation failed", + } + + test_response = api_request( + RequestDataTest( + method="POST", endpoint="/api/v1/reports", payload={} + ) + ) + assert test_response.status_code == 400 + assert test_response.json == expected_response, test_response.json + + @mock.patch("tern_api.api.v1.reports.status") + def test_post_reports_status(self, mock_reports_status, api_request): + expected_response = { + "data": { + "cache": True, + "id": "19f035a711644eab84ef5a38ceb5572e", + "message": "", + "report": {}, + "status": "PENDING", + } + } + + payload = { + "id": "fake-id", + } + + mock_reports_status.return_value = TernAPIResponse(expected_response) + + test_response = api_request( + RequestDataTest( + method="POST", + endpoint="/api/v1/reports/status", + payload=payload, + ) + ) + assert test_response.status_code == 200 + assert test_response.json.get("data") == expected_response + + @mock.patch("tern_api.api.v1.reports.status") + def test_post_reports_status_invalid_payload( + self, mock_reports_status, api_request + ): + expected_response = { + "errors": { + "id": "'id' is a required property", + }, + "message": "Input payload validation failed", + } + + payload = {} + + mock_reports_status.return_value = TernAPIResponse(expected_response) + + test_response = api_request( + RequestDataTest( + method="POST", + endpoint="/api/v1/reports/status", + payload=payload, + ) + ) + assert test_response.status_code == 200 + assert test_response.json.get("data") == expected_response diff --git a/tests/tern_api/v1/test_version.py b/tests/tern_api/v1/test_version.py index 3581e25..eca7941 100644 --- a/tests/tern_api/v1/test_version.py +++ b/tests/tern_api/v1/test_version.py @@ -6,7 +6,7 @@ from tests.utils import RequestDataTest -class TestVersion: +class TestAPIVersion: def test_version(self, api_request): expected_response = { "api": version, diff --git a/tox.ini b/tox.ini index 9ee0db3..39667d6 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,7 @@ commands = [testenv:test] commands = - coverage run -m pytest + coverage run -m pytest -vv coverage report [gh-actions] From 703c26e50b7eb64358a4f0f504c1db22e0b45b5d Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Sat, 19 Mar 2022 09:29:20 +0100 Subject: [PATCH 7/8] added tests for the offline API documentation The documentation offline is available on http://tern-tools.github.io/tern-rest-api and tests to see if the API documentation is updated is a must. The Makefile contains the entry to build the offline documentation. Signed-off-by: Kairo de Araujo --- .gitignore | 2 +- Makefile | 5 ++++- README.md | 22 ++++++++++++------ docs/swagger.json | 57 +++++++++++++++++++++++++++++++++++++---------- tox.ini | 11 ++++++--- 5 files changed, 73 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index b1aa18c..be915d9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # tern-rest-api specifics cached - +docs/.test_swagger.json # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Makefile b/Makefile index a77648b..3ed578f 100644 --- a/Makefile +++ b/Makefile @@ -18,4 +18,7 @@ stop: update-requirements: pipenv lock -r > requirements.txt - pipenv lock -r -d > requirements-dev.txt \ No newline at end of file + pipenv lock -r -d > requirements-dev.txt + +doc: + python -c "import app; app.export_swagger_json('docs/swagger.json')" diff --git a/README.md b/README.md index 1167754..af27b1c 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ files to help build your virtual environment. We also recommend using [Pipenv](https://pipenv.pypa.io/en/latest/) to manage your virtual environment. ```shell -$ Pip install pipenv +$ pip install pipenv $ pipenv shell ``` @@ -59,7 +59,7 @@ $ pipenv install -d ### Running the development Tern REST API -## As a Docker Container +#### As a Docker Container ```shell $ make serve-dev ``` @@ -70,7 +70,7 @@ container and makes the development easier. You can stop the sever using ``Ctrl+C`` and running ``make stop`` -## On your local machine +#### On your local machine Runing the API locally ```shell @@ -79,21 +79,29 @@ $ flask run --reload Open http://localhost:5000/ in your browser. -## Tests +### Tests We use [Tox](https://tox.wiki/en/latest/) to manage running the tests. -## As a Docker Container +#### As a Docker Container ```shell $ make tests ``` -## On your local machine +#### On your local machine ```shell $ tox ``` -## Managing the requirements +### Documentation + +```shell +$ make doc +``` + +The documentation tests are done also by ``tox``. + +### Requirements Installing new requirements diff --git a/docs/swagger.json b/docs/swagger.json index 29a88b0..d8016ac 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2,7 +2,7 @@ "swagger": "2.0", "basePath": "/", "paths": { - "/api/v1/report": { + "/api/v1/reports": { "post": { "responses": { "200": { @@ -30,7 +30,7 @@ ] } }, - "/api/v1/report/status": { + "/api/v1/reports/status": { "post": { "responses": { "200": { @@ -145,15 +145,14 @@ }, "report_parameters": { "required": [ + "cache", "image", "tag" ], "properties": { "registry": { "type": "string", - "description": "Registry Server", - "default": "https://registry.hub.docker.com", - "example": "http://registry.example.com" + "description": "Registry Server" }, "image": { "type": "string", @@ -164,6 +163,11 @@ "type": "string", "description": "Image tag", "example": "3.0" + }, + "cache": { + "type": "boolean", + "description": "Use cache data if available?", + "example": true } }, "type": "object" @@ -181,6 +185,7 @@ }, "async_response_model": { "required": [ + "cache", "id" ], "properties": { @@ -193,6 +198,11 @@ "type": "string", "description": "Unique Identification for request", "example": "19f035a711644eab84ef5a38ceb5572e" + }, + "cache": { + "type": "boolean", + "description": "Request uses cache?", + "example": true } }, "type": "object" @@ -220,21 +230,38 @@ }, "data_status_response": { "required": [ + "cache", "status" ], "properties": { + "cache": { + "type": "boolean", + "description": "Requested using cache?", + "example": true + }, + "id": { + "type": "string", + "description": "Unique Identification for request", + "example": "19f035a711644eab84ef5a38ceb5572e" + }, + "message": { + "type": "string", + "description": "Message" + }, + "report": { + "$ref": "#/definitions/report_mode" + }, "status": { "type": "string", "description": "Status of request", - "example": "DONE", + "example": "SUCCESS", "enum": [ - "UNKNOWN", - "FAILED", - "DONE" + "PENDING", + "RECEIVED", + "RUNNING", + "SUCCESS", + "FAILURE" ] - }, - "result": { - "$ref": "#/definitions/report_mode" } }, "type": "object" @@ -260,6 +287,7 @@ }, "image_report_data": { "required": [ + "cache", "name", "repotag", "tag" @@ -276,6 +304,11 @@ "tag": { "type": "string", "example": "3.0" + }, + "cache": { + "type": "string", + "description": "Use cache if available?", + "default": true } }, "type": "object" diff --git a/tox.ini b/tox.ini index 39667d6..80c93e5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py39,lint,test +envlist = py39,lint,test,doc-offline-api [flake8] exclude = ownca/__init__.py,venv,.venv,settings.py,.git,.tox,dist,docs,*lib/python*,*egg,build,tools @@ -15,9 +15,14 @@ commands = black -l79 --check --diff . [testenv:doc-offline-api] +allowlist_externals = + diff + rm commands = - python -c "import app; app.export_swagger_json('docs/swagger.json')" - git diff main -- docs/swagger.json + python -c "import app; app.export_swagger_json('docs/.test_swagger.json')" + diff docs/swagger.json docs/.test_swagger.json +commands_post = + rm docs/.test_swagger.json [testenv:test] commands = From e5fdd0eeae0e5c8b666fa16e5880f7a42b339a44 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Fri, 3 Jun 2022 13:01:31 +0200 Subject: [PATCH 8/8] Add to the README container environment variables Simple document the container evironment variables in the README.md Two variables can be used to change the default value. - ``TERN_API_CACHE_DIR``: The directory where the tern reports are cached. Default: ``/var/opt/tern-rest-api/cache`` - ``TERN_DEFAULT_REGISTRY``: Optional default registry to use when no registry is passed to the API requests. Default: ``docker.io`` Signed-off-by: Kairo de Araujo --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index af27b1c..75fa781 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,14 @@ container and makes the development easier. You can stop the sever using ``Ctrl+C`` and running ``make stop`` +Container environment variables: + +- ``TERN_API_CACHE_DIR``: The directory where the tern reports are cached. +Default: ``/var/opt/tern-rest-api/cache`` +- ``TERN_DEFAULT_REGISTRY``: Optional default registry to use when no registry +is passed to the API requests. Default: ``docker.io`` + + #### On your local machine Runing the API locally