Docker compose v2 fork

MANIFEST.in

@@ -1 +1 @@
-include Makefile CHANGES.rst LICENSE AUTHORS juniper/artifacts/*
+include Makefile CHANGES.rst LICENSE AUTHORS juniper/artifacts/* requirements/*

Makefile

@@ -6,6 +6,10 @@ install-dev:
 	pip install -q -e .[venv3]
 	pip install -r requirements/dev.txt
 
+deps:
+	pip-compile -U --allow-unsafe -o requirements/dev.txt requirements/requirements-dev.in requirements/requirements.in
+	pip-compile -U --allow-unsafe -o requirements/requirements.txt requirements/requirements.in
+
 test: clean-pyc
 	python -m pytest --cov .
 
@@ -36,19 +40,17 @@ gh-pages:
 
 release:
 	rm -rf ./dist
-	python3 -m pip install --upgrade build
 	python3 -m build
 	twine check dist/*
 	twine upload dist/*
 
 test-release:
 	rm -rf ./dist
-	python3 -m pip install --upgrade build
 	python3 -m build
 	twine check dist/*
 	python3 -m twine upload --repository-url https://test.pypi.org/legacy/ dist/*
 
 clean-pyc:
 	find . -name '*.pyc' -exec rm -f {} +
 	find . -name '*.pyo' -exec rm -f {} +
-	find . -name '*~' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +

README.rst

@@ -257,6 +257,7 @@ This list defines the entire scope of Juniper. Nothing more, nothing else.
 * Specify docker image to package lamdba functions using different python runtimes
 * Define pip command line arguments using a pip.conf file
 * Packaging of lambda layers
+* Support for building arm64 layers and lambdas (Graviton2) via specifying the platform
 
 Contributing
 ************

juniper/__init__.py

@@ -14,4 +14,4 @@
     limitations under the License.
 """
 
-__version__ = '0.5.5'
+__version__ = '0.8.1'

juniper/actions.py

@@ -18,6 +18,7 @@
 import shutil
 import subprocess
 from jinja2 import Template
+import functools
 
 from juniper.constants import DEFAULT_OUT_DIR, DEFAULT_DOCKER_IMAGE
 from juniper.io import (get_artifact, write_tmp_file, get_artifact_path)
@@ -36,25 +37,28 @@ def build_artifacts(logger, manifest):
     """
 
     compose_fn = build_compose(logger, manifest)
-    logger.debug(f'docker-compose -f {compose_fn} --project-directory . run sample-lambda bash')
+    logger.debug(f'docker compose -f {compose_fn} --project-directory . run sample-lambda bash')
     try:
         # Must copy the bin directory to the client's folder structure. This directory
         # will be promtly cleaned up after the artifacts are built.
         os.makedirs('./.juni/bin', exist_ok=True)
-        shutil.copy(get_artifact_path('package.sh'), './.juni/bin/')
-        shutil.copy(get_artifact_path('build_layer.sh'), './.juni/bin/')
+        with get_artifact_path('package.sh') as path:
+            shutil.copy(path, './.juni/bin/')
+        with get_artifact_path('build_layer.sh') as path:
+            shutil.copy(path, './.juni/bin/')
 
         # Use docker as a way to pip install dependencies, and copy the business logic
         # specified in the function definitions.
-        subprocess.run(["docker-compose", "-f", compose_fn, '--project-directory', '.', 'down'])
-        subprocess.run(["docker-compose", "-f", compose_fn, '--project-directory', '.', 'up'])
+        subprocess.run(["docker", "compose", "-f", compose_fn, '--project-directory', '.', 'down', '--remove-orphans'])
+        subprocess.run(["docker", "compose", "-f", compose_fn, '--project-directory', '.', 'up'])
+        subprocess.run(["docker", "compose", "-f", compose_fn, '--project-directory', '.', 'down'])
     finally:
         shutil.rmtree('./.juni', ignore_errors=True)
 
 
 def build_compose(logger, manifest):
     """
-    Builds a docker-compose file with the lambda functions defined in the manifest.
+    Builds a docker compose file with the lambda functions defined in the manifest.
     The definition of the lambda functions includes the name of the function as
     well as the set of dependencies to include in the packaging.
 
@@ -63,14 +67,14 @@ def build_compose(logger, manifest):
     """
 
     compose = _get_compose_template(manifest)
-    # Returns the name of the temp file that has the docker-compose definition.
+    # Returns the name of the temp file that has the docker compose definition.
     return write_tmp_file(compose)
 
 
 def _get_compose_template(manifest):
     """
     Build the service entry for each one of the functions in the given context.
-    Each docker-compose entry will depend on the same image and it's just a static
+    Each docker compose entry will depend on the same image and it's just a static
     definition that gets built from a template. The template is in the artifacts
     folder.
     """
@@ -81,6 +85,7 @@ def build_section(label):
             {
                 'name': name,
                 'image': _get_docker_image(manifest, sls_section),
+                'platform': _get_platform(manifest, sls_section),
                 'volumes': _get_volumes(manifest, sls_section)
             }
             for name, sls_section in manifest.get(label, {}).items()
@@ -132,21 +137,27 @@ def get_vol(include):
     return volumes
 
 
-def _get_docker_image(manifest, sls_function):
+def _get_attr(key, default, manifest, sls_function):
     """
-    Get the docker image that will be used to package a given function. Precedence
-    is as follows: function level override, global image override, default.
+    Get an attribute from the manifest, looking under the function first,
+    then global:, and finally using a default.
 
-    :params manfiest: The juniper manifest file.
+
+    :params key: The key to retrieve
+    :params default: The value to return if your key is not defined anywhere
+    :params manifest: The juniper manifest file.
     :params sls_function: The serverless function definition.
     """
+    function_value = sls_function.get(key)
+    if function_value:
+        return function_value
+
+    global_value = manifest.get('global', {}).get(key)
+    if global_value:
+        return global_value
 
-    function_image = sls_function.get('image')
-    if function_image:
-        return function_image
+    return default
 
-    global_image = manifest.get('global', {}).get('image')
-    if global_image:
-        return global_image
 
-    return DEFAULT_DOCKER_IMAGE
+_get_docker_image = functools.partial(_get_attr, "image", DEFAULT_DOCKER_IMAGE)
+_get_platform = functools.partial(_get_attr, "platform", None)

juniper/artifacts/compose-template.yml

@@ -11,6 +11,9 @@ services:
         - {{volume}}
       {% endfor %}
     command: sh /var/task/bin/package.sh {{lambda_fn.name}}
+{%- if lambda_fn.platform %}
+    platform: {{ lambda_fn.platform }}
+{%- endif %}
 {% endfor %}
 {% for layer in layers %}
   {{layer.name}}-layer:
@@ -22,4 +25,7 @@ services:
         - {{volume}}
       {% endfor %}
     command: sh /var/task/bin/build_layer.sh {{layer.name}}
+{%- if layer.platform %}
+    platform: {{ layer.platform }}
+{%- endif %}
 {% endfor %}

juniper/io.py

@@ -18,7 +18,9 @@
 import os
 import yaml
 import tempfile
-import pkg_resources
+import importlib.resources
+import contextlib
+import pathlib
 
 
 def reader(file_name):
@@ -39,12 +41,14 @@ def write_tmp_file(content):
 def get_artifact(template_name):
 
     fn_template_path = get_artifact_path(template_name)
-    with open(fn_template_path, 'r') as f:
-        return f.read()
+    with fn_template_path as path:
+        with path.open("r") as fd:
+            return fd.read()
 
 
-def get_artifact_path(artifact_name):
+def get_artifact_path(artifact_name) -> contextlib.AbstractContextManager[pathlib.Path]:
     """
     Reads an artifact out of the rio-tools project.
     """
-    return pkg_resources.resource_filename(__name__, os.path.join('artifacts', artifact_name))
+    files = importlib.resources.files("juniper")
+    return importlib.resources.as_file(files.joinpath(os.path.join('artifacts', artifact_name)))

requirements/dev.txt

@@ -1,15 +1,103 @@
-Click==7.0
-click-log==0.3.2
-ipdb==0.11
-coverage==4.5.1
-coverage-badge==0.2.0
-docker>=5
-docker-compose>=1.29
-pytest==3.5.1
-pytest-mock==1.10.0
-pytest-pythonpath==0.7.2
-pytest-cov==2.5.1
-flake8==3.5.0
-PyYAML==5.3
-Sphinx==1.7.6
-phix==0.6.1
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --allow-unsafe --output-file=requirements/dev.txt requirements/requirements-dev.in requirements/requirements.in
+#
+bleach==6.0.0
+    # via readme-renderer
+build==0.10.0
+    # via
+    #   -r requirements/requirements-dev.in
+    #   pip-tools
+certifi==2023.7.22
+    # via requests
+charset-normalizer==3.2.0
+    # via requests
+click==8.1.7
+    # via
+    #   -r requirements/requirements.in
+    #   click-log
+    #   pip-tools
+click-log==0.4.0
+    # via -r requirements/requirements.in
+docutils==0.20.1
+    # via readme-renderer
+idna==3.4
+    # via requests
+importlib-metadata==6.8.0
+    # via
+    #   keyring
+    #   twine
+iniconfig==2.0.0
+    # via pytest
+jaraco-classes==3.3.0
+    # via keyring
+jinja2==3.1.2
+    # via -r requirements/requirements.in
+keyring==24.2.0
+    # via twine
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.3
+    # via jinja2
+mdurl==0.1.2
+    # via markdown-it-py
+more-itertools==10.1.0
+    # via jaraco-classes
+packaging==23.1
+    # via
+    #   build
+    #   pytest
+pip-tools==7.3.0
+    # via -r requirements/requirements-dev.in
+pkginfo==1.9.6
+    # via twine
+pluggy==1.2.0
+    # via pytest
+pygments==2.16.1
+    # via
+    #   readme-renderer
+    #   rich
+pyproject-hooks==1.0.0
+    # via build
+pytest==7.4.0
+    # via
+    #   -r requirements/requirements-dev.in
+    #   pytest-mock
+pytest-mock==3.11.1
+    # via -r requirements/requirements-dev.in
+pyyaml==6.0.1
+    # via -r requirements/requirements.in
+readme-renderer==41.0
+    # via twine
+requests==2.31.0
+    # via
+    #   requests-toolbelt
+    #   twine
+requests-toolbelt==1.0.0
+    # via twine
+rfc3986==2.0.0
+    # via twine
+rich==13.5.2
+    # via twine
+six==1.16.0
+    # via bleach
+twine==4.0.2
+    # via -r requirements/requirements-dev.in
+urllib3==2.0.4
+    # via
+    #   requests
+    #   twine
+webencodings==0.5.1
+    # via bleach
+wheel==0.41.1
+    # via pip-tools
+zipp==3.16.2
+    # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==23.2.1
+    # via pip-tools
+setuptools==68.1.2
+    # via pip-tools

requirements/requirements-dev.in

@@ -0,0 +1,5 @@
+pytest==7.4.0
+pip-tools
+pytest-mock
+twine
+build

requirements/requirements.in

@@ -0,0 +1,4 @@
+click>=8.0
+click-log
+PyYAML>=6.0
+Jinja2>=3.0

requirements/requirements.txt

@@ -0,0 +1,18 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --allow-unsafe --output-file=requirements/requirements.txt requirements/requirements.in
+#
+click==8.1.7
+    # via
+    #   -r requirements/requirements.in
+    #   click-log
+click-log==0.4.0
+    # via -r requirements/requirements.in
+jinja2==3.1.2
+    # via -r requirements/requirements.in
+markupsafe==2.1.3
+    # via jinja2
+pyyaml==6.0.1
+    # via -r requirements/requirements.in