diff --git a/dev-requirements.in b/dev-requirements.in index fb90c597b9..3baa218102 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -20,6 +20,7 @@ IPython keyrings.alt setuptools_scm pytest-icdiff +jinja2 # Tensorflow is not available for python 3.12 yet: https://github.com/tensorflow/tensorflow/issues/62003 tensorflow; python_version<'3.12' diff --git a/dev-requirements.txt b/dev-requirements.txt index fa5840471b..301b6eae27 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -24,8 +24,6 @@ aioitertools==0.11.0 # via aiobotocore aiosignal==1.3.1 # via aiohttp -arrow==1.3.0 - # via cookiecutter asttokens==2.4.1 # via stack-data astunparse==1.6.3 @@ -47,8 +45,6 @@ azure-identity==1.15.0 # via adlfs azure-storage-blob==12.19.0 # via adlfs -binaryornot==0.4.4 - # via cookiecutter botocore==1.31.17 # via aiobotocore cachetools==5.3.2 @@ -63,21 +59,16 @@ cffi==1.16.0 # cryptography cfgv==3.4.0 # via pre-commit -chardet==5.2.0 - # via binaryornot charset-normalizer==3.3.2 # via requests click==8.1.7 # via - # cookiecutter # flytekit # rich-click cloudpickle==3.0.0 # via flytekit codespell==2.2.6 # via -r dev-requirements.in -cookiecutter==2.5.0 - # via flytekit coverage[toml]==7.4.1 # via # -r dev-requirements.in @@ -114,8 +105,10 @@ filelock==3.13.1 # virtualenv flatbuffers==23.5.26 # via tensorflow-macos -flyteidl==1.10.6 - # via flytekit +flyteidl @ git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl + # via + # -r dev-requirements.in + # flytekit frozenlist==1.4.1 # via # aiohttp @@ -176,6 +169,7 @@ googleapis-common-protos==1.62.0 # flytekit # google-api-core # grpcio-status + # protoc-gen-openapiv2 grpcio==1.60.1 # via # flytekit @@ -191,6 +185,8 @@ h5py==3.10.0 # via tensorflow-macos hypothesis==6.98.2 # via -r dev-requirements.in +icdiff==2.0.7 + # via pytest-icdiff identify==2.5.33 # via pre-commit idna==3.6 @@ -206,7 +202,9 @@ iniconfig==2.0.0 ipython==8.21.0 # via -r dev-requirements.in isodate==0.6.1 - # via azure-storage-blob + # via + # azure-storage-blob + # flytekit jaraco-classes==3.3.0 # via # keyring @@ -215,7 +213,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.3 # via - # cookiecutter + # -r dev-requirements.in # torch jmespath==1.0.1 # via botocore @@ -233,13 +231,15 @@ keyring==24.3.0 keyrings-alt==5.0.0 # via -r dev-requirements.in kubernetes==29.0.0 - # via flytekit + # via -r dev-requirements.in libclang==16.0.6 # via tensorflow-macos markdown==3.5.2 # via tensorboard markdown-it-py==3.0.0 - # via rich + # via + # flytekit + # rich markupsafe==2.1.5 # via # jinja2 @@ -307,7 +307,7 @@ oauthlib==3.2.2 # kubernetes # requests-oauthlib opt-einsum==3.3.0 - # via tensorflow + # via tensorflow-macos orjson==3.9.12 # via -r dev-requirements.in packaging==23.2 @@ -333,6 +333,8 @@ pluggy==1.4.0 # via pytest portalocker==2.8.2 # via msal-extensions +pprintpp==0.4.0 + # via pytest-icdiff pre-commit==3.6.0 # via -r dev-requirements.in prometheus-client==0.19.0 @@ -350,10 +352,10 @@ protobuf==4.23.4 # googleapis-common-protos # grpcio-status # proto-plus - # protoc-gen-swagger + # protoc-gen-openapiv2 # tensorboard # tensorflow-macos -protoc-gen-swagger==0.1.0 +protoc-gen-openapiv2==0.0.1 # via flyteidl ptyprocess==0.7.0 # via pexpect @@ -373,17 +375,17 @@ pyflakes==3.2.0 # via autoflake pygments==2.17.2 # via + # flytekit # ipython # rich pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt + # via msal pytest==7.4.4 # via # -r dev-requirements.in # pytest-asyncio # pytest-cov + # pytest-icdiff # pytest-mock # pytest-timeout # pytest-xdist @@ -391,6 +393,8 @@ pytest-asyncio==0.23.4 # via -r dev-requirements.in pytest-cov==4.1.0 # via -r dev-requirements.in +pytest-icdiff==0.9 + # via -r dev-requirements.in pytest-mock==3.12.0 # via -r dev-requirements.in pytest-timeout==2.2.0 @@ -399,7 +403,6 @@ pytest-xdist==3.5.0 # via -r dev-requirements.in python-dateutil==2.8.2 # via - # arrow # botocore # croniter # google-cloud-bigquery @@ -409,8 +412,6 @@ python-json-logger==2.0.7 # via flytekit python-magic==0.4.27 ; platform_system == "Darwin" or platform_system == "Linux" # via -r dev-requirements.in -python-slugify==8.0.3 - # via cookiecutter pytimeparse==1.1.8 # via flytekit pytz==2024.1 @@ -419,7 +420,6 @@ pytz==2024.1 # pandas pyyaml==6.0.1 # via - # cookiecutter # flytekit # kubernetes # pre-commit @@ -427,7 +427,6 @@ requests==2.31.0 # via # azure-core # azure-datalake-store - # cookiecutter # docker # flytekit # gcsfs @@ -444,7 +443,6 @@ requests-oauthlib==1.3.1 # kubernetes rich==13.7.0 # via - # cookiecutter # flytekit # rich-click rich-click==1.7.3 @@ -492,8 +490,6 @@ tensorflow-macos==2.15.0 # via tensorflow termcolor==2.4.0 # via tensorflow-macos -text-unidecode==1.3 - # via python-slugify threadpoolctl==3.2.0 # via scikit-learn torch==2.2.0 ; python_version < "3.12" @@ -504,12 +500,12 @@ traitlets==5.14.1 # matplotlib-inline types-croniter==2.0.0.20240106 # via -r dev-requirements.in +types-decorator==5.1.8.20240310 + # via -r dev-requirements.in types-mock==5.1.0.20240106 # via -r dev-requirements.in types-protobuf==4.24.0.20240129 # via -r dev-requirements.in -types-python-dateutil==2.8.19.20240106 - # via arrow types-requests==2.31.0.6 # via -r dev-requirements.in types-urllib3==1.26.25.14 diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py index ddedb8fdc7..3da1bb8e78 100644 --- a/flytekit/clis/sdk_in_container/init.py +++ b/flytekit/clis/sdk_in_container/init.py @@ -1,36 +1,61 @@ +import os +import re +from io import BytesIO +from zipfile import ZipFile + +import requests import rich_click as click -from cookiecutter.main import cookiecutter @click.command("init") @click.option( "--template", default="basic-template-imagespec", - help="cookiecutter template folder name to be used in the repo - https://github.com/flyteorg/flytekit-python-template.git", + help="template folder name to be used in the repo - https://github.com/flyteorg/flytekit-python-template.git", ) @click.argument("project-name") def init(template, project_name): """ Create flyte-ready projects. """ - config = { - "project_name": project_name, - "app": "flyte", - "workflow": "my_wf", - } - cookiecutter( - "https://github.com/flyteorg/flytekit-python-template.git", - checkout="main", - no_input=True, - # We do not want to clobber existing files/directories. - overwrite_if_exists=False, - extra_context=config, - # By specifying directory we can have multiple templates in the same repository, - # as described in https://cookiecutter.readthedocs.io/en/1.7.2/advanced/directories.html. - # The idea is to extend the number of templates, each in their own subdirectory, for example - # a tensorflow-based example. - directory=template, - ) + if os.path.exists(project_name): + raise click.ClickException(f"{project_name} directory already exists") + + template_zip_url = "https://github.com/flyteorg/flytekit-python-template/archive/refs/heads/main.zip" + + response = requests.get(template_zip_url) + + if response.status_code != 200: + raise click.ClickException("Unable to download template from github.com/flyteorg/flytekit-python-template") + + zip_content = BytesIO(response.content) + zip_root_name = "flytekit-python-template-main" + project_name_template = "{{cookiecutter.project_name}}" + prefix = os.path.join(zip_root_name, template, project_name_template, "") + prefix_len = len(prefix) + + # We use a regex here to be more compatible with cookiecutter templating + project_template_regex = re.compile(rb"\{\{ ?cookiecutter\.project_name ?\}\}") + + project_name_bytes = project_name.encode("utf-8") + + with ZipFile(zip_content, "r") as zip_file: + template_members = [m for m in zip_file.namelist() if m.startswith(prefix)] + + for member in template_members: + dest = os.path.join(project_name, member[prefix_len:]) + + # member is a directory + if dest.endswith(os.sep): + if not os.path.exists(dest): + os.mkdir(dest) + continue + + # member is a file + with zip_file.open(member) as zip_member, open(dest, "wb") as dest_file: + zip_contents = zip_member.read() + processed_contents = project_template_regex.sub(project_name_bytes, zip_contents) + dest_file.write(processed_contents) click.echo( f"Visit the {project_name} directory and follow the next steps in the Getting started guide (https://docs.flyte.org/en/latest/getting_started_with_workflow_development/index.html) to proceed." diff --git a/pyproject.toml b/pyproject.toml index 0373db6f67..e3d311a73a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,6 @@ dependencies = [ "adlfs>=2023.3.0", "click>=6.6,<9.0", "cloudpickle>=2.0.0", - "cookiecutter>=1.7.3", "croniter>=0.3.20,<4.0.0", "dataclasses-json>=0.5.2,<0.5.12", # TODO: remove upper-bound after fixing change in contract "diskcache>=5.2.1",