add util function to enable transparent local boto3 endpoints (#40)

Author: whummer

.github/workflows/ci.yml

@@ -13,6 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
+          - "3.10"
           - "3.9"
           - "3.8"
           - "3.7"

CHANGELOG.md

@@ -1,5 +1,6 @@
 # LocalStack Python Client Change Log
 
+* v1.38: Add `enable_local_endpoints()` util function; slight project refactoring, migrate from `nose` to `pytests`
 * v1.37: Add endpoint for Amazon Transcribe
 * v1.36: Add endpoints for Fault Injection Service (FIS) and Marketplace Metering
 * v1.35: Add endpoint for Amazon Managed Workflows for Apache Airflow (MWAA)

Makefile

@@ -14,16 +14,19 @@ install:           ## Install dependencies in local virtualenv folder
 
 publish:           ## Publish the library to the central PyPi repository
 	# build and upload archive
-	($(VENV_RUN); ./setup.py sdist && twine upload $(BUILD_DIR)/*.tar.gz)
+	$(VENV_RUN); ./setup.py sdist && twine upload $(BUILD_DIR)/*.tar.gz
 
 test:              ## Run automated tests
 	($(VENV_RUN); test `which localstack` || pip install .[test]) && \
-	$(VENV_RUN); DEBUG=$(DEBUG) PYTHONPATH=`pwd` nosetests --with-coverage --logging-level=WARNING --nocapture --no-skip --exe --cover-erase --cover-tests --cover-inclusive --cover-package=localstack_client --with-xunit --exclude='$(VENV_DIR).*' .
+	$(VENV_RUN); DEBUG=$(DEBUG) PYTHONPATH=. pytest -sv $(PYTEST_ARGS) tests
 
 lint:              ## Run code linter to check code style
-	($(VENV_RUN); pycodestyle --max-line-length=100 --ignore=E128 --exclude=node_modules,legacy,$(VENV_DIR),dist .)
+	$(VENV_RUN); flake8 --ignore=E501 localstack_client tests
+
+format:            ## Run code formatter (black)
+	$(VENV_RUN); black localstack_client tests; isort localstack_client tests
 
 clean:             ## Clean up virtualenv
 	rm -rf $(VENV_DIR)
 
-.PHONY: usage install clean publish test lint
+.PHONY: usage install clean publish test lint format

README.md

@@ -66,6 +66,27 @@ sqs = boto3.client('sqs')
 assert sqs.list_queues() is not None  # list SQS in localstack
 ```
 
+### Enabling Transparent Local Endpoints
+
+The library contains a small `enable_local_endpoints()` util function that can be used to transparently run all `boto3` requests against the local endpoints.
+
+The following sample illustrates how it can be used - after calling `enable_local_endpoints()`, the S3 `ListBuckets` call will be run against LocalStack, even though we're using the default boto3 module.
+```
+import boto3
+from localstack_client.patch import enable_local_endpoints()
+enable_local_endpoints()
+# the call below will automatically target the LocalStack endpoints
+buckets = boto3.client("s3").list_buckets()
+```
+
+The patch can also be unapplied by calling `disable_local_endpoints()`:
+```
+from localstack_client.patch import disable_local_endpoints()
+disable_local_endpoints()
+# the call below will target the real AWS cloud again
+buckets = boto3.client("s3").list_buckets()
+```
+
 ## Contributing
 
 If you are interested in contributing to LocalStack Python Client, start by reading our [`CONTRIBUTING.md`](CONTRIBUTING.md) guide. You can further navigate our codebase and [open issues](https://github.com/localstack/localstack-python-client/issues). We are thankful for all the contributions and feedback we receive.

localstack_client/config.py

@@ -1,164 +1,161 @@
-import os
 import json
-from botocore.serialize import Serializer
-
-# central entrypoint port for all LocalStack API endpoints
+import os
+from typing import Dict
 from urllib.parse import urlparse
 
-EDGE_PORT = int(os.environ.get('EDGE_PORT') or 4566)
+# note: leave this import here for now, as some upstream code is depending on it (TODO needs to be updated)
+from localstack_client.patch import patch_expand_host_prefix  # noqa
+
+# central entrypoint port for all LocalStack API endpoints
+EDGE_PORT = int(os.environ.get("EDGE_PORT") or 4566)
 
 # NOTE: The endpoints below will soon become deprecated/removed, as the default in the
 # latest version is to access all services via a single "edge service" (port 4566 by default)
 _service_endpoints_template = {
-    'edge': '{proto}://{host}:4566',
-    'apigateway': '{proto}://{host}:4567',
-    'apigatewayv2': '{proto}://{host}:4567',
-    'kinesis': '{proto}://{host}:4568',
-    'dynamodb': '{proto}://{host}:4569',
-    'dynamodbstreams': '{proto}://{host}:4570',
-    'elasticsearch': '{proto}://{host}:4571',
-    's3': '{proto}://{host}:4572',
-    'firehose': '{proto}://{host}:4573',
-    'lambda': '{proto}://{host}:4574',
-    'sns': '{proto}://{host}:4575',
-    'sqs': '{proto}://{host}:4576',
-    'redshift': '{proto}://{host}:4577',
-    'redshift-data': '{proto}://{host}:4577',
-    'es': '{proto}://{host}:4578',
-    'opensearch': '{proto}://{host}:4578',
-    'ses': '{proto}://{host}:4579',
-    'sesv2': '{proto}://{host}:4579',
-    'route53': '{proto}://{host}:4580',
-    'route53resolver': '{proto}://{host}:4580',
-    'cloudformation': '{proto}://{host}:4581',
-    'cloudwatch': '{proto}://{host}:4582',
-    'ssm': '{proto}://{host}:4583',
-    'secretsmanager': '{proto}://{host}:4584',
-    'stepfunctions': '{proto}://{host}:4585',
-    'logs': '{proto}://{host}:4586',
-    'events': '{proto}://{host}:4587',
-    'elb': '{proto}://{host}:4588',
-    'iot': '{proto}://{host}:4589',
-    'iotanalytics': '{proto}://{host}:4589',
-    'iotevents': '{proto}://{host}:4589',
-    'iotevents-data': '{proto}://{host}:4589',
-    'iotwireless': '{proto}://{host}:4589',
-    'iot-data': '{proto}://{host}:4589',
-    'iot-jobs-data': '{proto}://{host}:4589',
-    'cognito-idp': '{proto}://{host}:4590',
-    'cognito-identity': '{proto}://{host}:4591',
-    'sts': '{proto}://{host}:4592',
-    'iam': '{proto}://{host}:4593',
-    'rds': '{proto}://{host}:4594',
-    'rds-data': '{proto}://{host}:4594',
-    'cloudsearch': '{proto}://{host}:4595',
-    'swf': '{proto}://{host}:4596',
-    'ec2': '{proto}://{host}:4597',
-    'elasticache': '{proto}://{host}:4598',
-    'kms': '{proto}://{host}:4599',
-    'emr': '{proto}://{host}:4600',
-    'ecs': '{proto}://{host}:4601',
-    'eks': '{proto}://{host}:4602',
-    'xray': '{proto}://{host}:4603',
-    'elasticbeanstalk': '{proto}://{host}:4604',
-    'appsync': '{proto}://{host}:4605',
-    'cloudfront': '{proto}://{host}:4606',
-    'athena': '{proto}://{host}:4607',
-    'glue': '{proto}://{host}:4608',
-    'sagemaker': '{proto}://{host}:4609',
-    'sagemaker-runtime': '{proto}://{host}:4609',
-    'ecr': '{proto}://{host}:4610',
-    'qldb': '{proto}://{host}:4611',
-    'qldb-session': '{proto}://{host}:4611',
-    'cloudtrail': '{proto}://{host}:4612',
-    'glacier': '{proto}://{host}:4613',
-    'batch': '{proto}://{host}:4614',
-    'organizations': '{proto}://{host}:4615',
-    'autoscaling': '{proto}://{host}:4616',
-    'mediastore': '{proto}://{host}:4617',
-    'mediastore-data': '{proto}://{host}:4617',
-    'transfer': '{proto}://{host}:4618',
-    'acm': '{proto}://{host}:4619',
-    'codecommit': '{proto}://{host}:4620',
-    'kinesisanalytics': '{proto}://{host}:4621',
-    'kinesisanalyticsv2': '{proto}://{host}:4621',
-    'amplify': '{proto}://{host}:4622',
-    'application-autoscaling': '{proto}://{host}:4623',
-    'kafka': '{proto}://{host}:4624',
-    'apigatewaymanagementapi': '{proto}://{host}:4625',
-    'timestream': '{proto}://{host}:4626',
-    'timestream-query': '{proto}://{host}:4626',
-    'timestream-write': '{proto}://{host}:4626',
-    's3control': '{proto}://{host}:4627',
-    'elbv2': '{proto}://{host}:4628',
-    'support': '{proto}://{host}:4629',
-    'neptune': '{proto}://{host}:4594',
-    'docdb': '{proto}://{host}:4594',
-    'servicediscovery': '{proto}://{host}:4630',
-    'serverlessrepo': '{proto}://{host}:4631',
-    'appconfig': '{proto}://{host}:4632',
-    'ce': '{proto}://{host}:4633',
-    'mediaconvert': '{proto}://{host}:4634',
-    'resourcegroupstaggingapi': '{proto}://{host}:4635',
-    'resource-groups': '{proto}://{host}:4636',
-    'efs': '{proto}://{host}:4637',
-    'backup': '{proto}://{host}:4638',
-    'lakeformation': '{proto}://{host}:4639',
-    'waf': '{proto}://{host}:4640',
-    'wafv2': '{proto}://{host}:4640',
-    'config': '{proto}://{host}:4641',
-    'configservice': '{proto}://{host}:4641',
-    'mwaa': '{proto}://{host}:4642',
-    'fis': '{proto}://{host}:4643',
-    'meteringmarketplace': '{proto}://{host}:4644',
-    'transcribe': '{proto}://{host}:4566',
+    "edge": "{proto}://{host}:4566",
+    "apigateway": "{proto}://{host}:4567",
+    "apigatewayv2": "{proto}://{host}:4567",
+    "kinesis": "{proto}://{host}:4568",
+    "dynamodb": "{proto}://{host}:4569",
+    "dynamodbstreams": "{proto}://{host}:4570",
+    "elasticsearch": "{proto}://{host}:4571",
+    "s3": "{proto}://{host}:4572",
+    "firehose": "{proto}://{host}:4573",
+    "lambda": "{proto}://{host}:4574",
+    "sns": "{proto}://{host}:4575",
+    "sqs": "{proto}://{host}:4576",
+    "redshift": "{proto}://{host}:4577",
+    "redshift-data": "{proto}://{host}:4577",
+    "es": "{proto}://{host}:4578",
+    "opensearch": "{proto}://{host}:4578",
+    "ses": "{proto}://{host}:4579",
+    "sesv2": "{proto}://{host}:4579",
+    "route53": "{proto}://{host}:4580",
+    "route53resolver": "{proto}://{host}:4580",
+    "cloudformation": "{proto}://{host}:4581",
+    "cloudwatch": "{proto}://{host}:4582",
+    "ssm": "{proto}://{host}:4583",
+    "secretsmanager": "{proto}://{host}:4584",
+    "stepfunctions": "{proto}://{host}:4585",
+    "logs": "{proto}://{host}:4586",
+    "events": "{proto}://{host}:4587",
+    "elb": "{proto}://{host}:4588",
+    "iot": "{proto}://{host}:4589",
+    "iotanalytics": "{proto}://{host}:4589",
+    "iotevents": "{proto}://{host}:4589",
+    "iotevents-data": "{proto}://{host}:4589",
+    "iotwireless": "{proto}://{host}:4589",
+    "iot-data": "{proto}://{host}:4589",
+    "iot-jobs-data": "{proto}://{host}:4589",
+    "cognito-idp": "{proto}://{host}:4590",
+    "cognito-identity": "{proto}://{host}:4591",
+    "sts": "{proto}://{host}:4592",
+    "iam": "{proto}://{host}:4593",
+    "rds": "{proto}://{host}:4594",
+    "rds-data": "{proto}://{host}:4594",
+    "cloudsearch": "{proto}://{host}:4595",
+    "swf": "{proto}://{host}:4596",
+    "ec2": "{proto}://{host}:4597",
+    "elasticache": "{proto}://{host}:4598",
+    "kms": "{proto}://{host}:4599",
+    "emr": "{proto}://{host}:4600",
+    "ecs": "{proto}://{host}:4601",
+    "eks": "{proto}://{host}:4602",
+    "xray": "{proto}://{host}:4603",
+    "elasticbeanstalk": "{proto}://{host}:4604",
+    "appsync": "{proto}://{host}:4605",
+    "cloudfront": "{proto}://{host}:4606",
+    "athena": "{proto}://{host}:4607",
+    "glue": "{proto}://{host}:4608",
+    "sagemaker": "{proto}://{host}:4609",
+    "sagemaker-runtime": "{proto}://{host}:4609",
+    "ecr": "{proto}://{host}:4610",
+    "qldb": "{proto}://{host}:4611",
+    "qldb-session": "{proto}://{host}:4611",
+    "cloudtrail": "{proto}://{host}:4612",
+    "glacier": "{proto}://{host}:4613",
+    "batch": "{proto}://{host}:4614",
+    "organizations": "{proto}://{host}:4615",
+    "autoscaling": "{proto}://{host}:4616",
+    "mediastore": "{proto}://{host}:4617",
+    "mediastore-data": "{proto}://{host}:4617",
+    "transfer": "{proto}://{host}:4618",
+    "acm": "{proto}://{host}:4619",
+    "codecommit": "{proto}://{host}:4620",
+    "kinesisanalytics": "{proto}://{host}:4621",
+    "kinesisanalyticsv2": "{proto}://{host}:4621",
+    "amplify": "{proto}://{host}:4622",
+    "application-autoscaling": "{proto}://{host}:4623",
+    "kafka": "{proto}://{host}:4624",
+    "apigatewaymanagementapi": "{proto}://{host}:4625",
+    "timestream": "{proto}://{host}:4626",
+    "timestream-query": "{proto}://{host}:4626",
+    "timestream-write": "{proto}://{host}:4626",
+    "s3control": "{proto}://{host}:4627",
+    "elbv2": "{proto}://{host}:4628",
+    "support": "{proto}://{host}:4629",
+    "neptune": "{proto}://{host}:4594",
+    "docdb": "{proto}://{host}:4594",
+    "servicediscovery": "{proto}://{host}:4630",
+    "serverlessrepo": "{proto}://{host}:4631",
+    "appconfig": "{proto}://{host}:4632",
+    "ce": "{proto}://{host}:4633",
+    "mediaconvert": "{proto}://{host}:4634",
+    "resourcegroupstaggingapi": "{proto}://{host}:4635",
+    "resource-groups": "{proto}://{host}:4636",
+    "efs": "{proto}://{host}:4637",
+    "backup": "{proto}://{host}:4638",
+    "lakeformation": "{proto}://{host}:4639",
+    "waf": "{proto}://{host}:4640",
+    "wafv2": "{proto}://{host}:4640",
+    "config": "{proto}://{host}:4641",
+    "configservice": "{proto}://{host}:4641",
+    "mwaa": "{proto}://{host}:4642",
+    "fis": "{proto}://{host}:4643",
+    "meteringmarketplace": "{proto}://{host}:4644",
+    "transcribe": "{proto}://{host}:4566",
 }
 
 # TODO remove service port mapping above entirely
-if os.environ.get('USE_LEGACY_PORTS') not in ['1', 'true']:
+if os.environ.get("USE_LEGACY_PORTS") not in ["1", "true"]:
     for key, value in _service_endpoints_template.items():
-        if key not in ['dashboard', 'elasticsearch']:
-            _service_endpoints_template[key] = '%s:%s' % (value.rpartition(':')[0], EDGE_PORT)
-
-
-def get_service_endpoint(service, localstack_host=None):
+        if key not in ["dashboard", "elasticsearch"]:
+            _service_endpoints_template[key] = f"{value.rpartition(':')[0]}:{EDGE_PORT}"
+
+
+def get_service_endpoint(service: str, localstack_host: str = None) -> str:
+    """
+    Return the local endpoint URL for the given boto3 service (e.g., "s3").
+    If $AWS_ENDPOINT_URL is configured in the environment, it is returned directly.
+    Otherwise, the service endpoint is constructed from the dict of service ports (usually http://localhost:4566).
+    """
+    env_endpoint_url = os.environ.get("AWS_ENDPOINT_URL", "").strip()
+    if env_endpoint_url:
+        return env_endpoint_url
     endpoints = get_service_endpoints(localstack_host=localstack_host)
     return endpoints.get(service)
 
 
-def get_service_endpoints(localstack_host=None):
+def get_service_endpoints(localstack_host: str = None) -> Dict[str, str]:
     if localstack_host is None:
-        localstack_host = os.environ.get('LOCALSTACK_HOST', 'localhost')
-    protocol = 'https' if os.environ.get('USE_SSL') in ('1', 'true') else 'http'
+        localstack_host = os.environ.get("LOCALSTACK_HOST", "localhost")
+    protocol = "https" if os.environ.get("USE_SSL") in ("1", "true") else "http"
 
-    return json.loads(json.dumps(_service_endpoints_template)
-        .replace('{proto}', protocol).replace('{host}', localstack_host))
+    return json.loads(
+        json.dumps(_service_endpoints_template)
+        .replace("{proto}", protocol)
+        .replace("{host}", localstack_host)
+    )
 
 
-def get_service_port(service):
+def get_service_port(service: str) -> int:
     ports = get_service_ports()
     return ports.get(service)
 
 
-def get_service_ports():
+def get_service_ports() -> Dict[str, int]:
     endpoints = get_service_endpoints()
     result = {}
     for service, url in endpoints.items():
         result[service] = urlparse(url).port
     return result
-
-
-def patch_expand_host_prefix():
-    """Apply a patch to botocore, to skip adding host prefixes to endpoint URLs"""
-
-    def _expand_host_prefix(self, parameters, operation_model, *args, **kwargs):
-        result = _expand_host_prefix_orig(self, parameters, operation_model, *args, **kwargs)
-        # skip adding host prefixes, to avoid making requests to, e.g., http://data-localhost:4566
-        if operation_model.service_model.service_name == "servicediscovery" and result == "data-":
-            return None
-        if operation_model.service_model.service_name == "mwaa" and result == "api.":
-            return None
-        return result
-
-    _expand_host_prefix_orig = Serializer._expand_host_prefix
-    Serializer._expand_host_prefix = _expand_host_prefix