From 36abe29c64b4c396d77c839213963f967d21c086 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Fri, 9 Sep 2022 17:03:42 -0400 Subject: [PATCH 1/4] pyproject.toml --- .dockerignore | 6 +++ setup.cfg => .flake8 | 10 ----- pyproject.toml | 80 ++++++++++++++++++++++++++++++++++++ setup.py | 96 -------------------------------------------- 4 files changed, 86 insertions(+), 106 deletions(-) create mode 100644 .dockerignore rename setup.cfg => .flake8 (55%) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6cbf22d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git +*tests +**/tests +*.pyc +**/*.pyc +docker* diff --git a/setup.cfg b/.flake8 similarity index 55% rename from setup.cfg rename to .flake8 index 061896b..3438578 100644 --- a/setup.cfg +++ b/.flake8 @@ -1,16 +1,6 @@ -[metadata] -license_file = LICENSE - -[bdist_wheel] -universal=1 - [flake8] ignore = E203, E266, E501, W503, E302 max-line-length = 110 max-complexity = 18 select = B,C,E,F,W,T4,B9 exclude = .git,.ropeproject,.tox,build,env,venv,__pycache__ - -[tool:pytest] -DJANGO_SETTINGS_MODULE=tests.settings -addopts = -v --tb=short diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b9809b8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,80 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "django-elasticsearch-metrics" +description="Django app for storing time-series metrics in Elasticsearch." +requires-python = ">=3.7" +license = { file = "LICENSE" } +readme = "README.md" +dependencies = [ + "elasticsearch-dsl>=6.3,<7", +] +keywords = [ + "django", + "elastic", + "elasticsearch", + "elasticsearch-dsl", + "time-series", + "metrics", + "statistics", +] +classifiers = [ + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Framework :: Django", + "Framework :: Django :: 1.11", + "Framework :: Django :: 2.0", + "Framework :: Django :: 2.1", + "Framework :: Django :: 2.2", + "Framework :: Django :: 3.0", + "Framework :: Django :: 3.1", + "Framework :: Django :: 3.2", + "Framework :: Django :: 4.0", + "Framework :: Django :: 4.1", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries :: Python Modules", +] +authors = [ + { name = "Steven Loria", email = "steve@cos.io" }, + { name = "Dawn Pattison", email = "pattison.dawn@cos.io" }, +] +maintainers = [ + { name = "Abram Booth", email = "abram@cos.io" }, +] +dynamic = ["version"] + +[project.urls] +Issues = "https://github.com/CenterForOpenScience/django-elasticsearch-metrics/issues" +Changelog = "https://github.com/CenterForOpenScience/django-elasticsearch-metrics/blob/master/CHANGELOG.md" + +[project.optional-dependencies] +dev = [ + "pytest", + "mock", + "pytest-django==3.10.0", + "factory-boy==2.11.1", + "flake8==5.0.4", + "flake8-bugbear==18.8.0", + "pre-commit==2.17.0", + "konch>=3.0", + "tox", +] + +[tool.setuptools.dynamic] +version = { attr = "elasticsearch_metrics.__version__" } + +[tool.setuptools.packages.find] +exclude = ["tests*"] + +[tool.pytest.ini_options] +DJANGO_SETTINGS_MODULE = "tests.settings" +addopts = "-v --tb=short" diff --git a/setup.py b/setup.py deleted file mode 100644 index 596035a..0000000 --- a/setup.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import re -import os -from setuptools import setup, find_packages - -EXTRAS_REQUIRE = { - "tests": ["pytest", "mock", "pytest-django==3.10.0", "factory-boy==2.11.1"], - "lint": [ - "flake8==5.0.4", - 'flake8-bugbear==18.8.0; python_version >= "3.5"', - "pre-commit==2.17.0", - ], -} -EXTRAS_REQUIRE["dev"] = ( - EXTRAS_REQUIRE["tests"] + EXTRAS_REQUIRE["lint"] + ["konch>=3.0.0", "tox"] -) - - -def find_version(fname): - """Attempts to find the version number in the file names fname. - Raises RuntimeError if not found. - """ - version = "" - with open(fname, "r") as fp: - reg = re.compile(r'__version__ = [\'"]([^\'"]*)[\'"]') - for line in fp: - m = reg.match(line) - if m: - version = m.group(1) - break - if not version: - raise RuntimeError("Cannot find version information") - return version - - -def read(fname): - with open(fname) as fp: - content = fp.read() - return content - - -setup( - name="django-elasticsearch-metrics", - version=find_version(os.path.join("elasticsearch_metrics", "__init__.py")), - author="Steven Loria, Dawn Pattison", - author_email="steve@cos.io, pattison.dawn@cos.io", - description="Django app for storing time-series metrics in Elasticsearch.", - long_description=read("README.md"), - long_description_content_type="text/markdown", - url="http://github.com/CenterForOpenScience/django-elasticsearch-metrics", - license="MIT", - packages=find_packages(exclude=("tests",)), - keywords=( - "django", - "elastic", - "elasticsearch", - "elasticsearch-dsl", - "time-series", - "metrics", - "statistics", - ), - install_requires=["elasticsearch-dsl>=6.3.0,<7.0.0"], - extras_require=EXTRAS_REQUIRE, - classifiers=[ - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Framework :: Django", - "Framework :: Django :: 1.11", - "Framework :: Django :: 2.0", - "Framework :: Django :: 2.1", - "Framework :: Django :: 2.2", - "Framework :: Django :: 3.0", - "Framework :: Django :: 3.1", - "Framework :: Django :: 3.2", - "Framework :: Django :: 4.0", - "Framework :: Django :: 4.1", - "Environment :: Web Environment", - "Intended Audience :: Developers", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - zip_safe=False, - include_package_data=True, - project_urls={ - "Issues": "https://github.com/CenterForOpenScience/django-elasticsearch-metrics/issues", - "Changelog": "https://github.com/CenterForOpenScience/django-elasticsearch-metrics/blob/master/CHANGELOG.md", - }, -) From 5d7751904de42e6caa3cb836bcf23649e14151b0 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Fri, 9 Sep 2022 17:04:24 -0400 Subject: [PATCH 2/4] test against es6,7,8 --- .github/workflows/test_djelme.yml | 44 ++++++++++++++++--------------- docker-compose.yml | 26 ++++++++++++++---- testbox.Dockerfile | 2 ++ tox.ini | 11 +++++--- 4 files changed, 54 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test_djelme.yml b/.github/workflows/test_djelme.yml index d4f7841..e6936f6 100644 --- a/.github/workflows/test_djelme.yml +++ b/.github/workflows/test_djelme.yml @@ -11,44 +11,46 @@ permissions: jobs: lint: runs-on: ubuntu-latest + container: python:3.7-slim steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - id: setup-py - with: - python-version: '3.7' - - run: pip install -U tox - - name: alias pythonX.Y for tox - run: alias python${{ steps.setup-py.outputs.python-version }}=${{ steps.setup-py.outputs.python-path }} + - run: pip install tox - run: TOXENV=lint tox test: strategy: matrix: - python: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python: ['3.7', '3.8', '3.9', '3.10'] django: ['1.11', '2.0', '2.1', '2.2', '3.0', '3.1', '3.2', '4.0', '4.1'] - # TODO: elasticsearch: ['6', '7', '8', '9'] + elastic: ['6', '7', '8'] exclude: - - {python: '3.6', django: '4.0'} - - {python: '3.6', django: '4.1'} - {python: '3.7', django: '4.0'} - {python: '3.7', django: '4.1'} - {python: '3.10', django: '1.11'} - {python: '3.10', django: '2.0'} runs-on: ubuntu-latest + container: python:${{matrix.python}}-slim services: - elasticsearch: - image: elasticsearch:6.8.23 - ports: - - 9201:9200 + elasticsearch6: + image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23 + env: + discovery.type: single-node + elasticsearch7: + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.6 + env: + discovery.type: single-node + elasticsearch8: + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.1 + env: + discovery.type: single-node + ELASTIC_CLIENT_APIVERSIONING: 1 # act like es7 + xpack.security.enabled: "false" # it'll be fine + action.destructive_requires_name: "false" # we'll delete * in a moment + env: + ELASTICSEARCH_HOST: elasticsearch${{matrix.elastic}}:9200 steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - id: setup-py - with: - python-version: ${{ matrix.python }} - run: pip install -U tox - - run: alias python${{ steps.setup-py.outputs.python-version }}=${{ steps.setup-py.outputs.python-path }} - - run: TOXENV=`echo 'py${{ matrix.python }}-django${{matrix.django}}' | sed 's/\.//g'` tox + - run: TOXENV=`echo 'py${{matrix.python}}-django${{matrix.django}}-es${{matrix.elastic}}' | sed 's/\.//g'` tox diff --git a/docker-compose.yml b/docker-compose.yml index d84b530..3359b2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,31 @@ version: '3' services: - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:6.3.1 + elasticsearch6: + image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23 + environment: + discovery.type: single-node + elasticsearch7: + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.6 + environment: + discovery.type: single-node + elasticsearch8: + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.1 + environment: + discovery.type: single-node + ELASTIC_CLIENT_APIVERSIONING: 1 # act like es7 + xpack.security.enabled: "false" + action.destructive_requires_name: "false" testbox: build: context: . dockerfile: testbox.Dockerfile depends_on: - - elasticsearch + # - elasticsearch6 + # - elasticsearch7 + - elasticsearch8 environment: - TOXENV: py39-django40 - ELASTICSEARCH_HOST: elasticsearch:9200 + # ELASTICSEARCH_HOST: elasticsearch6:9200 + # ELASTICSEARCH_HOST: elasticsearch7:9200 + ELASTICSEARCH_HOST: elasticsearch8:9200 volumes: - ./:/code:cached diff --git a/testbox.Dockerfile b/testbox.Dockerfile index 15b5ad0..67cce5a 100644 --- a/testbox.Dockerfile +++ b/testbox.Dockerfile @@ -5,5 +5,7 @@ WORKDIR /code COPY ./ /code RUN pip install .[dev] +RUN pip install Django==3.2.15 +ENV TOXENV py39-django32-es7 CMD ["tox"] diff --git a/tox.ini b/tox.ini index e3ed29e..51925e3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,16 @@ [tox] +isolated_build = True envlist = lint - py{36,37,38,39,310}-django{111,20,21,22,30,31,32,40,41} + py{37,38,39,310}-django{111,20,21,22,30,31,32,40,41}-es{6,7} [testenv] -passenv=ELASTICSEARCH_HOST -extras = tests +passenv = ELASTICSEARCH_HOST +extras = + dev + es6: es6 + es7: es7 + es8: es8 deps = django111: Django>=1.11,<1.12 django20: Django>=2.0,<2.1 From a262459ef0a1133081d915fb5207bfc8027f0890 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Fri, 9 Sep 2022 17:04:44 -0400 Subject: [PATCH 3/4] pass tests --- elasticsearch_metrics/metrics.py | 2 +- tests/conftest.py | 2 +- tests/test_metrics.py | 27 +++++++++++++++++---------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/elasticsearch_metrics/metrics.py b/elasticsearch_metrics/metrics.py index ed1221a..cbefe56 100644 --- a/elasticsearch_metrics/metrics.py +++ b/elasticsearch_metrics/metrics.py @@ -164,7 +164,7 @@ def check_index_template(cls, using=None): """ client = connections.get_connection(using or "default") try: - template = client.indices.get_template(cls._template_name) + template = client.indices.get_template(name=cls._template_name) except NotFoundError as client_error: template_name = cls._template_name metric_name = cls.__name__ diff --git a/tests/conftest.py b/tests/conftest.py index ea95af8..f042419 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,7 +19,7 @@ def _es_marker(request, client): def teardown_es(): client.indices.delete(index="*") - client.indices.delete_template("*") + client.indices.delete_template(name="*") teardown_es() yield diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 8fe1ed4..75d0eab 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -27,6 +27,11 @@ ) +# support elasticsearch-dsl 6 and 7 +def mappings_fortytwo(mapping): + return mapping.get("doc", mapping) + + class PreprintView(metrics.Metric): provider_id = metrics.Keyword(index=True) user_id = metrics.Keyword(index=True) @@ -95,9 +100,9 @@ def test_get_index_template_respects_index_settings(self): def test_get_index_template_creates_template_with_mapping(self): template = PreprintView.get_index_template() - mappings = template.to_dict()["mappings"] - assert mappings["doc"]["_source"]["enabled"] is False - properties = mappings["doc"]["properties"] + mappings = mappings_fortytwo(template.to_dict()["mappings"]) + assert mappings["_source"]["enabled"] is False + properties = mappings["properties"] assert "timestamp" in properties assert properties["timestamp"] == {"doc_values": True, "type": "date"} assert properties["provider_id"] == {"type": "keyword", "index": True} @@ -108,10 +113,12 @@ def test_get_index_template_creates_template_with_mapping(self): def test_mappings_are_not_shared(self): template1 = DummyMetric.get_index_template() template2 = DummyMetricWithExplicitTemplateName.get_index_template() - assert "my_int" in template1.to_dict()["mappings"]["doc"]["properties"] - assert "my_keyword" not in template1.to_dict()["mappings"]["doc"]["properties"] - assert "my_int" not in template2.to_dict()["mappings"]["doc"]["properties"] - assert "my_keyword" in template2.to_dict()["mappings"]["doc"]["properties"] + mappings1 = mappings_fortytwo(template1.to_dict()["mappings"]) + mappings2 = mappings_fortytwo(template2.to_dict()["mappings"]) + assert "my_int" in mappings1["properties"] + assert "my_keyword" not in mappings1["properties"] + assert "my_int" not in mappings2["properties"] + assert "my_keyword" in mappings2["properties"] def test_declaring_metric_with_no_app_label_or_template_name_errors(self): with pytest.raises(RuntimeError): @@ -189,8 +196,8 @@ class Meta: template = MyMetric.get_index_template() template_dict = template.to_dict() - doc = template_dict["mappings"]["doc"] - assert doc["_source"]["enabled"] is True + mappings = mappings_fortytwo(template_dict["mappings"]) + assert mappings["_source"]["enabled"] is True class TestRecord: @@ -264,7 +271,7 @@ def test_init(self, client): PreprintView.init() name = PreprintView.get_index_name() mapping = client.indices.get_mapping(index=name) - properties = mapping[name]["mappings"]["doc"]["properties"] + properties = mappings_fortytwo(mapping[name]["mappings"])["properties"] assert properties["timestamp"] == {"type": "date"} assert properties["provider_id"] == {"type": "keyword"} assert properties["user_id"] == {"type": "keyword"} From 53ac7a47ec97e20d875db644991f066f892aab95 Mon Sep 17 00:00:00 2001 From: Abram Booth Date: Mon, 12 Sep 2022 07:30:00 -0400 Subject: [PATCH 4/4] fix lint git --- .github/workflows/test_djelme.yml | 7 +++++-- .pre-commit-config.yaml | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_djelme.yml b/.github/workflows/test_djelme.yml index e6936f6..9cbf3e7 100644 --- a/.github/workflows/test_djelme.yml +++ b/.github/workflows/test_djelme.yml @@ -11,10 +11,13 @@ permissions: jobs: lint: runs-on: ubuntu-latest - container: python:3.7-slim steps: - uses: actions/checkout@v3 - - run: pip install tox + - uses: actions/setup-python@v4 + with: + python-version: 3.9 # must match language_version in .pre-commit-config.yaml + - run: alias python${{ steps.setup-py.outputs.python-version }}=${{ steps.setup-py.outputs.python-path }} + - run: pip install -U tox - run: TOXENV=lint tox diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3750d2c..7a9a150 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: rev: 22.8.0 hooks: - id: black - language_version: python3.7 + language_version: python3.9 - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: