From 3bdd4e914d7195dbbd50f0fde59590efa0ca7947 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Wed, 29 Oct 2025 17:47:38 -0400 Subject: [PATCH] Misc updates - Doc formatting fixes - Make quiet=True the default - Rename branch -> checkout - Rename delete -> remove - Update gitignore - Update justfile - Update pyproject.toml --- .gitignore | 3 + .readthedocs.yaml | 25 +- WARP.md | 328 ++++++++++++++++++ django_mongodb_cli/repo.py | 68 ++-- django_mongodb_cli/utils.py | 40 ++- .../test-suites.rst | 40 ++- jira/{qe.py => INTPYTHON-527.py} | 13 +- justfile | 98 ++++-- pyproject.toml | 111 +++--- test/settings/qe.py | 45 +-- 10 files changed, 594 insertions(+), 177 deletions(-) create mode 100644 WARP.md rename jira/{qe.py => INTPYTHON-527.py} (84%) diff --git a/.gitignore b/.gitignore index 9a9d52e..7e54e8f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ django_mongodb_cli/__pycache__/ .idea server.log mongocryptd.pid +.python-version +build/ +uv.lock diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 9112659..18f9ef9 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,22 +1,25 @@ +# .readthedocs.yaml # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required version: 2 -# Set the OS, Python version, and other tools you might need -build: - os: ubuntu-24.04 - tools: - python: "3.13" - -# Build documentation in the "docs/" directory with Sphinx +# Build documentation in the doc/ directory with Sphinx sphinx: + builder: dirhtml configuration: docs/source/conf.py + fail_on_warning: true -# Optionally, but recommended, -# declare the Python requirements required to build your documentation -# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# Set the version of Python and requirements required to build the docs. python: install: - - requirements: docs/requirements.txt + - method: pip + path: . + extra_requirements: + - docs + +build: + os: ubuntu-lts-latest + tools: + python: "3.13" diff --git a/WARP.md b/WARP.md new file mode 100644 index 0000000..18ce422 --- /dev/null +++ b/WARP.md @@ -0,0 +1,328 @@ +# WARP.md + +This file provides guidance to WARP (warp.dev) when working with code in this repository. + +## Tooling and setup + +- This project is a Python package that installs a CLI named `dm` (Django MongoDB CLI). +- The CLI is intended to be used from the repository root, where `pyproject.toml` defines the `[tool.django-mongodb-cli]` configuration. + +### Environment and installation + +From a clean clone: + +```bash path=null start=null +python -m venv .venv +source .venv/bin/activate +pip install -e . +``` + +The docs also define a convenience target using `just` (preferred for local setup): + +```bash path=null start=null +just install +``` + +`just install` will: +- Ensure a virtualenv is active (`check-venv` recipe). +- Install system packages and Python dependencies (including editable install of this package). +- Install `pre-commit` hooks. + +## Common commands + +All commands below assume you are in the repo root with the virtualenv activated. + +### Working with the `dm` CLI + +The entry point for this project is the `dm` command, exposed by the `project.scripts` section of `pyproject.toml` and implemented in `django_mongodb_cli.__init__.py`. + +High-level subcommands: +- `dm app ...` — manage Django apps inside a project (`django_mongodb_cli/app.py`). +- `dm frontend ...` — manage a Django "frontend" app and its Node/npm tooling (`django_mongodb_cli/frontend.py`). +- `dm project ...` — scaffold and manage Django projects (`django_mongodb_cli/project.py`). +- `dm repo ...` — manage and test external Git repos configured in `[tool.django-mongodb-cli]` (`django_mongodb_cli/repo.py`, `django_mongodb_cli/utils.py`). + +#### Project lifecycle + +Project scaffolding and management all assume you are in a workspace where you want to create/run a Django project; commands operate relative to the current directory. + +- Create a new project from the bundled template: + + ```bash path=null start=null + dm project add [--add-frontend] + ``` + + This uses the `project_template` under `django_mongodb_cli/templates` via `django-admin startproject`. It also generates a per-project `pyproject.toml` preconfigured for MongoDB usage and testing. + +- Run a project (using `django-admin` instead of `manage.py`): + + ```bash path=null start=null + dm project run [--frontend] [--mongodb-uri MONGODB_URI] + ``` + + - Uses `DJANGO_SETTINGS_MODULE=.` where `settings_path` comes from `[tool.django-mongodb-cli.project.settings.path]` in the root `pyproject.toml` (defaults to a `settings.base`-style module if not overridden). + - If `--frontend` is passed, it will ensure the frontend is installed (`dm frontend install ...`) and run the chosen npm script alongside the Django server. + - `--mongodb-uri` (or the `MONGODB_URI` environment variable) is passed through to Django via `_build_mongodb_env`. + +- Migrations at the project level: + + ```bash path=null start=null + dm project makemigrations [app_label] [--mongodb-uri ...] + dm project migrate [app_label] [migration_name] [--database NAME] [--mongodb-uri ...] + ``` + +- Run arbitrary `django-admin` commands for a project: + + ```bash path=null start=null + dm project manage [command] [args...] [--mongodb-uri ...] [--database NAME] + ``` + + If `command` is omitted, `django-admin` is invoked with no arguments for the configured project. + +- Non-interactive superuser creation: + + ```bash path=null start=null + dm project su [--username USER] [--password PASS] [--email EMAIL] [--mongodb-uri ...] + ``` + + This uses `DJANGO_SUPERUSER_PASSWORD` and `MONGODB_URI` environment variables under the hood. + +#### App-level commands + +App commands assume an existing Django project directory under the specified `project_name`. + +- Create/remove a Django app using the packaged template: + + ```bash path=null start=null + dm app create [--directory PATH] + dm app remove [--directory PATH] + ``` + +- App-specific migrations (wrappers around `django-admin` with `DJANGO_SETTINGS_MODULE` set to `.settings`): + + ```bash path=null start=null + dm app makemigrations [--directory PATH] + dm app migrate [app_label] [migration_name] [--directory PATH] + ``` + +#### Frontend helpers + +Frontend helpers assume a `frontend` app inside the Django project (or another directory if overridden). + +- Scaffold the frontend app from the bundled template: + + ```bash path=null start=null + dm frontend create [--directory PATH] + ``` + +- Remove the frontend app: + + ```bash path=null start=null + dm frontend remove [--directory PATH] + ``` + +- Install npm dependencies in the frontend directory: + + ```bash path=null start=null + dm frontend install [--frontend-dir frontend] [--directory PATH] [--clean] + ``` + +- Run an npm script in the frontend directory (defaults to `watch`): + + ```bash path=null start=null + dm frontend run [--frontend-dir frontend] [--directory PATH] [--script SCRIPT] + ``` + +### Managing external repos (`dm repo`) + +External repositories and their Git URLs are defined under `[tool.django-mongodb-cli.repos]` in the root `pyproject.toml`. By default they are cloned under `path = "src"` from that same config. + +Key patterns: + +- List known repos from config and filesystem: + + ```bash path=null start=null + dm repo --list-repos + ``` + +- Clone one or more configured repos (optionally installing their Python packages): + + ```bash path=null start=null + dm repo clone [--install] + dm repo clone --all-repos [--install] + ``` + + Clone behavior (paths, branches, etc.) is driven by `Repo.get_map()` and `Repo.parse_git_url()` in `django_mongodb_cli/utils.py`. + +- Set up Git remotes and defaults via `dm repo remote` and `dm repo set-default` (these are wrapped in convenient `just git-remote` recipes for common groups like `django`, `langchain`, `mongo-arrow`). + +- Inspect and maintain repos: + + ```bash path=null start=null + dm repo status [--all-repos] + dm repo diff [--all-repos] + dm repo fetch [--all-repos] + dm repo pull [--all-repos] + dm repo push [--all-repos] + dm repo log [--all-repos] + dm repo open [--all-repos] + dm repo reset [--all-repos] + ``` + +- Manage branches for a repo or across repos: + + ```bash path=null start=null + dm repo checkout [branch_name] [--list-branches] [--all-repos] [--delete-branch] [--cloned-only] + ``` + +- Create Evergreen patches using project configuration in `[tool.django-mongodb-cli.evergreen.]`: + + ```bash path=null start=null + dm repo patch + ``` + +- Create GitHub PRs for a repo using `gh pr create`: + + ```bash path=null start=null + dm repo pr [--all-repos] + ``` + +### Running tests + +Tests are generally run *in external repositories* managed by this CLI; there are no standalone tests for the CLI package itself. + +#### Running test suites for a configured repo + +Testing behavior is driven by the `[tool.django-mongodb-cli.test.]` blocks in `pyproject.toml`. `django_mongodb_cli.utils.Test` reads this configuration to decide: +- Which command to run (`pytest`, `./runtests.py`, or `just`). +- Which directory to run tests from (`test_dir`). +- Any additional options (`test_options`, `env_vars`, `settings.module`, etc.). + +The high-level command is: + +```bash path=null start=null +dm repo test [modules...] [--keepdb] [--keyword PATTERN] [--list-tests] [--setenv] [--mongodb-uri URI] +``` + +Notes: +- If `--mongodb-uri` is provided (or `MONGODB_URI` is already set), it is exported to the environment before running tests. +- If `--list-tests` is passed, the tool recursively walks `test_dir` and prints discovered Python test files instead of executing them. +- If one or more `modules` are given, they are appended to the underlying test command, allowing you to scope test runs to a specific test module or package. + +Example patterns (adapt to a specific repo and path): + +- Run the default test suite for Django itself: + + ```bash path=null start=null + dm repo test django + ``` + +- Run tests for `django-rest-framework` with verbose pytest output (driven by its `test` config in `pyproject.toml`): + + ```bash path=null start=null + dm repo test django-rest-framework + ``` + +- Run a single test module (path is relative to the configured `test_dir` for that repo): + + ```bash path=null start=null + dm repo test path/to/test_module.py + ``` + +- Run tests whose names match a keyword: + + ```bash path=null start=null + dm repo test --keyword "mongo" + ``` + +### Docs (Sphinx) + +The `docs/` tree contains Sphinx documentation for this project (`docs/source/index.rst` is the root). There is both a Sphinx `Makefile` and `just` recipes for common operations. + +- Build HTML docs via Sphinx `Makefile`: + + ```bash path=null start=null + make -C docs html + ``` + +- Or use `just` helpers: + + ```bash path=null start=null + just sphinx-build # build HTML into docs/_build + just sphinx-autobuild # rebuild docs on changes + just sphinx-open # open docs/_build/index.html + just sphinx-clean # remove docs/_build + ``` + +Make sure any Sphinx-specific Python requirements are installed; `docs/requirements.txt` currently lists the extra packages used. + +## Architecture overview + +### Purpose and scope + +The primary purpose of this repository is to provide the `dm` CLI, which helps maintain and test a ecosystem of repositories around `django-mongodb-backend`, third-party Django libraries, and MongoDB-focused integrations. Most of the heavy lifting (tests, app code, etc.) happens in those external repos; this project coordinates their cloning, configuration, and test execution, and it can also scaffold new Django projects and apps that are pre-wired for MongoDB. + +### CLI entrypoint and subcommand layout + +- `django_mongodb_cli/__init__.py` defines the top-level Typer app (`dm`) and attaches four sub-Typer instances: `app`, `frontend`, `project`, and `repo`. +- The CLI uses Typer’s callback mechanism to show help when no subcommand is invoked. +- The executable entrypoint is wired in `pyproject.toml` under `[project.scripts]` as `dm = "django_mongodb_cli:dm"`. + +### Configuration via `pyproject.toml` + +`django_mongodb_cli.utils.Repo` loads the root `pyproject.toml` and extracts the `[tool.django-mongodb-cli]` section into `self._tool_cfg`. This configuration drives almost all higher-level behavior: + +- `path` — base directory where external Git repos are cloned (currently `src`). +- `repos` — list of `"name @ git+ssh://..."` strings parsed into a mapping of logical repo names to Git URLs. +- `install.` — per-repo installation metadata (e.g., `install_dir`, extra `env_vars`). Used by `Package.install_package` when called via `dm repo install` or `dm repo clone --install`. +- `test.` — per-repo test configuration, used by `Test.run_tests` and `dm repo test` to: + - Determine `test_dir` and optional `test_dirs`. + - Specify `test_command` (`pytest`, `./runtests.py`, or `just`). + - Provide additional `test_options`, settings modules, and environment variables. + - Configure templates for MongoDB-specific settings files, migrations, and app configs that are copied into external repos prior to running tests. +- `origin.` and `evergreen.` — control how `Repo.get_repo_origin` rewrites origin URLs and how Evergreen patches are created for selected repos. +- `project.settings.path` — default settings module path used by `dm project` when constructing `DJANGO_SETTINGS_MODULE` (e.g., `settings.qe`). + +Because `Repo`, `Package`, and `Test` all read from this shared configuration, changes to `pyproject.toml` propagate through the tooling without needing to modify CLI code. + +### Repository orchestration (`Repo`, `Package`, `Test`) + +- `django_mongodb_cli/utils.py` defines three core classes: + - `Repo` — generic Git/repo operations (clone, status, diff, log, checkout, fetch, pull, push, reset, open, etc.), plus helpers for listing and mapping repos from config. + - `Package(Repo)` — extends `Repo` to handle installing/uninstalling Python packages from cloned repositories, including support for per-repo install directories and environment variables. + - `Test(Repo)` — extends `Repo` with a testing abstraction that: + - Reads per-repo `test` configuration from `pyproject.toml`. + - Prepares test environments by copying MongoDB-specific settings, migrations, and app configuration into external repos. + - Builds and runs the appropriate test command (pytest, runtests, or just) with optional module filters, `--keepdb`, and keyword expressions. + +- `django_mongodb_cli/repo.py` builds a Typer-based CLI layer on top of these classes. It centralizes common argument patterns via `repo_command`, and each command function (`clone`, `status`, `test`, `patch`, etc.) instantiates and configures the appropriate helper (`Repo`, `Package`, or `Test`). + +This separation lets the CLI stay thin while keeping the operational logic (Git, installs, tests) in `utils.py`. + +### Project and app scaffolding + +- `django_mongodb_cli/project.py` implements the `dm project` subcommands and encapsulates how a Django project is created, configured, and run in a MongoDB-aware way. + - `add` uses `django-admin startproject` with the bundled `project_template` from `django_mongodb_cli/templates/project_template`. + - `_create_pyproject_toml` writes a project-specific `pyproject.toml` into each generated project, pre-populating dependencies (`django-mongodb-backend`, `django-debug-toolbar`, `python-webpack-boilerplate`) and pytest/Django settings. + - `_django_manage_command` wraps `django-admin` for project commands, wiring in `DJANGO_SETTINGS_MODULE` and `PYTHONPATH` using the root `pyproject.toml` configuration and the chosen project name. + - `_build_mongodb_env` centralizes how `MONGODB_URI` is resolved from CLI options vs environment variables. + - `run`, `migrate`, `makemigrations`, `manage`, and `su` are thin wrappers around `_django_manage_command` plus MongoDB-specific environment handling. + +- `django_mongodb_cli/app.py` provides a similar set of thin wrappers for app-level operations within a Django project: + - Uses `django-admin startapp --template` with `django_mongodb_cli.templates.app_template` to generate new apps. + - Provides `makemigrations` and `migrate` wrappers that set `DJANGO_SETTINGS_MODULE` and `PYTHONPATH` appropriately before calling `django-admin`. + +- `django_mongodb_cli/frontend.py` specializes app scaffolding for a `frontend` app that also has Node/npm dependencies: + - `create` scaffolds the app from `templates/frontend_template`. + - `install` and `run` operate on `package.json` and invoke `npm` commands in the configured frontend directory. + +The template directories under `django_mongodb_cli/templates/` (project, app, frontend) are the main extension points for changing the default structure of generated projects/apps. + +### Documentation structure + +- The Sphinx docs under `docs/source/` mirror the conceptual structure of the tool: + - `index.rst` introduces "Django MongoDB CLI" and its purpose (testing Django MongoDB Backend, third-party libraries, and MongoDB’s Django fork). + - `installation.rst` describes the same install flow used above (clone, venv, `pip install -e .`, or `just install`). + - `usage/` and `third-party-library-support/` subtrees document how to use `dm` with various third-party libraries and test suites. + +When modifying or extending CLI behavior, consider updating the corresponding sections in these docs and the `justfile` recipes so that local workflows and documentation stay aligned. diff --git a/django_mongodb_cli/repo.py b/django_mongodb_cli/repo.py index 8d14566..ccc49ca 100644 --- a/django_mongodb_cli/repo.py +++ b/django_mongodb_cli/repo.py @@ -3,7 +3,7 @@ from .utils import Package, Repo, Test -repo = typer.Typer() +repo = typer.Typer(help="Manage Git repositories") repo_remote = typer.Typer() repo.add_typer(repo_remote, name="remote", help="Manage Git repositories") @@ -35,9 +35,7 @@ def main( list_repos: bool = typer.Option( False, "--list-repos", "-l", help="List available repositories." ), - quiet: bool = typer.Option( - False, "--quiet", "-q", help="Suppress output messages." - ), + quiet: bool = typer.Option(True, "--quiet", "-q", help="Suppress output messages."), ): if list_repos: Repo().list_repos() @@ -104,7 +102,28 @@ def remote_remove( @repo.command() -def branch( +def cd( + ctx: typer.Context, + repo_name: str = typer.Argument(None), +): + """ + Change directory to the specified repository. + """ + repo = Repo() + repo.ctx = ctx + + repo_command( + False, + repo_name, + all_msg=None, + missing_msg="Please specify a repository name.", + single_func=repo.cd_repo, + all_func=repo.cd_repo, + ) + + +@repo.command() +def checkout( ctx: typer.Context, repo_name: str = typer.Argument(None), branch_name: str = typer.Argument(None, help="Branch name"), @@ -129,7 +148,14 @@ def branch( repo.ctx = ctx repo_list = repo.map - # Repo().checkout_branch(repo_name, branch_name) + if (all_repos and not list_branches) or (all_repos and repo_name): + typer.echo( + typer.style( + "Cannot use --all-repos with repo name or without --list-branches.", + fg=typer.colors.RED, + ) + ) + raise typer.Exit() if delete_branch and branch_name: repo.delete_branch(repo_name, branch_name) @@ -141,34 +167,13 @@ def branch( all_repos, repo_name, all_msg=None, - missing_msg="Please specify a repository name or use -a,--all-repos to show branches of all repositories.", + missing_msg="Please specify a repository name or use -a,--all-repos with -l,list-repos to show branches of all repositories.", single_func=lambda repo_name: repo.get_repo_branch(repo_name, branch_name), all_func=lambda repo_name: repo.get_repo_branch(repo_name, branch_name), repo_list=repo_list, ) -@repo.command() -def cd( - ctx: typer.Context, - repo_name: str = typer.Argument(None), -): - """ - Change directory to the specified repository. - """ - repo = Repo() - repo.ctx = ctx - - repo_command( - False, - repo_name, - all_msg=None, - missing_msg="Please specify a repository name.", - single_func=repo.cd_repo, - all_func=repo.cd_repo, - ) - - @repo.command() def clone( repo_name: str = typer.Argument(None), @@ -230,7 +235,8 @@ def do_commit(name): @repo.command() -def delete( +def remove( + ctx: typer.Context, repo_name: str = typer.Argument(None), all_repos: bool = typer.Option( False, "--all-repos", "-a", help="Delete all repositories" @@ -244,11 +250,13 @@ def delete( If --all-repos is used, delete all repositories. If --uninstall is used, uninstall the package before deleting. """ + repo = Repo() + repo.ctx = ctx def do_delete(name): if uninstall: Package().uninstall_package(name) - Repo().delete_repo(name) + repo.delete_repo(name) repo_command( all_repos, diff --git a/django_mongodb_cli/utils.py b/django_mongodb_cli/utils.py index 1354857..513a1cb 100644 --- a/django_mongodb_cli/utils.py +++ b/django_mongodb_cli/utils.py @@ -55,9 +55,15 @@ def err(self, text: str) -> None: def title(self, text: str) -> None: typer.echo(text) - def run(self, args, cwd: Path | str | None = None, check: bool = True) -> bool: + def run( + self, + args, + cwd: Path | str | None = None, + check: bool = True, + env: str | None = None, + ) -> bool: try: - subprocess.run(args, cwd=str(cwd) if cwd else None, check=check) + subprocess.run(args, cwd=str(cwd) if cwd else None, check=check, env=env) return True except subprocess.CalledProcessError as e: self.err(f"Command failed: {' '.join(str(a) for a in args)} ({e})") @@ -68,7 +74,7 @@ def ensure_repo( ) -> tuple[Path | None, GitRepo | None]: path = self.get_repo_path(repo_name) if must_exist and not path.exists(): - if not self.ctx.obj.get("quiet", False): + if not self.ctx.obj.get("quiet", True): self.err(f"Repository '{repo_name}' not found at path: {path}") return None, None repo = self.get_repo(str(path)) if path.exists() else None @@ -221,6 +227,7 @@ def delete_repo(self, repo_name: str) -> None: self.info(f"Deleting repository: {repo_name}") path, _ = self.ensure_repo(repo_name) if not path: + self.err(f"❌ Failed to delete {repo_name}: path not found.") return try: shutil.rmtree(path) @@ -278,7 +285,7 @@ def get_repo_remote(self, repo_name: str) -> None: if not repo: return - quiet = self.ctx.obj.get("quiet", False) + quiet = self.ctx.obj.get("quiet", True) self.info(f"Remotes for {repo_name}:") for remote in repo.remotes: try: @@ -506,9 +513,13 @@ def open_repo(self, repo_name: str) -> None: self.ok(f"✅ Successfully opened {repo_name} in browser.") def reset_repo(self, repo_name: str) -> None: - self.info(f"Resetting repository: {repo_name}") _, repo = self.ensure_repo(repo_name) + quiet = self.ctx.obj.get("quiet", True) + if not quiet: + self.info(f"Resetting repository: {repo_name}") if not repo: + if not quiet: + self.err(f"❌ Failed to reset {repo_name}: path not found.") return try: repo.git.reset("--hard") @@ -570,7 +581,7 @@ def remote_add(self, remote_name: str, remote_url: str) -> None: try: repo.create_remote(remote_name, remote_url) self.ok( - f"✅ Successfully added remote '{remote_name}' with URL '{remote_url}'." + f"Successfully added remote '{remote_name}' with URL '{remote_url}'." ) except Exception: self.info( @@ -579,7 +590,7 @@ def remote_add(self, remote_name: str, remote_url: str) -> None: repo.delete_remote(remote_name) repo.create_remote(remote_name, remote_url) self.ok( - f"✅ Successfully added remote '{remote_name}' with URL '{remote_url}'." + f"Successfully added remote '{remote_name}' with URL '{remote_url}'." ) def remote_remove(self, remote_name: str) -> None: @@ -634,7 +645,16 @@ def install_package(self, repo_name: str) -> None: path = Path(path / install_dir).resolve() self.info(f"Using custom install directory: {path}") - if self.run(["uv", "pip", "install", "-e", str(path)]): + env = os.environ.copy() + env_vars_list = ( + self.tool_cfg.get("install", {}).get(repo_name, {}).get("env_vars") + ) + if env_vars_list: + typer.echo("Setting environment variables for installation:") + typer.echo(env_vars_list) + env.update({item["name"]: str(item["value"]) for item in env_vars_list}) + + if self.run(["uv", "pip", "install", "-e", str(path)], env=env): self.ok(f"Installed {repo_name}") def uninstall_package(self, repo_name: str) -> None: @@ -751,7 +771,7 @@ def _list_tests(self, repo_name: str) -> None: test_files = [ f for f in files if f.endswith(".py") and not f.startswith("__") ] - quiet = self.ctx.obj.get("quiet", False) + quiet = self.ctx.obj.get("quiet", True) if not quiet or test_files: self.ok(f"\n📂 {display_path}") @@ -826,9 +846,9 @@ def run_tests(self, repo_name: str) -> None: self._list_tests(repo_name) return - self.info(f"Running tests for repository: {repo_name}") path, _ = self.ensure_repo(repo_name) if not path: + self.err(f"❌ Failed to run tests for {repo_name}: path not found.") return self._run_tests(repo_name) diff --git a/docs/source/third-party-library-support/test-suites.rst b/docs/source/third-party-library-support/test-suites.rst index 3212384..e013d5f 100644 --- a/docs/source/third-party-library-support/test-suites.rst +++ b/docs/source/third-party-library-support/test-suites.rst @@ -1,30 +1,32 @@ -.. _test_suites: - +=========== Test suites ------------ +=========== + +For each third-party library that is supported, the following tasks are performed: + +#. **Configure the test suite to run with Django MongoDB Backend** -For each third party library that is supported, the following tasks are performed: + a. Evaluate test runner configuration -#. **The test suite is configured to run with Django MongoDB Backend.** + i. Depending on the test runner, updating the settings may require + copying ``mongo_apps.py`` and ``mongo_migrations`` to a module + that is already included in ``sys.path``. - a. Evaluate test runner configuration + b. Update Django settings - i. Depending on the test runner, updating the settings may require - copying ``mongo_apps.py`` and ``mongo_migrations`` to a module that is - already included in ``sys.path``. + i. Replace the database backend with ``django_mongodb_backend`` + #. Replace contrib apps with MongoDB-compatible apps + #. Replace test suite apps with MongoDB-compatible apps - b. Update django settings + c. Update or disable migrations - i. Replace the database backend with ``django_mongodb_backend`` - #. Replace contrib apps with MongoDB compatible apps - #. Replace test suite apps with MongoDB compatible apps + i. Use MongoDB-compatible migrations if not disabled - c. Update or disable migrations +#. **Run the test suite with Django MongoDB Backend configured** - i. Use MongoDB compatible migrations if not disabled +#. **Log the test run results** -2. **The test suite is run with Django MongoDB Backend configured.** -#. **The test run results are logged.** -#. **The test suite tests are updated as needed.** +#. **Update test suite tests as needed** - a. Replace static primary key references with dynamic references or static ``ObjectId`` references + a. Replace static primary key references with dynamic references + or static ``ObjectId`` references diff --git a/jira/qe.py b/jira/INTPYTHON-527.py similarity index 84% rename from jira/qe.py rename to jira/INTPYTHON-527.py index d55e19f..37e5bf2 100644 --- a/jira/qe.py +++ b/jira/INTPYTHON-527.py @@ -2,21 +2,22 @@ from pymongo import MongoClient from pymongo.encryption import ClientEncryption, AutoEncryptionOpts from pymongo.errors import EncryptedCollectionError +import os -from django_mongodb_backend.encryption import KMS_PROVIDERS - -KEY_VAULT_NAMESPACE = "encryption.__keyVault" client = MongoClient( auto_encryption_opts=AutoEncryptionOpts( - key_vault_namespace=KEY_VAULT_NAMESPACE, - kms_providers=KMS_PROVIDERS, + key_vault_namespace="encryption.__keyVault", + kms_providers={"local": {"key": os.urandom(96)}}, ) ) codec_options = CodecOptions() client_encryption = ClientEncryption( - KMS_PROVIDERS, KEY_VAULT_NAMESPACE, client, codec_options + client.options.auto_encryption_opts._kms_providers, + client.options.auto_encryption_opts._key_vault_namespace, + client, + codec_options, ) COLLECTION_NAME = "patient" diff --git a/justfile b/justfile index d2b89b6..21ce45f 100644 --- a/justfile +++ b/justfile @@ -1,20 +1,71 @@ default: echo 'Hello, world!' -install: pip-install git-clone -alias i := install - # ---------------------------------------- git ---------------------------------------- [group('git')] -git-clone: - dm repo clone django --install - dm repo clone django-mongodb-backend --install - dm repo clone django-mongodb-demo --install - dm repo clone django-mongodb-extensions --install - dm repo clone drivers-evergreen-tools - dm repo clone libmongocrypt --install - dm repo clone mongo-python-driver --install +git-clone repo: + @if [ "{{repo}}" = "django" ]; then \ + dm repo clone django --install; \ + dm repo clone django-mongodb-backend --install; \ + dm repo clone django-mongodb-extensions --install; \ + dm repo clone libmongocrypt --install; \ + dm repo clone mongo-python-driver --install; \ + elif [ "{{repo}}" = "langchain" ]; then \ + dm repo clone langchain-mongodb; \ + dm repo clone pymongo-search-utils; \ + elif [ "{{repo}}" = "mongo-arrow" ]; then \ + dm repo clone mongo-arrow --install; \ + else \ + echo "Please provide a valid repo name: django-mongodb-backend, django-mongodb-extensions, or mongo-python-driver"; \ + exit 1; \ + fi + +[group('git')] +git-remote repo: + @if [ "{{repo}}" = "django" ]; then \ + echo "Setting remotes for django-mongodb-backend"; \ + dm repo remote django-mongodb-backend add origin git+ssh://git@github.com/aclark4life/django-mongodb-backend; \ + dm repo remote django-mongodb-backend add upstream git+ssh://git@github.com/mongodb/django-mongodb-backend; \ + dm repo set-default django-mongodb-backend; \ + dm repo fetch django-mongodb-backend; \ + dm repo remote django-mongodb-extensions add origin git+ssh://git@github.com/aclark4life/django-mongodb-extensions; \ + dm repo remote django-mongodb-extensions add upstream git+ssh://git@github.com/mongodb/django-mongodb-extensions; \ + dm repo set-default django-mongodb-extensions; \ + dm repo fetch django-mongodb-extensions; \ + elif [ "{{repo}}" = "django-mongodb-extensions" ]; then \ + echo "Setting remotes for django-mongodb-extensions"; \ + dm repo remote django-mongodb-extensions add origin git+ssh://git@github.com/aclark4life/django-mongodb-extensions; \ + dm repo remote django-mongodb-extensions add upstream git+ssh://git@github.com/mongodb-labs/django-mongodb-extensions; \ + dm repo set-default django-mongodb-extensions; \ + dm repo fetch django-mongodb-extensions; \ + elif [ "{{repo}}" = "mongo-python-driver" ]; then \ + echo "Setting remotes for mongo-python-driver"; \ + dm repo remote mongo-python-driver add origin git+ssh://git@github.com/aclark4life/mongo-python-driver; \ + dm repo remote mongo-python-driver add upstream git+ssh://git@github.com/mongodb/mongo-python-driver; \ + dm repo set-default mongo-python-driver; \ + dm repo fetch mongo-python-driver; \ + elif [ "{{repo}}" = "langchain" ]; then \ + echo "Setting remotes for langchain-mongodb"; \ + dm repo remote langchain-mongodb add origin git+ssh://git@github.com/aclark4life/langchain-mongodb; \ + dm repo remote langchain-mongodb add upstream git+ssh://git@github.com/langchain-ai/langchain-mongodb; \ + dm repo set-default langchain-mongodb; \ + dm repo fetch langchain-mongodb; \ + echo "Setting remotes for pymongo-search-utils"; \ + dm repo remote pymongo-search-utils add origin git+ssh://git@github.com/aclark4life/pymongo-search-utils; \ + dm repo remote pymongo-search-utils add upstream git+ssh://git@github.com/mongodb-labs/pymongo-search-utils; \ + dm repo set-default pymongo-search-utils; \ + dm repo fetch pymongo-search-utils; \ + elif [ "{{repo}}" = "mongo-arrow" ]; then \ + echo "Setting remotes for pymongoarrow"; \ + dm repo remote mongo-arrow add origin git+ssh://git@github.com/aclark4life/mongo-arrow; \ + dm repo remote mongo-arrow add upstream git+ssh://git@github.com/mongodb-labs/mongo-arrow; \ + dm repo set-default mongo-arrow; \ + dm repo fetch mongo-arrow; \ + else \ + echo "Please provide a valid repo name: django-mongodb-backend, django-mongodb-extensions, or mongo-python-driver"; \ + exit 1; \ + fi # ---------------------------------------- django ---------------------------------------- @@ -41,16 +92,22 @@ alias su := django-createsuperuser # ---------------------------------------- mongodb ---------------------------------------- [group('mongodb')] -db-init: - # mongosh `echo ${MONGODB_URI}` --eval 'db.dropDatabase()' - mongosh `echo mongodb://0.0.0.0/backend` --eval 'db.dropDatabase()' +drop db: + @if [ -z "{{ db }}" ]; then \ + echo "Please provide a database name using the 'db' parameter."; \ + exit 1; \ + else \ + echo "Dropping database: {{ db }}"; \ + mongosh --eval "db.dropDatabase()"; \ + fi +alias d := drop # ---------------------------------------- python ---------------------------------------- # install python dependencies and activate pre-commit hooks [group('python')] pip-install: check-venv - # brew install libxml2 libxmlsec1 pkg-config + brew install libxml2 libxmlsec1 mongo-c-driver mongo-c-driver@1 pkg-config pip install lxml==5.3.2 --no-binary :all: pip install -U pip pip install -e . @@ -68,9 +125,9 @@ check-venv: exit 1 fi -[group('npm')] -npm-install: - npm install +[group('python')] +install: pip-install +alias i := install # ---------------------------------------- sphinx ---------------------------------------- @@ -97,11 +154,12 @@ alias so := sphinx-open # ---------------------------------------- jira ---------------------------------------- +[group('jira')] INTPYTHON-527: - python jira/qe.py - + python jira/INTPYTHON-527.py alias q := INTPYTHON-527 +[group('jira')] PYTHON-5564 group="" package="": python3.10 -m venv .venv python3.10 -m pip install -U pip diff --git a/pyproject.toml b/pyproject.toml index 1e564b3..68e5b65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,13 @@ name = "django-mongodb-cli" version = "0.1.0" dependencies = [ - # --- allauth --- + "GitPython", + "toml", + "typer", +] + +[project.optional-dependencies] +django-allauth = [ "django-ninja", "fido2", "psycopg2", @@ -10,43 +16,21 @@ dependencies = [ "python3-saml", "pyjwt[crypto]", "requests-oauthlib", - - # --- DRF --- - "pytest-django", - "setuptools", - - # --- debug toolbar --- +] +django-debug-toolbar = [ "django-debug-toolbar", "html5lib", "pytest-django", - - # --- filter --- +] +django-filter = [ "pytz", - - # --- everything else --- - "GitPython", - "Sphinx", - "black", - "django-extensions", - "dj-database-url", - "djhtml", - "faker", - "pymongocrypt", - "pymongo-auth-aws", - "pytest", - "pytest-asyncio", - "pytest-html", - "python-webpack-boilerplate", - "rich", - "sphinx-autobuild", - "sphinx-copybutton", - "toml", - "typer", - "wagtail", ] - -[project.optional-dependencies] +django-rest-framework = [ + "pytest-django", + "setuptools", +] docs = [ + "Sphinx", "furo", "sphinx-copybutton", ] @@ -62,52 +46,58 @@ packages = ["django_mongodb_cli"] [tool.django-mongodb-cli] repos = [ - # 1. Third-Party Libs + "django @ git+ssh://git@github.com/mongodb-forks/django@mongodb-5.2.x", "django-allauth @ git+ssh://git@github.com/pennersr/django-allauth@main", - "xmlsec @ git+ssh://git@github.com/xmlsec/python-xmlsec@main", - "django-rest-framework @ git+ssh://git@github.com/encode/django-rest-framework@main", - "django-filter @ git+ssh://git@github.com/carltongibson/django-filter@main", "django-debug-toolbar @ git+ssh://git@github.com/django-commons/django-debug-toolbar@main", - "python-webpack-boilerplate @ git+ssh://git@github.com/AccordBox/python-webpack-boilerplate@master", - "wagtail @ git+ssh://git@github.com/mongodb-forks/wagtail@main", - "wagtail-mongodb-project @ git+ssh://git@github.com/mongodb-labs/wagtail-mongodb-project@main", - - # 2. Django MongoDB Backend - "django @ git+ssh://git@github.com/mongodb-forks/django@mongodb-5.2.x", + "django-filter @ git+ssh://git@github.com/carltongibson/django-filter@main", "django-mongodb-app @ git+ssh://git@github.com/mongodb-labs/django-mongodb-app@5.2.x", "django-mongodb-backend @ git+ssh://git@github.com/mongodb/django-mongodb-backend@main", + "django-mongodb-demo @ git+ssh://git@github.com/aclark4life/django-mongodb-demo@main", "django-mongodb-extensions @ git+ssh://git@github.com/mongodb-labs/django-mongodb-extensions@main", - "django-mongodb-polls @ git+ssh://git@github.com/aclark4life/django-mongodb-polls@main", "django-mongodb-project @ git+ssh://git@github.com/mongodb-labs/django-mongodb-project@5.2.x", "django-mongodb-project-benchmark @ git+ssh://git@github.com/NoahStapp/django-mongodb-backend-benchmark.git@main", - "libmongocrypt @ git+ssh://git@github.com/mongodb-labs/libmongocrypt@master", - "mongo-python-driver @ git+ssh://git@github.com/mongodb/mongo-python-driver@master", - "pymongo-auth-aws @ git+ssh://git@github.com/mongodb/pymongo-auth-aws@master", - - # 3. LangChain - "langchain-mongodb @ git+ssh://git@github.com/langchain-ai/langchain-mongodb@main", - "pymongo-search-utils@ git+ssh://git@github.com/mongodb-labs/pymongo-search-utils@main", - - # 4. MongoDB - "mongo @ git+ssh://git@github.com/mongodb/mongo@master", - "drivers-evergreen-tools @ git+ssh://git@github.com/mongodb-labs/drivers-evergreen-tools@master", + "django-rest-framework @ git+ssh://git@github.com/encode/django-rest-framework@main", "docs @ git+ssh://git@github.com/mongodb/docs@main", "docs-sample-apps @ git+ssh://git@github.com/mongodb/docs-sample-apps@main", + "drivers-evergreen-tools @ git+ssh://git@github.com/mongodb-labs/drivers-evergreen-tools@master", "flask-pymongo @ git+ssh://git@github.com/mongodb-labs/flask-pymongo", + "GenAI-Showcase @ git+ssh://git@github.com/aclark4life/GenAI-Showcase.git@main", + "langchain-mongodb @ git+ssh://git@github.com/langchain-ai/langchain-mongodb@main", + "libmongocrypt @ git+ssh://git@github.com/mongodb-labs/libmongocrypt@master", + "mongo @ git+ssh://git@github.com/mongodb/mongo@master", "mongo-arrow @ git+ssh://git@github.com/mongodb-labs/mongo-arrow@main", "mongo-orchestration @ git+ssh://git@github.com/mongodb-labs/mongo-orchestration@master", + "mongo-python-driver @ git+ssh://git@github.com/mongodb/mongo-python-driver@master", + "pymongo-auth-aws @ git+ssh://git@github.com/mongodb/pymongo-auth-aws@master", + "pymongo-search-utils@ git+ssh://git@github.com/mongodb-labs/pymongo-search-utils@main", + "python-webpack-boilerplate @ git+ssh://git@github.com/AccordBox/python-webpack-boilerplate@master", "specifications @ git+ssh://git@github.com/mongodb/specifications@master", + "test-supercharge-action @ git+ssh://git@github.com/aclark4life/test-supercharge-action.git", + "wagtail @ git+ssh://git@github.com/mongodb-forks/wagtail@main", + "wagtail-mongodb-project @ git+ssh://git@github.com/mongodb-labs/wagtail-mongodb-project@main", "winkerberos @ git+ssh://git@github.com/mongodb-labs/winkerberos@main", - - # 5. Other - "django-mongodb-demo @ git+ssh://git@github.com/aclark4life/django-mongodb-demo@main", - "GenAI-Showcase @ git+ssh://git@github.com/aclark4life/GenAI-Showcase.git@main", + "xmlsec @ git+ssh://git@github.com/xmlsec/python-xmlsec@main", ] path = "src" [tool.django-mongodb-cli.install.libmongocrypt] install_dir = "bindings/python" +[tool.django-mongodb-cli.install.mongo-arrow] +install_dir = "bindings/python" + +[[tool.django-mongodb-cli.install.mongo-arrow.env_vars]] +name = "LDFLAGS" +value = "-L/opt/homebrew/opt/mongo-c-driver@1/lib" + +[[tool.django-mongodb-cli.install.mongo-arrow.env_vars]] +name = "CPPFLAGS" +value = "-I/opt/homebrew/opt/mongo-c-driver@1/include" + +[[tool.django-mongodb-cli.install.mongo-arrow.env_vars]] +name = "PKG_CONFIG_PATH" +value = "/opt/homebrew/opt/mongo-c-driver@1/lib/pkgconfig" + [tool.django-mongodb-cli.install.langchain-mongodb] install_dir = "libs/langchain-mongodb" @@ -115,6 +105,10 @@ install_dir = "libs/langchain-mongodb" test_command = "pytest" test_dir = "src/langchain-mongodb/libs/langchain-mongodb/tests" +[tool.django-mongodb-cli.test.pymongo-search-utils] +test_command = "pytest" +test_dir = "src/pymongo-search-utils/tests" + [tool.django-mongodb-cli.test.mongo-python-driver] test_command = "just" test_dir = "src/mongo-python-driver/test" @@ -340,7 +334,6 @@ migrations = "allauth.mongo_settings" source = "mongo_migrations" target = "src/django-allauth/allauth/mongo_migrations" - [[tool.django-mongodb-cli.origin.django-mongodb-backend]] user = "aclark4life" repo = "git+ssh://git@github.com/aclark4life/django-mongodb-backend" diff --git a/test/settings/qe.py b/test/settings/qe.py index 995c52c..186d05d 100644 --- a/test/settings/qe.py +++ b/test/settings/qe.py @@ -2,16 +2,10 @@ from pymongo.encryption import AutoEncryptionOpts -from django_mongodb_backend.utils import model_has_encrypted_fields MONGODB_URI = os.environ.get("MONGODB_URI", "mongodb://localhost:27017") -KMS_CREDENTIALS = { - "local": { - "key": os.urandom(96), - }, -} DATABASES = { "default": { "ENGINE": "django_mongodb_backend", @@ -30,36 +24,43 @@ "OPTIONS": { "auto_encryption_opts": AutoEncryptionOpts( key_vault_namespace="djangotests_encrypted.__keyVault", - kms_providers=KMS_CREDENTIALS, + kms_providers={ + "local": { + "key": os.urandom(96), + }, + }, ), }, + "KMS_CREDENTIALS": { + "aws": {}, + }, }, } class EncryptedRouter: - def allow_migrate(self, db, app_label, model_name=None, **hints): - if hints.get("model"): - if model_has_encrypted_fields(hints["model"]): - return db == "encrypted" - else: - return db == "default" - return None - def db_for_read(self, model, **hints): - if model_has_encrypted_fields(model): + if model._meta.app_label == "encryption_": return "encrypted" - return "default" - - def kms_provider(self, model): - if model_has_encrypted_fields(model): - return "local" return None db_for_write = db_for_read + def allow_migrate(self, db, app_label, model_name=None, **hints): + # The encryption_ app's models are only created in the encrypted + # database. + if app_label == "encryption_": + return db == "encrypted" + # Don't create other app's models in the encrypted database. + if db == "encrypted": + return False + return None + + def kms_provider(self, model): + return "local" + -DATABASE_ROUTERS = [EncryptedRouter()] +DATABASE_ROUTERS = ["django_mongodb_backend.routers.MongoRouter", EncryptedRouter()] DEFAULT_AUTO_FIELD = "django_mongodb_backend.fields.ObjectIdAutoField" PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",) SECRET_KEY = "django_tests_secret_key"