feat: migrate standalone scripts to their own uv projects (4/5)

.github/workflows/check-consistent-dependencies.yml

@@ -39,7 +39,7 @@ jobs:
           git fetch origin "$BASE_SHA"
 
           # The ^"? is because git may quote weird file paths
-          if git diff --name-only "$BASE_SHA" | grep -P '^"?((requirements/)|(scripts/.*?/requirements/)|(pyproject\.toml)|(uv\.lock))'; then
+          if git diff --name-only "$BASE_SHA" | grep -P '^"?((requirements/)|(scripts/.*?/requirements/)|(scripts/[^/]+/pyproject\.toml)|(scripts/[^/]+/uv\.lock)|(pyproject\.toml)|(uv\.lock))'; then
             echo "RELEVANT=true" >> "$GITHUB_ENV"
           fi
 

.github/workflows/compile-python-requirements.yml

@@ -57,6 +57,9 @@ jobs:
           branch-suffix: short-commit-hash
           add-paths: |
             requirements
+            scripts/**/pyproject.toml
+            scripts/**/uv.lock
+            scripts/**/requirements*
             pyproject.toml
             uv.lock
           commit-message: |

.github/workflows/upgrade-one-python-dependency.yml

@@ -122,6 +122,8 @@ jobs:
           branch-suffix: short-commit-hash
           add-paths: |
             requirements
+            scripts/**/pyproject.toml
+            scripts/**/uv.lock
             scripts/**/requirements*
           commit-message: |
             feat: Upgrade Python dependency ${{ inputs.package }}

Makefile

@@ -81,21 +81,14 @@ test-requirements: ## install only testing dependencies (used by CI and tox)
 
 requirements: dev-requirements ## install development environment requirements
 
-# The scripts/* one-off script directories are not yet migrated to uv (tracked
-# in https://github.com/openedx/public-engineering/issues/543) and are still
-# compiled with pip-compile below. Order is important: files must appear
-# after everything they include!
-REQ_FILES = \
-	scripts/xblock/requirements \
-	scripts/user_retirement/requirements/base \
-	scripts/user_retirement/requirements/testing \
-	scripts/structures_pruning/requirements/base \
-	scripts/structures_pruning/requirements/testing
-
 # uv-managed sub-projects, each with their own pyproject.toml + uv.lock,
-# independent of the root project's dependency graph.
-UV_SUBPROJECTS = \
-	requirements/edx-sandbox
+# independent of the root project's dependency graph:
+#   requirements/edx-sandbox, scripts/xblock, scripts/user_retirement,
+#   scripts/structures_pruning
+# Their compatibility .txt exports (for anyone still installing via plain
+# pip) don't follow a uniform naming/grouping scheme, so each is handled
+# explicitly in compile-requirements below rather than through one generic
+# loop over a shared list.
 
 define COMMON_CONSTRAINTS_TEMP_COMMENT
 # This is a temporary solution to override the real common_constraints.txt\n# In edx-lint, until the pyjwt constraint in edx-lint has been removed.\n# See BOM-2721 for more details.\n# Below is the copied and edited version of common_constraints\n
@@ -143,33 +136,51 @@ compile-requirements: pre-requirements ## Regenerate uv.lock for the root projec
 	sed 's/pip<25.3//g' requirements/common_constraints.txt > requirements/common_constraints.tmp
 	mv requirements/common_constraints.tmp requirements/common_constraints.txt
 
-	pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip-tools.txt requirements/pip-tools.in
-	pip install -r requirements/pip-tools.txt
-
-	@ export REBUILD='--rebuild'; \
-	for f in $(REQ_FILES); do \
+	@# requirements/edx-sandbox and scripts/xblock: single compat export, no dependency-groups.
+	@for d in requirements/edx-sandbox scripts/xblock; do \
 		echo ; \
-		echo "== $$f ===============================" ; \
-		echo "pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in"; \
-		pip-compile -v $$REBUILD ${COMPILE_OPTS} -o $$f.txt $$f.in || exit 1; \
-		export REBUILD=''; \
+		echo "== $$d ===============================" ; \
+		uv run --no-project --with edx-lint edx_lint write_uv_constraints $$d/pyproject.toml && \
+		(cd $$d && uv lock ${UV_LOCK_OPTS}) \
+		|| exit 1; \
 	done
+	@{ \
+		echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \
+		echo "# Compatibility export for anyone still 'pip install -r requirements/edx-sandbox/base.txt'"; \
+		echo "# directly instead of using uv. Source of truth: requirements/edx-sandbox/pyproject.toml / uv.lock."; \
+		(cd requirements/edx-sandbox && uv export --frozen --no-hashes --no-emit-project); \
+	} > requirements/edx-sandbox/base.txt
+	@{ \
+		echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \
+		echo "# Compatibility export for anyone still 'pip install -r scripts/xblock/requirements.txt'"; \
+		echo "# directly instead of using uv. Source of truth: scripts/xblock/pyproject.toml / uv.lock."; \
+		(cd scripts/xblock && uv export --frozen --no-hashes --no-emit-project); \
+	} > scripts/xblock/requirements.txt
 
-	@for d in $(UV_SUBPROJECTS); do \
+	@# scripts/user_retirement and scripts/structures_pruning: base + testing (test group) compat exports.
+	@for d in scripts/user_retirement scripts/structures_pruning; do \
 		echo ; \
 		echo "== $$d ===============================" ; \
 		uv run --no-project --with edx-lint edx_lint write_uv_constraints $$d/pyproject.toml && \
-		(cd $$d && uv lock ${UV_LOCK_OPTS}) && \
-		( \
+		(cd $$d && uv lock ${UV_LOCK_OPTS}) \
+		|| exit 1; \
+	done
+	@for d in scripts/user_retirement scripts/structures_pruning; do \
+		{ \
 			echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \
-			echo "# Compatibility export for anyone still 'pip install -r $$d/base.txt'"; \
+			echo "# Compatibility export for anyone still 'pip install -r $$d/requirements/base.txt'"; \
 			echo "# directly instead of using uv. Source of truth: $$d/pyproject.toml / uv.lock."; \
 			(cd $$d && uv export --frozen --no-hashes --no-emit-project); \
-		) > $$d/base.txt \
-		|| exit 1; \
+		} > $$d/requirements/base.txt; \
+		{ \
+			echo "# GENERATED FILE, DO NOT EDIT DIRECTLY."; \
+			echo "# Compatibility export for anyone still 'pip install -r $$d/requirements/testing.txt'"; \
+			echo "# directly instead of using uv. Source of truth: $$d/pyproject.toml (test group) / uv.lock."; \
+			(cd $$d && uv export --frozen --no-hashes --group test --no-emit-project); \
+		} > $$d/requirements/testing.txt; \
 	done
 
-upgrade: $(COMMON_CONSTRAINTS_TXT) ## update all dependencies (uv.lock for the root project and uv sub-projects, pip-compile for the not-yet-migrated sub-projects) to the latest releases satisfying our constraints
+upgrade: $(COMMON_CONSTRAINTS_TXT) ## update all dependencies (uv.lock for the root project and all uv sub-projects) to the latest releases satisfying our constraints
 	$(MAKE) compile-requirements COMPILE_OPTS="--upgrade" UV_LOCK_OPTS="--upgrade"
 
 upgrade-package: ## update just one package to the latest usable release

requirements/README.rst

@@ -9,17 +9,21 @@ locked in the root ``uv.lock``, managed with `uv`_.
 
 This ``requirements/`` directory now only holds:
 
-- ``edx-sandbox``, the requirements for Codejail's isolated sandbox environment
+- ``edx-sandbox``, its own standalone ``uv``-managed project (``pyproject.toml`` +
+  ``uv.lock``) for Codejail's isolated sandbox environment
 - ``constraints.txt`` / ``common_constraints.txt`` / ``pip-tools.in`` / ``pip-tools.txt``,
-  still needed to ``pip-compile`` ``edx-sandbox`` and the ``scripts/*`` one-off
-  script directories, none of which are on ``uv`` yet
-
-These are being migrated to their own standalone ``uv``-managed projects too,
-tracked in `public-engineering#543`_. Until that's done, they're still
-manipulated using the Makefile targets below in a Linux environment (to match
-our build and deploy systems); for developers on Mac, this can be achieved by
-using the GitHub workflows or by running Make targets from inside devstack's
-lms-shell or another Linux environment.
+  now unused leftovers from the pip-compile era, pending removal (tracked in
+  `public-engineering#543`_)
+
+The three standalone script directories (``scripts/xblock``,
+``scripts/user_retirement``, ``scripts/structures_pruning``) each have their
+own ``pyproject.toml`` + ``uv.lock`` too, independent of both the main app and
+each other.
+
+All of these are manipulated using the Makefile targets below in a Linux
+environment (to match our build and deploy systems); for developers on Mac,
+this can be achieved by using the GitHub workflows or by running Make targets
+from inside devstack's lms-shell or another Linux environment.
 
 .. _uv: https://docs.astral.sh/uv/
 .. _public-engineering#543: https://github.com/openedx/public-engineering/issues/543

scripts/structures_pruning/pyproject.toml

@@ -0,0 +1,38 @@
+[project]
+name = "edx-platform-structures-pruning-scripts"
+version = "0.1"
+requires-python = ">=3.12"
+dependencies = [
+    "click",
+    "click-log",
+    "edx-opaque-keys",
+    "pymongo",
+]
+
+[dependency-groups]
+test = [
+    "ddt",
+    "pytest",
+]
+
+[tool.uv]
+# This isn't installed as a package itself -- it's just a locked dependency
+# set for running the standalone script in this directory.
+package = false
+
+#  DO NOT EDIT constraint-dependencies DIRECTLY.
+#  This list is managed by `edx_lint write_uv_constraints`
+#  and will be overwritten the next time `make upgrade` is run.
+#  - GLOBAL constraints: edit edx_lint/files/common_constraints.txt
+#  - REPO-SPECIFIC constraints: edit [tool.edx_lint].uv_constraints in this file
+constraint-dependencies = [
+    "Django<6.0",
+    "elasticsearch<7.14.0",
+    "pymongo<4.4.1",
+]
+[tool.edx_lint]
+# Adding pin to avoid any major upgrade -- see the root pyproject.toml's
+# [tool.edx_lint].uv_constraints for the full rationale.
+uv_constraints = [
+    "pymongo<4.4.1",
+]

scripts/structures_pruning/requirements/base.txt

@@ -1,24 +1,24 @@
-#
-# This file is autogenerated by pip-compile with Python 3.12
-# by the following command:
-#
-#    make upgrade
-#
-click==8.4.1
+# GENERATED FILE, DO NOT EDIT DIRECTLY.
+# Compatibility export for anyone still 'pip install -r scripts/structures_pruning/requirements/base.txt'
+# directly instead of using uv. Source of truth: scripts/structures_pruning/pyproject.toml / uv.lock.
+# This file was autogenerated by uv via the following command:
+#    uv export --frozen --no-hashes --no-emit-project
+click==8.4.2
     # via
-    #   -r scripts/structures_pruning/requirements/base.in
     #   click-log
+    #   edx-platform-structures-pruning-scripts
 click-log==0.4.0
-    # via -r scripts/structures_pruning/requirements/base.in
+    # via edx-platform-structures-pruning-scripts
+colorama==0.4.6 ; sys_platform == 'win32'
+    # via click
 dnspython==2.8.0
     # via pymongo
 edx-opaque-keys==4.0.0
-    # via -r scripts/structures_pruning/requirements/base.in
+    # via edx-platform-structures-pruning-scripts
 pymongo==4.4.0
     # via
-    #   -c requirements/constraints.txt
-    #   -r scripts/structures_pruning/requirements/base.in
     #   edx-opaque-keys
+    #   edx-platform-structures-pruning-scripts
 stevedore==5.8.0
     # via edx-opaque-keys
 typing-extensions==4.15.0

scripts/structures_pruning/requirements/testing.txt

@@ -1,23 +1,23 @@
-#
-# This file is autogenerated by pip-compile with Python 3.12
-# by the following command:
-#
-#    make upgrade
-#
-click==8.4.1
+# GENERATED FILE, DO NOT EDIT DIRECTLY.
+# Compatibility export for anyone still 'pip install -r scripts/structures_pruning/requirements/testing.txt'
+# directly instead of using uv. Source of truth: scripts/structures_pruning/pyproject.toml (test group) / uv.lock.
+# This file was autogenerated by uv via the following command:
+#    uv export --frozen --no-hashes --group test --no-emit-project
+click==8.4.2
     # via
-    #   -r scripts/structures_pruning/requirements/base.txt
     #   click-log
+    #   edx-platform-structures-pruning-scripts
 click-log==0.4.0
-    # via -r scripts/structures_pruning/requirements/base.txt
+    # via edx-platform-structures-pruning-scripts
+colorama==0.4.6 ; sys_platform == 'win32'
+    # via
+    #   click
+    #   pytest
 ddt==1.7.2
-    # via -r scripts/structures_pruning/requirements/testing.in
 dnspython==2.8.0
-    # via
-    #   -r scripts/structures_pruning/requirements/base.txt
-    #   pymongo
+    # via pymongo
 edx-opaque-keys==4.0.0
-    # via -r scripts/structures_pruning/requirements/base.txt
+    # via edx-platform-structures-pruning-scripts
 iniconfig==2.3.0
     # via pytest
 packaging==26.2
@@ -28,15 +28,10 @@ pygments==2.20.0
     # via pytest
 pymongo==4.4.0
     # via
-    #   -r scripts/structures_pruning/requirements/base.txt
     #   edx-opaque-keys
-pytest==9.1.0
-    # via -r scripts/structures_pruning/requirements/testing.in
+    #   edx-platform-structures-pruning-scripts
+pytest==9.1.1
 stevedore==5.8.0
-    # via
-    #   -r scripts/structures_pruning/requirements/base.txt
-    #   edx-opaque-keys
+    # via edx-opaque-keys
 typing-extensions==4.15.0
-    # via
-    #   -r scripts/structures_pruning/requirements/base.txt
-    #   edx-opaque-keys
+    # via edx-opaque-keys