From cbc3e96ff2c0c2f51f6c7de547a8b07a013e1201 Mon Sep 17 00:00:00 2001 From: Ross Young Date: Sun, 29 Mar 2026 00:38:09 +0000 Subject: [PATCH 1/9] remove psycopg2 dependency and change IntegrityError to Django's IntegrityError --- mapit/views/areas.py | 2 +- mapit_gb/management/commands/mapit_UK_add_ons_to_gss.py | 2 +- mapit_gb/management/commands/mapit_UK_update_ons_ids.py | 2 +- mapit_gb/management/commands/mapit_UK_update_ons_ids2.py | 2 +- setup.py | 1 - 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mapit/views/areas.py b/mapit/views/areas.py index 1791f41e..2cc6073a 100644 --- a/mapit/views/areas.py +++ b/mapit/views/areas.py @@ -1,5 +1,5 @@ import re -from psycopg2 import InternalError +from django.db import InternalError from django.db.utils import DatabaseError from django.utils.translation import gettext as _ diff --git a/mapit_gb/management/commands/mapit_UK_add_ons_to_gss.py b/mapit_gb/management/commands/mapit_UK_add_ons_to_gss.py index cee7ccb1..1d781d53 100644 --- a/mapit_gb/management/commands/mapit_UK_add_ons_to_gss.py +++ b/mapit_gb/management/commands/mapit_UK_add_ons_to_gss.py @@ -7,7 +7,7 @@ import sys from django.core.management.base import BaseCommand from mapit.models import Area, CodeType -from psycopg2 import IntegrityError +from django.db import IntegrityError python_version = sys.version_info[0] diff --git a/mapit_gb/management/commands/mapit_UK_update_ons_ids.py b/mapit_gb/management/commands/mapit_UK_update_ons_ids.py index cb2e420b..e758db9e 100644 --- a/mapit_gb/management/commands/mapit_UK_update_ons_ids.py +++ b/mapit_gb/management/commands/mapit_UK_update_ons_ids.py @@ -3,7 +3,7 @@ import csv from django.core.management.base import BaseCommand from mapit.models import Area, Generation, CodeType -from psycopg2 import IntegrityError +from django.db import IntegrityError class Command(BaseCommand): diff --git a/mapit_gb/management/commands/mapit_UK_update_ons_ids2.py b/mapit_gb/management/commands/mapit_UK_update_ons_ids2.py index afba95e2..81e69b47 100644 --- a/mapit_gb/management/commands/mapit_UK_update_ons_ids2.py +++ b/mapit_gb/management/commands/mapit_UK_update_ons_ids2.py @@ -4,7 +4,7 @@ import csv from django.core.management.base import BaseCommand from mapit.models import Area, Generation, CodeType -from psycopg2 import IntegrityError +from django.db import IntegrityError class Command(BaseCommand): diff --git a/setup.py b/setup.py index a31ad4e2..ebc3a60a 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,6 @@ def read_file(filename): install_requires=[ 'Django >= 4.2, <6.0', 'libsass >= 0.13.3', - 'psycopg2', 'PyYAML', 'Shapely', 'uk-postcode-utils', From f2fa083fced277a7bdc768d51af576ccd33ea2b2 Mon Sep 17 00:00:00 2001 From: Ross Young Date: Sun, 29 Mar 2026 02:34:01 +0100 Subject: [PATCH 2/9] Add extras_require for psycopg2 and psycopg3 in setup.py --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index ebc3a60a..5fa3421f 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,10 @@ def read_file(filename): 'Shapely', 'uk-postcode-utils', ], + extras_require={ + 'psycopg2': ['psycopg2'], + 'psycopg3': ['psycopg'], + }, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Web Environment', From 375267ee4999836370f5cb4cd88e8b762648cb7e Mon Sep 17 00:00:00 2001 From: Ross Young Date: Sun, 29 Mar 2026 02:54:54 +0100 Subject: [PATCH 3/9] Add extras for psycopg2 in tox.ini --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 52cf6e5d..0acd45db 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,8 @@ envlist = flake8, py310-{4.2,5.2} commands = flake8: flake8 mapit mapit_gb mapit_it mapit_no mapit_se mapit_za project py310: python -W all -W ignore::PendingDeprecationWarning -m coverage run --source mapit manage.py test mapit mapit_gb +extras = + py310: psycopg2 deps = py310: coverage flake8: flake8 From 03fee24581ff145f2dea4c73ff5f1600c696b047 Mon Sep 17 00:00:00 2001 From: ross Date: Sun, 29 Mar 2026 20:14:03 +0100 Subject: [PATCH 4/9] Update psycopg dependencies in setup.py and tox.ini --- setup.py | 3 +-- tox.ini | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 5fa3421f..b161656c 100644 --- a/setup.py +++ b/setup.py @@ -58,8 +58,7 @@ def read_file(filename): 'uk-postcode-utils', ], extras_require={ - 'psycopg2': ['psycopg2'], - 'psycopg3': ['psycopg'], + 'psycopg', }, classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/tox.ini b/tox.ini index 0acd45db..5fe660d6 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ commands = flake8: flake8 mapit mapit_gb mapit_it mapit_no mapit_se mapit_za project py310: python -W all -W ignore::PendingDeprecationWarning -m coverage run --source mapit manage.py test mapit mapit_gb extras = - py310: psycopg2 + py310: psycopg deps = py310: coverage flake8: flake8 From b78a58a925c25bedc855d645dc37737a1afd036d Mon Sep 17 00:00:00 2001 From: ross Date: Sun, 29 Mar 2026 20:17:08 +0100 Subject: [PATCH 5/9] Fix extras_require for psycopg in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b161656c..f670a849 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def read_file(filename): 'uk-postcode-utils', ], extras_require={ - 'psycopg', + 'psycopg', ['psycopg'], }, classifiers=[ 'Development Status :: 5 - Production/Stable', From e38dd2d2b63f1de7d120b8abf4ce49307899b570 Mon Sep 17 00:00:00 2001 From: ross Date: Sun, 29 Mar 2026 20:23:36 +0100 Subject: [PATCH 6/9] Fix extras_require syntax for psycopg in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f670a849..f99bdc11 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def read_file(filename): 'uk-postcode-utils', ], extras_require={ - 'psycopg', ['psycopg'], + 'psycopg': ['psycopg'], }, classifiers=[ 'Development Status :: 5 - Production/Stable', From 5804af3e02787abdc785a0b78c27e525c35d99b2 Mon Sep 17 00:00:00 2001 From: ross Date: Sun, 29 Mar 2026 20:43:11 +0100 Subject: [PATCH 7/9] =?UTF-8?q?Root=20cause:=20psycopg2=20automatically=20?= =?UTF-8?q?adapts=20a=20Python=20tuple=20passed=20as=20a=20parameter=20to?= =?UTF-8?q?=20an=20IN=20(...)=20clause=20=E2=80=94=20e.g.,=20('SML',)=20be?= =?UTF-8?q?comes=20('SML')=20in=20SQL.=20psycopg3=20removed=20this=20magic?= =?UTF-8?q?=20and=20treats=20tuples/values=20literally,=20causing=20the=20?= =?UTF-8?q?IN=20'(SML)'=20syntax=20error.=20Fix:=20Changed=20IN=20%s=20(tu?= =?UTF-8?q?ple)=20to=20=3D=20ANY(%s)=20(array),=20passing=20list(types)=20?= =?UTF-8?q?instead=20of=20tuple(types).=20psycopg3=20natively=20adapts=20P?= =?UTF-8?q?ython=20lists=20to=20PostgreSQL=20arrays,=20so=20=3D=20ANY(%s)?= =?UTF-8?q?=20with=20['SML']=20generates=20valid=20SQL:=20=3D=20ANY(ARRAY[?= =?UTF-8?q?'SML']),=20which=20is=20semantically=20equivalent=20to=20IN=20(?= =?UTF-8?q?'SML').?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mapit/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mapit/models.py b/mapit/models.py index d51d9164..52458aec 100644 --- a/mapit/models.py +++ b/mapit/models.py @@ -191,8 +191,8 @@ def intersect(self, query_type, area, types, generation): params = [area.id, area.id, generation.id, generation.id] if types: - params.append(tuple(types)) - query_area_type = ' AND mapit_area.type_id IN (SELECT id FROM mapit_type WHERE code IN %s) ' + params.append(list(types)) + query_area_type = ' AND mapit_area.type_id IN (SELECT id FROM mapit_type WHERE code = ANY(%s)) ' else: query_area_type = '' From 3090ff807c0dd9b3085b6b31b2377d713cb6dba4 Mon Sep 17 00:00:00 2001 From: ross Date: Sun, 29 Mar 2026 20:57:46 +0100 Subject: [PATCH 8/9] The materialized() function uses connection.cursor().connection.server_version, which is psycopg2-specific. In psycopg3 the version lives at conn.info.server_version, so this raises AttributeError, which bubbles up through filter_by_area and gets caught by the bare except: in example_postcode_for_area, silently setting pc = None. Django's PostgreSQL backend exposes connection.pg_version (works with both psycopg2 and psycopg3): --- mapit/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mapit/models.py b/mapit/models.py index 52458aec..3bb5eeb5 100644 --- a/mapit/models.py +++ b/mapit/models.py @@ -18,9 +18,8 @@ def materialized(): - version = connection.cursor().connection.server_version materialized = '' - if version >= 120000: + if connection.pg_version >= 120000: materialized = 'MATERIALIZED' return materialized From 333b3dd690eacf27131df64baccafeb986a6f4d1 Mon Sep 17 00:00:00 2001 From: ross Date: Sun, 29 Mar 2026 21:31:05 +0100 Subject: [PATCH 9/9] =?UTF-8?q?To=20summarize=20the=20three=20psycopg3=20i?= =?UTF-8?q?ncompatibilities=20found=20so=20far:=201.=20IN=20%s=20with=20tu?= =?UTF-8?q?ple=20=E2=86=92=20=3D=20ANY(%s)=20with=20list=20(psycopg3=20doe?= =?UTF-8?q?sn't=20auto-adapt=20tuples=20for=20IN)=202.=20connection.cursor?= =?UTF-8?q?().connection.server=5Fversion=20=E2=86=92=20connection.pg=5Fve?= =?UTF-8?q?rsion=20(psycopg3=20uses=20conn.info.server=5Fversion=20interna?= =?UTF-8?q?lly)=203.=20location::bytea=20=E2=86=92=20location=20(psycopg2?= =?UTF-8?q?=20returns=20bytea=20as=20memoryview,=20psycopg3=20as=20bytes;?= =?UTF-8?q?=20GEOSGeometry=20handles=20memoryview=20but=20fails=20on=20raw?= =?UTF-8?q?=20binary=20bytes)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mapit/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapit/models.py b/mapit/models.py index 3bb5eeb5..08de91bb 100644 --- a/mapit/models.py +++ b/mapit/models.py @@ -451,7 +451,7 @@ def filter_by_area(self, area, limit=''): WHERE area_id = %s''' query = ''' WITH target AS %s ( %s ) -SELECT "mapit_postcode"."id", "mapit_postcode"."postcode", "mapit_postcode"."location"::bytea +SELECT "mapit_postcode"."id", "mapit_postcode"."postcode", "mapit_postcode"."location" FROM mapit_postcode, target WHERE ST_CoveredBy(location, target.division) %s