From 384e958bf7ec134d35ae8b501c3f56fd1ad3478e Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:09:51 +0000 Subject: [PATCH 01/21] Update shebangs to Python 3 --- benchmark.py | 2 +- save_speedtest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark.py b/benchmark.py index ab6b3a9..ed1b755 100755 --- a/benchmark.py +++ b/benchmark.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 import scraperwiki import os from six.moves import range diff --git a/save_speedtest.py b/save_speedtest.py index d3c1407..14637b4 100755 --- a/save_speedtest.py +++ b/save_speedtest.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 import scraperwiki from six.moves import range From 33ff7ab83bf98771ab68292e14713b2947232bf8 Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:10:22 +0000 Subject: [PATCH 02/21] Remove unneeded shebang and author comments Authors are in the Git commit history, and more people worked on these files. --- scraperwiki/__init__.py | 3 --- scraperwiki/utils.py | 5 ----- tests.py | 1 - 3 files changed, 9 deletions(-) diff --git a/scraperwiki/__init__.py b/scraperwiki/__init__.py index 4f9aaf1..a854c2c 100644 --- a/scraperwiki/__init__.py +++ b/scraperwiki/__init__.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# Thomas Levine, ScraperWiki Limited - ''' Local version of ScraperWiki Utils, documentation here: https://scraperwiki.com/docs/python/python_help_documentation/ diff --git a/scraperwiki/utils.py b/scraperwiki/utils.py index 2eed864..370409d 100644 --- a/scraperwiki/utils.py +++ b/scraperwiki/utils.py @@ -1,8 +1,3 @@ -#!/usr/bin/env python -# utils.py -# David Jones, ScraperWiki Limited -# Thomas Levine, ScraperWiki Limited - ''' Local version of ScraperWiki Utils, documentation here: https://scraperwiki.com/docs/python/python_help_documentation/ diff --git a/tests.py b/tests.py index a5f7e22..97088ca 100755 --- a/tests.py +++ b/tests.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import datetime import json import os From 34fdd50c73eb7a0fa709897a0143473864758120 Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:11:15 +0000 Subject: [PATCH 03/21] Remove uses of `six.moves.range` --- benchmark.py | 1 - save_speedtest.py | 1 - 2 files changed, 2 deletions(-) diff --git a/benchmark.py b/benchmark.py index ed1b755..58f8c90 100755 --- a/benchmark.py +++ b/benchmark.py @@ -1,7 +1,6 @@ #! /usr/bin/env python3 import scraperwiki import os -from six.moves import range rows = [{'id': i, 'test': i * 2, 's': "abc"} for i in range(1000)] diff --git a/save_speedtest.py b/save_speedtest.py index 14637b4..c26b09a 100755 --- a/save_speedtest.py +++ b/save_speedtest.py @@ -1,6 +1,5 @@ #! /usr/bin/env python3 import scraperwiki -from six.moves import range rows = [{'id': i, 'test': i * 2, 's': "xx"*i} for i in range(10)] From 6b6dc88fa839da8b82536a6b3c09b0e303d531ae Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:11:35 +0000 Subject: [PATCH 04/21] Use more appropriate Python 3 error --- benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark.py b/benchmark.py index 58f8c90..49468bb 100755 --- a/benchmark.py +++ b/benchmark.py @@ -6,7 +6,7 @@ try: os.remove('scraperwiki.sqlite') -except OSError: +except FileNotFoundError: pass scraperwiki.sql.save(['id'], rows) From c8936fc84fe7b1e02c81eb03985a455d77933aab Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:12:04 +0000 Subject: [PATCH 05/21] Remove `main()` from tests We run these with a test runner that finds the tests. --- tests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests.py b/tests.py index 97088ca..40f11f1 100755 --- a/tests.py +++ b/tests.py @@ -384,6 +384,3 @@ def test_import_scraperwiki_utils(self): def test_import_scraperwiki_special_utils(self): self.sw.pdftoxml - -if __name__ == '__main__': - main() From 778c7dc82cfd976cb82063e8980b68250275d04f Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:12:41 +0000 Subject: [PATCH 06/21] Use Python 3 file handling The file will be read as a string instead of bytes, so we don't need to decode it. The `pdftohtml` output is encoded as UTF-8. --- scraperwiki/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scraperwiki/utils.py b/scraperwiki/utils.py index 370409d..974d347 100644 --- a/scraperwiki/utils.py +++ b/scraperwiki/utils.py @@ -38,7 +38,7 @@ def pdftoxml(pdfdata, options=""): pdffout.write(pdfdata) pdffout.flush() - xmlin = tempfile.NamedTemporaryFile(mode='r', suffix='.xml') + xmlin = tempfile.NamedTemporaryFile(mode='r', suffix='.xml', encoding="utf-8") tmpxml = xmlin.name # "temph.xml" cmd = 'pdftohtml -xml -nodrm -zoom 1.5 -enc UTF-8 -noframes {} "{}" "{}"'.format( options, pdffout.name, os.path.splitext(tmpxml)[0]) @@ -50,7 +50,7 @@ def pdftoxml(pdfdata, options=""): #xmlfin = open(tmpxml) xmldata = xmlin.read() xmlin.close() - return xmldata.decode('utf-8') + return xmldata def _in_box(): From e99665a01bbc5f9a730319a3e49f1479e393465f Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:13:49 +0000 Subject: [PATCH 07/21] Update code to remove use of `six` And tidy slightly. --- scraperwiki/utils.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/scraperwiki/utils.py b/scraperwiki/utils.py index 974d347..27a6e07 100644 --- a/scraperwiki/utils.py +++ b/scraperwiki/utils.py @@ -6,15 +6,15 @@ import sys import warnings import tempfile -import six.moves.urllib.parse -import six.moves.urllib.request +import urllib.parse +import urllib.request import requests def scrape(url, params=None, user_agent=None): ''' Scrape a URL optionally with parameters. - This is effectively a wrapper around urllib2.urlopen. + This is effectively a wrapper around urllib.request.urlopen. ''' headers = {} @@ -22,12 +22,14 @@ def scrape(url, params=None, user_agent=None): if user_agent: headers['User-Agent'] = user_agent - data = params and six.moves.urllib.parse.urlencode(params) or None - req = six.moves.urllib.request.Request(url, data=data, headers=headers) - f = six.moves.urllib.request.urlopen(req) + data = None + if params: + data = urllib.parse.urlencode(params).encode('utf-8') - text = f.read() - f.close() + req = urllib.request.Request(url, data=data, headers=headers) + + with urllib.request.urlopen(req) as f: + text = f.read() return text From 2ea25aae4fdd96a4274276006553512139fb889e Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:14:35 +0000 Subject: [PATCH 08/21] Tidy up class --- scraperwiki/sql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scraperwiki/sql.py b/scraperwiki/sql.py index 367ca5d..fd806d5 100644 --- a/scraperwiki/sql.py +++ b/scraperwiki/sql.py @@ -23,10 +23,11 @@ SqliteError = sqlalchemy.exc.SQLAlchemyError class Blob(bytes): - """ Represents a blob as a string. """ + pass + PYTHON_SQLITE_TYPE_MAP = { str: sqlalchemy.types.Text, str: sqlalchemy.types.Text, From 3ca2082040b2c5e1e9d7d47a4cfdf8b13d58407a Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:14:51 +0000 Subject: [PATCH 09/21] Use `row.mapping` directly --- scraperwiki/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scraperwiki/sql.py b/scraperwiki/sql.py index fd806d5..a9561be 100644 --- a/scraperwiki/sql.py +++ b/scraperwiki/sql.py @@ -184,7 +184,7 @@ def select(query, data=None): rows = [] for row in result: - rows.append(dict(list(row._mapping.items()))) + rows.append(dict(row._mapping)) return rows From 99de9ecbadd46e6cbc63c2047136035b4ff6e6d9 Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:15:51 +0000 Subject: [PATCH 10/21] Remove use of `six.text_type` --- tests.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests.py b/tests.py index 40f11f1..b747c48 100755 --- a/tests.py +++ b/tests.py @@ -12,7 +12,6 @@ from unittest import TestCase, main import scraperwiki -import six import sys # scraperwiki.sql._State.echo = True @@ -63,9 +62,9 @@ def test_date(self): date1 = datetime.datetime.now() date2 = datetime.date.today() scraperwiki.sql.save_var(u"weird\u1234", date1) - self.assertEqual(scraperwiki.sql.get_var(u"weird\u1234"), six.text_type(date1)) + self.assertEqual(scraperwiki.sql.get_var(u"weird\u1234"), str(date1)) scraperwiki.sql.save_var(u"weird\u1234", date2) - self.assertEqual(scraperwiki.sql.get_var(u"weird\u1234"), six.text_type(date2)) + self.assertEqual(scraperwiki.sql.get_var(u"weird\u1234"), str(date2)) def test_save_multiple_values(self): scraperwiki.sql.save_var(u'foo\xc3', u'hello') @@ -268,7 +267,7 @@ def test_lxml_string(self): self.save_and_check( {"text": s}, "lxml", - [(six.text_type(s),)] + [(str(s),)] ) def test_save_and_drop(self): @@ -319,7 +318,7 @@ def test_save_date(self): scraperwiki.sql.select("* FROM swdata")) self.assertEqual( - {u'keys': [u'birthday\xaa'], u'data': [(six.text_type(d),)]}, + {u'keys': [u'birthday\xaa'], u'data': [(str(d),)]}, scraperwiki.sql.execute("SELECT * FROM swdata")) self.assertEqual(str(d), self.rawdate(column=u"birthday\xaa")) @@ -330,7 +329,7 @@ def test_save_datetime(self): scraperwiki.sql.save([], {"birthday": d}, table_name="datetimetest") - exemplar = six.text_type(d) + exemplar = str(d) # SQLAlchemy appears to convert with extended precision. exemplar += ".000000" From 8a7cea6bd202cd9473c719777d829d576175c36d Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:16:30 +0000 Subject: [PATCH 11/21] Decode value directly instead of converting --- tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.py b/tests.py index b747c48..1250757 100755 --- a/tests.py +++ b/tests.py @@ -92,7 +92,7 @@ def test_insert(self): """) ((colname, value, _type),) = self.cursor.fetchall() expected = [(u"birthday\xfe", u"\u1234November 30, 1888", "text",)] - observed = [(colname, type(b'')(value).decode('utf-8'), _type)] + observed = [(colname, value.decode('utf-8'), _type)] self.assertEqual(observed, expected) class SaveAndCheck(TestCase): From f96c956fb8650ff281628b858adb4ea4003fd68b Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:20:05 +0000 Subject: [PATCH 12/21] Remove unused `six` import --- scraperwiki/sql.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scraperwiki/sql.py b/scraperwiki/sql.py index a9561be..53d72c8 100644 --- a/scraperwiki/sql.py +++ b/scraperwiki/sql.py @@ -10,7 +10,6 @@ import alembic.ddl import sqlalchemy -import six DATABASE_NAME = os.environ.get("SCRAPERWIKI_DATABASE_NAME", "sqlite:///scraperwiki.sqlite") From 36352ddb8b719970493af8d988ee8e0cc9bc91a0 Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:20:28 +0000 Subject: [PATCH 13/21] Use `str` instead of `unicode` --- scraperwiki/sql.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scraperwiki/sql.py b/scraperwiki/sql.py index 53d72c8..fb22e1a 100644 --- a/scraperwiki/sql.py +++ b/scraperwiki/sql.py @@ -16,7 +16,6 @@ DATABASE_TIMEOUT = float(os.environ.get("SCRAPERWIKI_DATABASE_TIMEOUT", 300)) SECONDS_BETWEEN_COMMIT = 2 -unicode = str # The scraperwiki.sqlite.SqliteError exception SqliteError = sqlalchemy.exc.SQLAlchemyError @@ -268,7 +267,7 @@ def save_var(name, value): if column_type == sqlalchemy.types.LargeBinary: value_blob = value else: - value_blob = unicode(value).encode('utf-8') + value_blob = str(value).encode('utf-8') values = dict(name=name, value_blob=value_blob, From 5f1f0043ed8038645da6417db261d59ac3462fa4 Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:21:01 +0000 Subject: [PATCH 14/21] Update types for Python 3 * We don't need the conditional for Python 2.7 handling. * `long` doesn't exist in Python 3. --- scraperwiki/sql.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/scraperwiki/sql.py b/scraperwiki/sql.py index fb22e1a..3138e68 100644 --- a/scraperwiki/sql.py +++ b/scraperwiki/sql.py @@ -27,7 +27,6 @@ class Blob(bytes): pass PYTHON_SQLITE_TYPE_MAP = { - str: sqlalchemy.types.Text, str: sqlalchemy.types.Text, int: sqlalchemy.types.BigInteger, bool: sqlalchemy.types.Boolean, @@ -36,19 +35,10 @@ class Blob(bytes): datetime.date: sqlalchemy.types.Date, datetime.datetime: sqlalchemy.types.DateTime, + bytes: sqlalchemy.types.LargeBinary, Blob: sqlalchemy.types.LargeBinary, } -if bytes is not str: - # On 2.7, bytes *is* str, so we don't want to overwrite that. - PYTHON_SQLITE_TYPE_MAP[bytes] = sqlalchemy.types.LargeBinary - -try: - PYTHON_SQLITE_TYPE_MAP[long] = sqlalchemy.types.BigInteger -except NameError: - pass - - class _State: """ From ab498bb596bc25a49cbad060f3489e3e019ca560 Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:22:25 +0000 Subject: [PATCH 15/21] Reinstate missing warning Removed when updating to `pyproject.toml`. --- scraperwiki/utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scraperwiki/utils.py b/scraperwiki/utils.py index 27a6e07..92f72b4 100644 --- a/scraperwiki/utils.py +++ b/scraperwiki/utils.py @@ -3,6 +3,7 @@ https://scraperwiki.com/docs/python/python_help_documentation/ ''' import os +import shutil import sys import warnings import tempfile @@ -36,6 +37,13 @@ def scrape(url, params=None, user_agent=None): def pdftoxml(pdfdata, options=""): """converts pdf file to xml file""" + if not shutil.which('pdftohtml'): + warnings.warn( + 'scraperwiki.pdftoxml requires pdftohtml, but pdftohtml was not found ' + 'in the PATH. If you wish to use this function, you probably need to ' + 'install pdftohtml.' + ) + return None pdffout = tempfile.NamedTemporaryFile(suffix='.pdf') pdffout.write(pdfdata) pdffout.flush() From 76524115276e728016475e468397469e03730263 Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:23:10 +0000 Subject: [PATCH 16/21] Remove `Dockerfile` It was not being maintained. --- Dockerfile | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 3799529..0000000 --- a/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM ubuntu:14.04 - -RUN apt-get update && apt-get install -y python3-pip python3-nose ack-grep vim python3-lxml python-lxml python-nose python-pip - -RUN locale-gen en_GB.UTF-8 - -RUN mkdir -p /home/nobody && \ - chown nobody /home/nobody -USER nobody -ENV HOME=/home/nobody \ - PATH=/home/nobody/.local/bin:$PATH \ - LANG=en_GB.UTF-8 -WORKDIR /home/nobody - -RUN mkdir -p /home/nobody/.local/bin -RUN echo python3 $* > /home/nobody/.local/bin/python -RUN chmod +x /home/nobody/.local/bin/python -RUN pip3 install --user requests sqlalchemy alembic -RUN pip install --user requests sqlalchemy alembic -COPY . /home/nobody/ -RUN python3 tests.py -RUN python2 tests.py From b77732ea5fe0b312b34132fd18c632ff89419d6a Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:23:32 +0000 Subject: [PATCH 17/21] Remove `six` dependency --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 59b8bbd..a3bcadd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,6 @@ classifiers = [ ] dependencies = [ "requests==2.32.5", - "six==1.17.0", "sqlalchemy==2.0.45", "alembic==1.17.2", ] From eb0c8bce7f8fcf792aeb64b84ddfa50519ba7fc5 Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:28:46 +0000 Subject: [PATCH 18/21] Deprecate `status()` It's no longer useful at all, but it's possible that code still calls it, so make it a stub. --- scraperwiki/utils.py | 22 +++------------------- tests.py | 10 ++++------ 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/scraperwiki/utils.py b/scraperwiki/utils.py index 92f72b4..3dea8b6 100644 --- a/scraperwiki/utils.py +++ b/scraperwiki/utils.py @@ -9,7 +9,6 @@ import tempfile import urllib.parse import urllib.request -import requests def scrape(url, params=None, user_agent=None): @@ -63,26 +62,11 @@ def pdftoxml(pdfdata, options=""): return xmldata -def _in_box(): - return os.environ.get('HOME', None) == '/home' - - def status(type, message=None): - assert type in ['ok', 'error'] - - # if not running in a ScraperWiki platform box, silently do nothing - if not _in_box(): - return "Not in box" - - url = os.environ.get("SW_STATUS_URL", "https://app.quickcode.io/api/status") - if url == "OFF": - # For development mode - return + """Retained for backwards compatibility.""" + warnings.warn("status() is no longer in use following ScraperWiki/Quickcode application shutdown", DeprecationWarning, stacklevel=2) + return - # send status update to the box - r = requests.post(url, data={'type': type, 'message': message}) - r.raise_for_status() - return r.content def swimport(scrapername): return __import__(scrapername) diff --git a/tests.py b/tests.py index 1250757..9951bcc 100755 --- a/tests.py +++ b/tests.py @@ -345,13 +345,11 @@ def test_save_datetime(self): class TestStatus(TestCase): 'Test that the status endpoint works.' - def test_does_nothing_if_called_outside_box(self): - scraperwiki.status('ok') - - def test_raises_exception_with_invalid_type_field(self): - self.assertRaises(AssertionError, scraperwiki.status, 'hello') + def test_status(self): + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) - # XXX neeed some mocking tests for case of run inside a box + self.assertEqual(scraperwiki.status('ok'), None) class TestUnicodeColumns(TestCase): maxDiff = None From aa7f2033df011d7ef59c0505cf47dc7e7314e080 Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:30:51 +0000 Subject: [PATCH 19/21] Remove `requests` dependency --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a3bcadd..fcf51fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,6 @@ classifiers = [ "Topic :: Database :: Front-Ends", ] dependencies = [ - "requests==2.32.5", "sqlalchemy==2.0.45", "alembic==1.17.2", ] From 6418451175ebd8aa975341d9c1269e9247b9685b Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:32:19 +0000 Subject: [PATCH 20/21] Fix path to README --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fcf51fc..67fa290 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ { name = "ScraperWiki Developers", email = "hello@scraperwiki.com" } ] license = { text = "GPLv3+" } -readme = "README.md" # Assumes you have a README file +readme = "README.rst" urls = { Repository = "https://github.com/cantabular/scraperwiki-python" } classifiers = [ "Intended Audience :: Developers", From 5d1e79fbcc1a261e6bde3507e7de96bb883c377d Mon Sep 17 00:00:00 2001 From: Steven Maude Date: Fri, 12 Dec 2025 16:32:28 +0000 Subject: [PATCH 21/21] Specify more relaxed dependencies We only need really pin SQLAlchemy now, as that should determine the version of alembic. --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 67fa290..58e37e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,8 +25,8 @@ classifiers = [ "Topic :: Database :: Front-Ends", ] dependencies = [ - "sqlalchemy==2.0.45", - "alembic==1.17.2", + "sqlalchemy>=2,<3", + "alembic", ] [tool.setuptools.packages.find]