From 380f92ebcad6f35784519e815119483d9ffd1bed Mon Sep 17 00:00:00 2001 From: Greg Lindahl Date: Thu, 26 Dec 2024 20:03:36 +0000 Subject: [PATCH 1/6] try macos/windows --- .github/workflows/ci-slow.yaml | 57 ++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/ci-slow.yaml diff --git a/.github/workflows/ci-slow.yaml b/.github/workflows/ci-slow.yaml new file mode 100644 index 0000000..ac2112c --- /dev/null +++ b/.github/workflows/ci-slow.yaml @@ -0,0 +1,57 @@ +name: CI-slow + +on: workflow_dispatch + +jobs: + unit-tests: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + #max-parallel: 1 + matrix: + include: + - python-version: '3.11' + os: macos-latest + EXTRA: true + - python-version: '3.13' + os: macos-latest + EXTRA: true + - python-version: '3.7' + os: windows-latest + EXTRA: true + - python-version: '3.13' + os: windows-latest + EXTRA: true + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install packages + run: make install + + # crash tests from here on out + - name: make iterate + run: make iterate + + - name: make cdxj + run: make cdxj + + - name: make extract + run: make extract + + - name: make cdx_toolkit + run: make cdx_toolkit + + - name: make download_collinfo + run: make download_collinfo + + - name: make duck_cloudfront + run: make duck_cloudfront + + - name: make wreck_the_warc + run: make wreck_the_warc From 9ac210a10bfbc3d3838ab77065822f74e65b2a3b Mon Sep 17 00:00:00 2001 From: Greg Lindahl Date: Thu, 26 Dec 2024 20:18:22 +0000 Subject: [PATCH 2/6] try macos/windows --- .github/workflows/ci-slow.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-slow.yaml b/.github/workflows/ci-slow.yaml index ac2112c..6c9de72 100644 --- a/.github/workflows/ci-slow.yaml +++ b/.github/workflows/ci-slow.yaml @@ -1,6 +1,10 @@ name: CI-slow -on: workflow_dispatch +on: + workflow_dispatch + pull_request: + branches: + - main jobs: unit-tests: From 4084a0c7bc012f4f03782322d53bed5514e03766 Mon Sep 17 00:00:00 2001 From: Greg Lindahl Date: Thu, 26 Dec 2024 20:20:42 +0000 Subject: [PATCH 3/6] try macos/windows --- .github/workflows/ci-slow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-slow.yaml b/.github/workflows/ci-slow.yaml index 6c9de72..566c210 100644 --- a/.github/workflows/ci-slow.yaml +++ b/.github/workflows/ci-slow.yaml @@ -1,7 +1,7 @@ name: CI-slow on: - workflow_dispatch + workflow_dispatch: pull_request: branches: - main From 0719e04176a7e10c1a8ecdeb25ad9fcb2844c866 Mon Sep 17 00:00:00 2001 From: Greg Lindahl Date: Thu, 26 Dec 2024 21:02:43 +0000 Subject: [PATCH 4/6] try macos/windows --- duck.py | 1 + 1 file changed, 1 insertion(+) diff --git a/duck.py b/duck.py index f9b8e85..58c8b71 100644 --- a/duck.py +++ b/duck.py @@ -100,6 +100,7 @@ def main(algo, crawl): else: raise + duckdb.sql('.utf8') duckdb.sql('SET enable_progress_bar = true;') duckdb.sql('SET http_retries = 100;') #duckdb.sql("SET enable_http_logging = true;SET http_logging_output = 'duck.http.log'") From f3b9dd63737324389a7ab6f9e4c5b948f7c84856 Mon Sep 17 00:00:00 2001 From: Greg Lindahl Date: Thu, 26 Dec 2024 21:35:58 +0000 Subject: [PATCH 5/6] try macos/windows --- .github/workflows/ci-slow.yaml | 4 ++++ duck.py | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-slow.yaml b/.github/workflows/ci-slow.yaml index 566c210..59a5ee5 100644 --- a/.github/workflows/ci-slow.yaml +++ b/.github/workflows/ci-slow.yaml @@ -23,9 +23,13 @@ jobs: - python-version: '3.7' os: windows-latest EXTRA: true + env: + PYTHONIOENCODING: "utf8" - python-version: '3.13' os: windows-latest EXTRA: true + env: + PYTHONIOENCODING: "utf8" steps: - name: checkout uses: actions/checkout@v4 diff --git a/duck.py b/duck.py index 58c8b71..f9b8e85 100644 --- a/duck.py +++ b/duck.py @@ -100,7 +100,6 @@ def main(algo, crawl): else: raise - duckdb.sql('.utf8') duckdb.sql('SET enable_progress_bar = true;') duckdb.sql('SET http_retries = 100;') #duckdb.sql("SET enable_http_logging = true;SET http_logging_output = 'duck.http.log'") From aa9f5474c2e0ac0f3f3eec3fb398f36aa100da3b Mon Sep 17 00:00:00 2001 From: Greg Lindahl Date: Thu, 26 Dec 2024 23:00:06 +0000 Subject: [PATCH 6/6] try macos/windows --- .github/workflows/ci-slow.yaml | 65 ---------------------------------- .github/workflows/ci.yaml | 19 +++++++++- 2 files changed, 18 insertions(+), 66 deletions(-) delete mode 100644 .github/workflows/ci-slow.yaml diff --git a/.github/workflows/ci-slow.yaml b/.github/workflows/ci-slow.yaml deleted file mode 100644 index 59a5ee5..0000000 --- a/.github/workflows/ci-slow.yaml +++ /dev/null @@ -1,65 +0,0 @@ -name: CI-slow - -on: - workflow_dispatch: - pull_request: - branches: - - main - -jobs: - unit-tests: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - #max-parallel: 1 - matrix: - include: - - python-version: '3.11' - os: macos-latest - EXTRA: true - - python-version: '3.13' - os: macos-latest - EXTRA: true - - python-version: '3.7' - os: windows-latest - EXTRA: true - env: - PYTHONIOENCODING: "utf8" - - python-version: '3.13' - os: windows-latest - EXTRA: true - env: - PYTHONIOENCODING: "utf8" - steps: - - name: checkout - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install packages - run: make install - - # crash tests from here on out - - name: make iterate - run: make iterate - - - name: make cdxj - run: make cdxj - - - name: make extract - run: make extract - - - name: make cdx_toolkit - run: make cdx_toolkit - - - name: make download_collinfo - run: make download_collinfo - - - name: make duck_cloudfront - run: make duck_cloudfront - - - name: make wreck_the_warc - run: make wreck_the_warc diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f0888c2..e235105 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,7 +9,7 @@ on: - main jobs: - unit-tests: + tests: runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -22,6 +22,23 @@ jobs: - python-version: '3.7' os: ubuntu-20.04 # oldest version on github actions EXTRA: true + - python-version: '3.11' + os: macos-latest + EXTRA: true + - python-version: '3.13' + os: macos-latest + EXTRA: true + #- python-version: '3.7' + # os: windows-latest + # EXTRA: true + # env: + # this doesn't actually work + # PYTHONIOENCODING: "utf8" + #- python-version: '3.13' + # os: windows-latest + # EXTRA: true + # env: + # PYTHONIOENCODING: "utf8" steps: - name: checkout