Skip to content

Commit 374c45e

Browse files
harendra-kumarharendra-kumar
committed
Make some caching changes to CIs
1 parent d6643f5 commit 374c45e

4 files changed

Lines changed: 168 additions & 67 deletions

File tree

‎.github/workflows/haskell.yml‎

Lines changed: 40 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -51,28 +51,33 @@ jobs:
5151
GHCUP_VERSION: 0.1.50.2
5252
GHCVER: ${{ matrix.ghc_version }}
5353

54-
# ------------------------------------------------------------------------
55-
# stack options (if using stack builds)
56-
# ------------------------------------------------------------------------
57-
# Note requiring a specific version of stack using STACKVER may fail due to
58-
# github API limit while checking and upgrading/downgrading to the specific
59-
# version.
60-
#STACKVER: "1.6.5"
61-
#STACK_UPGRADE: "y"
62-
STACK_YAML: "stack.yaml"
63-
6454
# ------------------------------------------------------------------------
6555
# cabal options
6656
# ------------------------------------------------------------------------
6757
CABAL_CHECK_RELAX: y
6858

69-
# Build paralellism can cause memory issues. We need to have sufficient
70-
# per CPU memory, if not we can use a lower -j option; default is ncpus.
59+
# CABAL_BUILD_OPTIONS="-j1"
60+
# Enable the above option if you:
61+
# 1. want logs in serial order to be able to correlate them better.
62+
#
63+
# 2. run into memory issues due to Build paralellism. We need to
64+
# have sufficient per CPU memory, if not we can use a lower -j
65+
# option; cabal default is ncpus.
66+
#
7167
# Currently (2026-03-12) from packcheck output we see:
7268
# Linux: 15GB, 4 cpus
7369
# macOS: 7GB, 3 cpus
7470
# Windows: 16GB, 4 cpus
75-
# CABAL_BUILD_OPTIONS="-j1"
71+
72+
# ------------------------------------------------------------------------
73+
# stack options (if using stack builds)
74+
# ------------------------------------------------------------------------
75+
# Note requiring a specific version of stack using STACKVER may fail due to
76+
# github API limit while checking and upgrading/downgrading to the specific
77+
# version.
78+
#STACKVER: "1.6.5"
79+
#STACK_UPGRADE: "y"
80+
STACK_YAML: "stack.yaml"
7681

7782
# ------------------------------------------------------------------------
7883
# Location of packcheck.sh (the shell script invoked to perform CI tests ).
@@ -335,40 +340,51 @@ jobs:
335340
steps:
336341
- uses: actions/checkout@v4
337342

343+
# This should happen before cache restore.
344+
- name: Remove ~/.ghcup symlink
345+
run: |
346+
rm -f ~/.ghcup
347+
348+
- uses: actions/cache@v4
349+
name: Cache hackage package index (non-Windows)
350+
if: runner.os != 'Windows'
351+
with:
352+
path: |
353+
~/.cache/cabal/packages
354+
# Bump the key version to clear the cache
355+
key: cache-cabal-packages
356+
338357
# See the "cabal path" output in the CI logs to tweak the cache locations
339358
# ghc is big but cheap to install, if cache size is a concern we can avoid
340359
# caching it.
341-
- uses: actions/cache@v4
342-
name: Cache ghcup (non-Windows)
360+
- name: Cache ghcup and ghc (non-Windows)
361+
uses: actions/cache@v4
343362
if: runner.os != 'Windows'
344363
with:
345364
path: |
346365
~/.ghcup
347-
key: ${{ matrix.runner }}-ghcup-${{ env.GHCUP_VERSION }}-${{ matrix.ghc_version }}-v1
366+
key: ${{ matrix.runner }}-ghcup-${{ env.GHCUP_VERSION }}-${{ matrix.ghc_version }}
348367

349368
# XXX If we want to invalidate the cache on resolver change we
350369
# should use a separate cache for stack as it should be keyed on
351370
# resolver as well. But that requires resolver as a matrix element
352371
# and then copied to pack_options from there, needs some work. It
353372
# should be fine as resolver minor version change can only increase
354373
# the cache size.
355-
- uses: actions/cache@v4
356-
name: Cache common directories (non-Windows)
374+
- name: Cache build dependencies (non-Windows)
375+
uses: actions/cache@v4
357376
if: runner.os != 'Windows'
358377
with:
359378
# cabal: ~/.local/bin, ~/.local/state/cabal
360-
# ~/.cache/cabal/packages can be cached, but will increase the
361-
# cache size. But can save the "cabal update" time.
362379
path: |
363380
~/.local/bin
364381
~/.local/state/cabal
365382
~/.stack
366383
# Bump the key version to clear the cache
367-
# Key is same as the CI name
368-
key: ${{ matrix.runner }}-${{ matrix.command }}-${{ matrix.ghc_version }}-${{ matrix.name }}-v1
384+
key: ${{ matrix.runner }}-${{ matrix.command }}-${{ matrix.ghc_version }}-${{ matrix.name }}
369385

370-
- uses: actions/cache@v4
371-
name: Cache common directories (Windows)
386+
- name: Cache common directories (Windows)
387+
uses: actions/cache@v4
372388
if: runner.os == 'Windows'
373389
with:
374390
path: |

‎.github/workflows/markdown-doctest.yml‎

Lines changed: 45 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,67 @@
1-
name: Markdown Doctest
1+
name: DOCS
22

3-
on: pull_request
3+
on:
4+
workflow_dispatch:
5+
pull_request:
46

57
jobs:
68
markdown-doctest:
9+
name: "Verify haskell snippets in markdown docs"
10+
env:
11+
GHCVER: 9.8.4
12+
GHCUP_VERSION: 0.1.50.2
713

814
runs-on: ubuntu-latest
915

1016
steps:
1117

12-
- name: Download ghc
18+
# This should happen before cache restore.
19+
- name: Remove ~/.ghcup symlink
1320
run: |
14-
GHCUP_VER=0.1.20.0
15-
curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VER/x86_64-linux-ghcup-$GHCUP_VER
16-
chmod +x ./ghcup
17-
GHCVER=9.8.1
18-
./ghcup install ghc $GHCVER
19-
./ghcup set ghc $GHCVER
20-
cabal update
21+
rm -f ~/.ghcup
2122
22-
- uses: actions/cache@v4
23-
name: Cache ~/.cabal
23+
- name: Cache ghcup and ghc (non-Windows)
24+
uses: actions/cache@v4
25+
if: runner.os != 'Windows'
2426
with:
2527
path: |
26-
~/.cabal
28+
~/.ghcup
29+
key: ${{ runner.os }}-ghcup-${{ env.GHCUP_VERSION }}-${{ env.GHCVER }}
30+
31+
- name: Cache hackage package index (non-Windows)
32+
uses: actions/cache@v4
33+
if: runner.os != 'Windows'
34+
with:
35+
path: |
36+
~/.cache/cabal/packages
2737
# Bump the key version to clear the cache
28-
key: cache-v2
38+
key: cache-cabal-packages
39+
40+
- name: Cache cabal build dependencies (non-Windows)
41+
uses: actions/cache@v4
42+
if: runner.os != 'Windows'
43+
with:
44+
path: |
45+
~/.local/state/cabal
46+
# Bump the key version to clear the cache
47+
key: markdown-doctest-deps
2948

3049
- uses: actions/checkout@v4
3150
with:
3251
ref: ${{ github.event.pull_request.head.sha }}
3352

53+
- name: Download ghc
54+
run: |
55+
if ! ghc --version 2>/dev/null | grep -q "$GHCVER"
56+
then
57+
curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VERSION/x86_64-linux-ghcup-$GHCUP_VERSION
58+
chmod +x ./ghcup
59+
./ghcup install ghc $GHCVER
60+
./ghcup set ghc $GHCVER
61+
fi
62+
# XXX update only if not present
63+
cabal update
64+
3465
- name: Create env files for ghci
3566
run: cabal build streamly --write-ghc-environment-files=always
3667

‎.github/workflows/packdiff.yml‎

Lines changed: 48 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,70 @@
1-
name: Packdiff
1+
name: API
22

3-
on: pull_request
3+
on:
4+
workflow_dispatch:
5+
pull_request:
46

57
jobs:
68
packdiff:
9+
name: "Signature diff: master vs PR"
10+
env:
11+
GHCVER: 9.8.4
12+
GHCUP_VERSION: 0.1.50.2
713

814
runs-on: ubuntu-latest
915

1016
steps:
1117

12-
- name: Download ghc
18+
# This should happen before cache restore.
19+
- name: Remove ~/.ghcup symlink
1320
run: |
14-
GHCUP_VER=0.1.20.0
15-
curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VER/x86_64-linux-ghcup-$GHCUP_VER
16-
chmod +x ./ghcup
17-
GHCVER=9.8.1
18-
./ghcup install ghc $GHCVER
19-
./ghcup set ghc $GHCVER
20-
cabal update
21+
rm -f ~/.ghcup
2122
22-
- uses: actions/cache@v4
23-
name: Cache ~/.cabal
23+
- name: Cache ghcup and ghc (non-Windows)
24+
uses: actions/cache@v4
25+
if: runner.os != 'Windows'
2426
with:
2527
path: |
26-
~/.cabal
28+
~/.ghcup
29+
key: ${{ runner.os }}-ghcup-${{ env.GHCUP_VERSION }}-${{ env.GHCVER }}
30+
31+
- name: Cache hackage package index (non-Windows)
32+
uses: actions/cache@v4
33+
if: runner.os != 'Windows'
34+
with:
35+
path: |
36+
~/.cache/cabal/packages
2737
# Bump the key version to clear the cache
28-
key: cache-v2
38+
key: cache-cabal-packages
39+
40+
- name: Cache cabal build dependencies (non-Windows)
41+
uses: actions/cache@v4
42+
if: runner.os != 'Windows'
43+
with:
44+
path: |
45+
~/.local/state/cabal
46+
# Bump the key version to clear the cache
47+
key: packdiff-deps
2948

3049
- name: Checkout the current branch
31-
uses: actions/checkout@v2
50+
uses: actions/checkout@v4
3251
with:
3352
fetch-depth: 0
3453

35-
- name: Run packdiff on streamly-core
54+
- name: Download ghc
55+
run: |
56+
if ! ghc --version 2>/dev/null | grep -q "$GHCVER"
57+
then
58+
curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VERSION/x86_64-linux-ghcup-$GHCUP_VERSION
59+
chmod +x ./ghcup
60+
./ghcup install ghc $GHCVER
61+
./ghcup set ghc $GHCVER
62+
fi
63+
# XXX update only if not present
64+
cabal update
65+
66+
- name: API diff for streamly-core
3667
run: cabal run packdiff --project-file=cabal.project.packdiff -- diff streamly-core $(git rev-parse origin/master) streamly-core $(git rev-parse HEAD)
3768

38-
- name: Run packdiff on streamly
69+
- name: API diff for streamly
3970
run: cabal run packdiff --project-file=cabal.project.packdiff -- diff streamly $(git rev-parse origin/master) streamly $(git rev-parse HEAD)

‎.github/workflows/regression-check.yml‎

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@ on:
2020
# Removed Data.Fold.Prelude - was failing for some reason (memory issue?)
2121
jobs:
2222
build:
23-
name: "ubuntu-latest ghc-9.10.3 master vs PR"
23+
name: "Comparison: master vs PR (ubuntu-latest ghc-9.10.3)"
2424
env:
2525
GHCVER: 9.10.3
26-
GHCUP_VER: 0.1.50.2
26+
GHCUP_VERSION: 0.1.50.2
2727
CI_BENCHMARKS_WITH_CUTOFF: >-
2828
Data.Array
2929
Data.Array.Generic
@@ -65,15 +65,36 @@ jobs:
6565
- name: Checkout the PR branch
6666
uses: actions/checkout@v4
6767

68-
- name: Cache ~/.cabal
68+
# This should happen before cache restore.
69+
- name: Remove ~/.ghcup symlink
70+
run: |
71+
rm -f ~/.ghcup
72+
73+
- name: Cache ghcup and ghc (non-Windows)
74+
uses: actions/cache@v4
75+
if: runner.os != 'Windows'
76+
with:
77+
path: |
78+
~/.ghcup
79+
key: ${{ runner.os }}-ghcup-${{ env.GHCUP_VERSION }}-${{ env.GHCVER }}
80+
81+
- name: Cache hackage package index (non-Windows)
82+
uses: actions/cache@v4
83+
if: runner.os != 'Windows'
84+
with:
85+
path: |
86+
~/.cache/cabal/packages
87+
# Bump the key version to clear the cache
88+
key: cache-cabal-packages
89+
90+
- name: Cache cabal build dependencies (non-Windows)
6991
uses: actions/cache@v4
92+
if: runner.os != 'Windows'
7093
with:
7194
path: |
72-
~/.cabal
73-
~/.ghcup
74-
~/.local
95+
~/.local/state/cabal
7596
# Bump the key version to clear the cache
76-
key: cache-v2
97+
key: performance-comparison-deps
7798

7899
- name: Update CI_BENCHMARKS environment variable
79100
run: |
@@ -89,11 +110,12 @@ jobs:
89110
run: |
90111
if ! ghc --version 2>/dev/null | grep -q "$GHCVER"
91112
then
92-
curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VER/x86_64-linux-ghcup-$GHCUP_VER
113+
curl -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VERSION/x86_64-linux-ghcup-$GHCUP_VERSION
93114
chmod +x ./ghcup
94115
./ghcup install ghc $GHCVER
95116
./ghcup set ghc $GHCVER
96117
fi
118+
# XXX update only if not present
97119
cabal update
98120
99121
- name: Install bench-runner from PR branch
@@ -113,8 +135,10 @@ jobs:
113135
- name: Run benchmarks for "master" branch
114136
run: |
115137
# CABAL_BUILD_OPTIONS="--cabal-build-options \"-j1\""
116-
#./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw
117-
./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --cabal-build-options "-j1" --targets "$CI_BENCHMARKS" --raw
138+
./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw
139+
# XXX Print dependencies of streamly-core and streamly for comparison
140+
# in case there is a regression due to version changes.
141+
# ghc-pkg --package-db=./dist-newstyle/packagedb/ghc-9.10.3/ field streamly depends
118142
119143
# -----------------------------------------------------------------
120144
# -- Download, generate reports for the current branch and append
@@ -129,8 +153,7 @@ jobs:
129153
- name: Run benchmarks and append
130154
run: |
131155
# CABAL_BUILD_OPTIONS="--cabal-build-options \"-j1\""
132-
#./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw --append
133-
./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --cabal-build-options "-j1" --targets "$CI_BENCHMARKS" --raw --append
156+
./bench-runner --package-name streamly-benchmarks --package-version 0.0.0 --targets "$CI_BENCHMARKS" --raw --append
134157
135158
# -----------------------------------------------------------------
136159
# -- Compare

0 commit comments

Comments
 (0)