synchronizing main (for deployment)

.dockerignore

@@ -2,4 +2,12 @@ public/
 .github/
 .git/
 fireside-scraper
-scraped-data
+scraped-data
+# ignoring when dockerfiles themselves
+Dockerfile*
+# ignoring all of tests directory
+#   container only needs dependencies, and tests are run during it's execution
+test/
+# allowing Pipfiles, since used in Dockerfile.tests
+!/test/Pipfile
+!/test/Pipfile.lock

.github/workflows/build_container-prod.yml

@@ -0,0 +1,92 @@
+name: Building Prod Container
+
+# built with the following guides/examples:
+# - https://github.com/marketplace/actions/push-to-registry#examples
+# - https://github.com/marketplace/actions/buildah-build#building-using-containerfiles
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+    # don't re-build prod when there's only test changes
+    paths-ignore:
+      - ./Dockerfile.tests
+      - "./tests/**"
+
+  workflow_dispatch:
+
+  workflow_run:
+    workflows: ["Scrape and commit"]
+    types:
+      - completed
+
+jobs:
+  # this sets up some variables so the next job can use them for tags
+  env_setup:
+    runs-on: ubuntu-latest
+    outputs:
+      date_output: ${{ steps.date.outputs.date_fmt }}
+      registry_user: ${{ steps.registry_user.outputs.username }}
+      current_branch: ${{ steps.current_branch.outputs.current_branch }}
+    steps:
+      # used for tagging the container with the current date down to the seconds
+      #   since we have so many changes down to the seconds was required. Example: 2022-09-08T05_42_23-04_00
+      - id: date
+        run: echo "::set-output name=date_fmt::$(date -Isec | tr ':' '_' | tr '+' '-' )"
+      # the username for the quay registry, since it can be derived from the quay secret's robot username
+      - id: registry_user
+        run: echo "::set-output name=username::$(cut -d '+' -f 1 <<< ${{ secrets.REGISTRY_USER }})"
+      - id: current_branch
+        run: echo "::set-output name=current_branch::$( [ "${GITHUB_REF_NAME}" != "main" ] && echo "-${GITHUB_REF_NAME:-}" )"
+
+  prod:
+    strategy:
+      # this causes containers to be built at the same time
+      matrix:
+        # these 2 containers are being built seperately because they need to have different --baseURL parameters
+        container_env:
+          # the tests_web one needs to have --baseURL='/', this allows the tests to run from any domain and allows
+          #   it to be relative off of that domain. i.e. http://jb.io is the base domain, and they'll all be off of it's root path
+          - 'TESTS_WEB'
+          # this is the normal production site being built and versioned
+          - 'PROD'
+    runs-on: ubuntu-latest
+    # requires the previous job to complete before attempting to run this job
+    needs: env_setup
+    # location where the credentials are stored for quay
+    environment: production_containers
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Building container image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: jb_web_container
+          # tagging with 2 different tags, this will let us have a consistent target of prod-latest or tests_web-latest,
+          #   but still allow us to properly version each build of the containers (with the current time). This double
+          #   tagging strategy will allow us to rollback changes on the prod environment if something goes awry
+          # example location as to where to change that in prod's ansible IaC
+          # https://github.com/JupiterBroadcasting/infra/blob/3b9490a46119ffdc70018b2ac8315346d947a06d/ansible/group_vars/core.yaml#L517
+          # tag example: prod-latest, prod-develop-latest, prod-develop-2022-09-12t23_15_22-04_00, or prod-2022-09-12t23_15_22-04_00
+          tags: ${{ matrix.container_env }}${{ needs.env_setup.outputs.current_branch }}-latest ${{ matrix.container_env }}${{ needs.env_setup.outputs.current_branch }}-${{ needs.env_setup.outputs.date_output }}
+          # prod docker container docker file
+          containerfiles: |
+            ./Dockerfile
+          # changes the build base URL based on the name passed into this build arguement
+          build-args: |
+            BASE_URL=${{ matrix.container_env }}
+
+      - name: Push To quay.io
+        id: push-to-quay
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/${{ needs.env_setup.outputs.registry_user }}
+          username: ${{ secrets.REGISTRY_USER }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+
+      - name: Print image url
+        run: echo "Image pushed to ${{ steps.push-to-quay.outputs.registry-paths }}"

.github/workflows/build_container-tests.yml

@@ -0,0 +1,67 @@
+name: Building Tests Container
+
+# built with the following guides/examples:
+# - https://github.com/marketplace/actions/push-to-registry#examples
+# - https://github.com/marketplace/actions/buildah-build#building-using-containerfiles
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+    paths:
+      - ./Dockerfile.tests
+      - ./tests/Pipfile.lock
+
+  workflow_dispatch:
+
+jobs:
+  # this sets up some variables so the next job can use them for tags
+  env_setup:
+    runs-on: ubuntu-latest
+    outputs:
+      date_output: ${{ steps.date.outputs.date_fmt }}
+      registry_user: ${{ steps.registry_user.outputs.username }}
+      current_branch: ${{ steps.current_branch.outputs.current_branch }}
+    steps:
+      # used for tagging the container with the current date down to the seconds
+      #   since we have so many changes down to the seconds was required. Example: 2022-09-08T05_42_23-04_00
+      - id: date
+        run: echo "::set-output name=date_fmt::$(date -Isec | tr ':' '_' | tr '+' '-' )"
+      # the username for the quay registry, since it can be derived from the quay secret's robot username
+      - id: registry_user
+        run: echo "::set-output name=username::$(cut -d '+' -f 1 <<< ${{ secrets.REGISTRY_USER }})"
+      - id: current_branch
+        run: echo "::set-output name=current_branch::$( [ "${GITHUB_REF_NAME}" != "main" ] && echo "-${GITHUB_REF_NAME:-}" )"
+
+  tests:
+    runs-on: ubuntu-latest
+    needs: env_setup
+    # location where the credentials are stored for quay
+    environment: production_containers
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build Image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: jb_web_container
+          # tagging with 2 different tags, this will let us have a consistent target of prod-latest or tests_web-latest,
+          #   but still allow us to properly version each build of the containers (with the current time). This double
+          #   tagging strategy will allow us to rollback changes on the prod environment if something goes awry
+          tags: tests${{ needs.env_setup.outputs.current_branch }}-latest tests${{ needs.env_setup.outputs.current_branch }}-${{ needs.env_setup.outputs.date_output }}
+          containerfiles: |
+            ./Dockerfile.tests
+
+      - name: Push To quay.io
+        id: push-to-quay
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/${{ needs.env_setup.outputs.registry_user }}
+          username: ${{ secrets.REGISTRY_USER }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+
+      - name: Print image url
+        run: echo "Image pushed to ${{ steps.push-to-quay.outputs.registry-paths }}"

.github/workflows/build_containers.md

@@ -0,0 +1,30 @@
+# Containers Docs
+
+This is the documentation for how the build_container-*.yml files works, and the needed permissions/setup around it.
+
+## Registry
+
+We're using quay.io for the container registry, reasons why are outlined in [this comment](https://github.com/JupiterBroadcasting/jupiterbroadcasting.com/issues/244#issuecomment-1213146790)
+
+## Authentication to registry
+
+1. Naviage to: <https://quay.io/signin/>
+2. Create a Red Hat account if you don't have one, or sign in
+3. Navigate to the following (be sure to replace `<username>` with your quay.io username): `https://quay.io/user/<username>?tab=repos`
+4. Click "Create New Repository" (near the top right)
+5. Give the repo a name (this will be the name of you container) i.e. jb_web_container
+6. Make sure you've selected the Repository Visibility as Public and you're going to be choosing "(Empty repository)" for the Initialize repository section
+7. Navigate to the following (be sure to replace `<username>` with your quay.io username): `https://quay.io/user/<username>?tab=robots`
+8. Click "Create Robot Account"
+9. Give the robot a name (i.e. `jb_web_container`), and a description
+10. find the repo you just created, and select the dropdown that says "None"
+11. Choose the "Write" permission
+12. Click "Add permissions"
+13. Click on the gear on the far right side of the screen for your new robot account
+14. Click "View Credentials"
+15. Create an environment variables based on [this guide](https://blog.elreydetoda.site/github-action-security/), and have the environment name called production_containers.
+16. Click "Add Secret" under Environment secrets, and add the following secrets
+    * `REGISTRY_PASSWORD` = your robot account token (long string of random characters)
+17. Then go to this link (replace `GH_user_name` with your GitHub username): `https://github.com/jupiterbroadcasting/jupiterbroadcasting.com/settings/secrets/actions/new`, to create a repo secret
+    * `REGISTRY_USER` = your robot account username
+      * This isn't necessarily that sensative, so creating a repo secret instead of an environment secret is ok

.github/workflows/deploy-prod.yml

@@ -12,7 +12,8 @@ on:
   #   scraper will run and not commit anything, but after #215 is solved
   #   then it should only run after castablasta pushes a new episode
   workflow_run:
-    workflows: ["Scrape and commit"]
+    # putting the comment here to enable prod changes in the future to leverage the created containers
+    workflows: ["Scrape and commit"] #, "Building Prod Container"]
     types:
       - completed
 

.github/workflows/e2e.yml

@@ -1,14 +1,12 @@
 name: E2E Tests
 
 on:
-  # keeping this till we figure out #371, so we can at least validate the
-  #   scrapper PRs aren't breaking things
-  push:
-    branches: [ main ]
 
-  # https://frontside.com/blog/2020-05-26-github-actions-pull_request/
   pull_request:
-
+  push:
+    branches:
+      - main
+      - develop
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
@@ -18,36 +16,33 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: run hugo build
-        uses: jakejarvis/hugo-build-action@master 
-        with:
-          args: --gc --config ./config.toml -b http://localhost:1313
-      - name: run server
-        run: |
-          cd public && python -m http.server 1313 &
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Install dependencies
-        working-directory: ./test
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-      - name: Ensure browsers are installed
-        run: python -m playwright install --with-deps
+
+      # using docker-compose, so we can re-use local Dockerfiles
+      - name: starting up containers
+        working-directory: ./test/
+        env:
+          # https://stackoverflow.com/a/52192327
+          DOCKER_BUILDKIT: 1
+          # https://stackoverflow.com/a/69500575
+          BUILDKIT_PROGRESS: plain
+        run: docker-compose up -d --build
+
       - name: Run your tests
-        working-directory: ./test
-        run: pytest --base-url http://localhost:1313 -v --junitxml report.xml e2e/*
+        working-directory: ./test/
+        # https://github.com/actions/runner/issues/241#issuecomment-577360161
+        run: script -e -c "docker-compose exec testing pytest --base-url http://jb-com --junitxml report.xml"
+
       - name: Save screenshots
         uses: actions/upload-artifact@v2
         with:
           name: screenshots
-          path: test/screenshots/
+          path: screenshots/
+
       - name: Publish Test Report
         uses: dorny/test-reporter@v1
-        if: always()
+        # TODO: potential solution in the future: https://stackoverflow.com/a/70448851
+        if: always() && github.event_name != 'pull_request'
         with:
           name: E2E Tests
-          path: './test/report.xml'
+          path: './report.xml'
           reporter: java-junit

.gitignore

@@ -18,6 +18,7 @@ $RECYCLE.BIN/
 .vscode
 .kateproject
 .kateproject.local
+.fleet
 
 
 # Ignore all the scraped data

CONTRIBUTING.md

@@ -57,6 +57,8 @@ Our base of operations for group discussions on this project is our Matrix chat
 * ✅ Create a markdown checklist to indicate what's done and what's still missing ([see this example](https://github.com/JupiterBroadcasting/jupiterbroadcasting.com/pull/112)).
 * ⚙ If you are closing a specific issue we encourage you to use [Github's closing keywords](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) to automatically close the issue once the PR is merged.
 * 🚮 Feel free to delete your feature branch after it has been merged via the PR. There’s a handy button on the GitHub interface, and in the worst case they can be reverted easily.
+* 🎯 We currently use a `develop` branch as our default, so expect your PRs to target `develop` and not the otherwise standard `main` branch
+* 🚀 We've also integrate PR deploy previews for the current `develop` branch - see the Environment section of the repo's main page sidebar. All PRs also get their own unique deploy previews found as a comment under the respective PR.
 
 
 ### What is the stack used for the site?

Dockerfile

@@ -1,7 +1,16 @@
 FROM registry.gitlab.com/pages/hugo/hugo_extended:0.101.0 as builder
+# this defaults to an empty variable
+ARG BASE_URL
 WORKDIR /site
 COPY . /site
-RUN hugo
+# when the argument is empty or PROD, do a normal production build
+#   else if it's TESTS_WEB use the specified baseURL "/", or
+#   exit 1, because that use case hasn't been provided
+#   https://explainshell.com/ is an awesome website which can explain this in more detail
+#   set -x : changes the shell's default verbosity and puts it in "debug" mode
+RUN set -x && ( [ -z "${BASE_URL}" ] || [ "${BASE_URL}" == "PROD" ] ) \
+    && hugo --gc \
+    || ( [ "${BASE_URL}" == 'TESTS_WEB' ] && hugo --gc --baseURL "/" || exit 1 )
 
 FROM nginx:alpine
 RUN rm -rf /usr/share/nginx/html/*

Dockerfile.tests

@@ -1,20 +1,13 @@
-ARG PY_VERSION=3.10
-
-FROM docker.io/python:${PY_VERSION}
+# using v1.25.2-jammy because it's the only version which has
+#   python3.10, and is latest version (python verison pinned in the test/Pipfile as well (make sure to change there as well))
+FROM mcr.microsoft.com/playwright/python:v1.25.2-jammy
 
 RUN pip install pipenv
 
 CMD [ "--base-url", "http://localhost:1313" ]
 ENTRYPOINT [ "pytest" ]
 
-RUN adduser --disabled-login --disabled-password \
-    --gecos '' --uid 1000 pytest
 COPY test/Pipfile test/Pipfile.lock ./
-RUN pipenv --bare sync --system \
-    && playwright install-deps \
-    && rm -rf /var/lib/apt/lists/*
-    # since the previous command (playwright install-deps) runs an apt-get update
-
-USER pytest
-
-RUN playwright install
+# installing depenencies to the container's system,
+#   so we don't have to worry about venvs
+RUN pipenv --bare sync --system

Makefile

@@ -14,4 +14,4 @@ deploy-prod:
 
 tests:
 	docker build --rm -f Dockerfile.tests -t jb_tests:latest .
-	docker run --rm --net=host --volume "$${PWD}":/app -w /app jb_tests:latest
+	docker run --rm --net=host --user 1000:1000 --volume "$${PWD}":/app -w /app jb_tests:latest

README.md

@@ -40,6 +40,13 @@ Deployment is done with Github Actions, see workflow file in `.github/workflows/
 At the moment it is only triggered when something in the `main` branch is changing, but it can also be set up to run at certain times.
 This would also enable scheduled publishing, since Hugo per default only builds pages which have set `date` in frontmatter to <= `now`
 
+When upgrading the hugo version these are the locations whicht it needs to be updated at:
+
+* `Dockerfile`
+* `Dockerfile.dev`
+
+Currently we're at hugo version `0.101.0`
+
 ### Credits
 
 - Parts of the functionality are from the Castanet Theme: https://github.com/mattstratton/castanet

config.toml

@@ -1,6 +1,7 @@
 baseURL = 'https://www.jupiterbroadcasting.com/'
 languageCode = 'en-us'
 title = 'Jupiter Broadcasting'
+# enableGitInfo = true
 
 theme = 'jb'
 
@@ -19,7 +20,6 @@ paginate = 12
 [params.footer.right]
   headline = "Get Involved"
 
-
 # Allow HTML in Markdown
 [markup]
   [markup.goldmark]

config/_default/params.toml

@@ -27,3 +27,6 @@ calendar.embedd= "jalb5frk4cunnaedbfemuqbhv4@group.calendar.google.com"
 analytics = "https://plausible.ktz.cloud/js/plausible.js"
 
 github_repo = "https://github.com/JupiterBroadcasting/jupiterbroadcasting.com" 
+
+# Sponsor Page
+sponsors.lookbackdays = -90