Add support for fixing style as a task in another process

Author: scottwittenburg

.env-dummy

@@ -1,6 +1,12 @@
 # Go to smee.io to generate a URL here
 SMEE_URL=https://smee.io/CHANGEME
 
+# No need to change this for testing
+REDIS_HOST=rq-server
+
+# Optionally customize port redis listens on
+REDIS_PORT=6379
+
 # You don't need to change this unless you change the docker-compose volumes
 GITHUB_PRIVATE_KEY=/app/spackbot/spack-bot-develop.private-key.pem
 

.github/workflows/build-deploy.yaml

@@ -12,19 +12,24 @@ on:
 jobs:
   deploy-test-containers:
     runs-on: ubuntu-latest
-    name: Build Spackbot Container
+    strategy:
+      fail-fast: false
+      # matrix: [tag, path to Dockerfile, label]
+      matrix:
+        dockerfile: [[spack-bot, ./Dockerfile, Spackbot],
+                     [spackbot-workers, ./workers/Dockerfile, "Spackbot Workers"]]
+    name: Build ${{matrix.dockerfile[2]}} Container
     steps:
       - name: Checkout
         uses: actions/checkout@v2        
 
       - name: Build and Run Test Container
         run: |
-            docker build -t ghcr.io/spack/spack-bot:latest .
-            docker tag ghcr.io/spack/spack-bot:latest ghcr.io/spack/spack-bot:${GITHUB_SHA::8} 
-
+            docker build -f ${{matrix.dockerfile[1]}} -t ghcr.io/spack/${{matrix.dockerfile[0]}}:latest .
+            docker tag ghcr.io/spack/${{matrix.dockerfile[0]}}:latest ghcr.io/spack/${{matrix.dockerfile[0]}}:${GITHUB_SHA::8}
       - name: Login and Deploy Test Container
         if: (github.event_name != 'pull_request')
         run: |
             docker images
             echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ secrets.GHCR_USERNAME }} --password-stdin
-            docker push --all-tags ghcr.io/spack/spack-bot
+            docker push --all-tags ghcr.io/spack/${{matrix.dockerfile[0]}}

docker-compose.yml

@@ -8,6 +8,25 @@ services:
       context: .
       dockerfile: smee/Dockerfile
 
+  rq-worker:
+    build:
+      context: .
+      dockerfile: workers/Dockerfile
+    env_file:
+      - ./.env
+    deploy:
+      replicas: 1
+
+  rq-server:
+    env_file:
+      - ./.env
+    image: redis:alpine
+    expose:
+      - ${REDIS_PORT}
+    volumes:
+      - redis-data:/data
+      - redis-conf:/usr/local/etc/redis/redis.conf
+
   spackbot:
     build:
       context: .
@@ -29,3 +48,7 @@ services:
       - ./.env
     links:
       - smee
+
+volumes:
+  redis-data:
+  redis-conf:

requirements.txt

@@ -1,6 +1,7 @@
 aiohttp
 gidgethub
 python_dotenv
+rq
 sh
 
 # Add these so we don't wait for install

spackbot/handlers/style.py

@@ -5,10 +5,12 @@
 
 import spackbot.comments as comments
 import spackbot.helpers as helpers
-from sh.contrib import git
 import logging
-import os
-import sh
+
+from spackbot.workers import fix_style_task, work_queue
+
+# If we don't provide a timeout, the default in RQ is 180 seconds
+WORKER_JOB_TIMEOUT = 6 * 60 * 60
 
 logger = logging.getLogger(__name__)
 
@@ -31,13 +33,6 @@ async def style_comment(event, gh):
             await gh.post(comments_url, {}, data={"body": comments.style_message})
 
 
-def is_up_to_date(output):
-    """
-    A commit can fail if there are no changes!
-    """
-    return "branch is up to date" in output
-
-
 async def fix_style(event, gh):
     """
     Respond to a request to fix style.
@@ -58,7 +53,7 @@ async def fix_style(event, gh):
         return f"Sorry {sender}, I cannot do that for you. Only users with write can make this request!"
 
     # Tell the user the style fix is going to take a minute or two
-    message = "Let me see if I can fix that for you! This might take a moment..."
+    message = "Let me see if I can fix that for you!"
     await gh.post(event.data["issue"]["comments_url"], {}, data={"body": message})
 
     # Get the username of the original committer
@@ -72,49 +67,15 @@ async def fix_style(event, gh):
     full_name = pr["head"]["repo"]["full_name"]
     fork_url = f"git@github.com:{full_name}.git"
 
-    # At this point, we can clone the repository and make the change
-    with helpers.temp_dir() as cwd:
-
-        # Clone a fresh spack develop to use for spack style
-        git("clone", helpers.spack_upstream, "spack-develop")
-
-        spack = sh.Command(f"{cwd}/spack-develop/bin/spack")
-
-        # clone the develop repository to another folder for our PR
-        git("clone", "spack-develop", "spack")
-
-        os.chdir("spack")
-        git("config", "user.name", user)
-        git("config", "user.email", email)
-
-        # This will authenticate the push with the added ssh credentials
-        git("remote", "add", "upstream", helpers.spack_upstream)
-        git("remote", "set-url", "origin", fork_url)
-
-        # we're on upstream/develop. Fetch and check out just the PR branch
-        git("fetch", "origin", f"{branch}:{branch}")
-        git("checkout", branch)
-
-        # Save the message for the user
-        res, err = helpers.run_command(spack, ["--color", "never", "style", "--fix"])
-        message = comments.get_style_message(res)
-
-        # Commit (allow for no changes)
-        res, err = helpers.run_command(
-            git,
-            ["commit", "-a", "-m", f"[spackbot] updating style on behalf of {user}"],
-        )
-
-        # Continue differently if the branch is up to date or not
-        if is_up_to_date(res):
-            message += "\nI wasn't able to make any further changes, but please see the message above for remaining issues you can fix locally!"
-            return message
-        message += "\nI've updated the branch with isort fixes."
-
-        # Finally, try to push, update the message if permission not allowed
-        try:
-            git("push", "origin", branch)
-        except Exception:
-            message += "\n\nBut it looks like I'm not able to push to your branch. 😭️ Did you check maintainer can edit when you opened the PR?"
+    task_q = work_queue.get_queue()
+    fix_style_job = task_q.enqueue(
+        fix_style_task,
+        user,
+        email,
+        fork_url,
+        branch,
+        job_timeout=WORKER_JOB_TIMEOUT,
+    )
+    logger.info(f"Fix style job queued: {fix_style_job.id}")
 
     return message

spackbot/workers.py

@@ -0,0 +1,92 @@
+import logging
+import os
+from sh.contrib import git
+import sh
+
+from redis import Redis
+from rq import Queue
+
+import spackbot.comments as comments
+import spackbot.helpers as helpers
+
+logger = logging.getLogger(__name__)
+
+REDIS_HOST = os.environ.get("REDIS_HOST", "localhost")
+REDIS_PORT = int(os.environ.get("REDIS_PORT", "6379"))
+
+
+class WorkQueue:
+    def __init__(self):
+        logger.info(f"WorkQueue creating redis connection ({REDIS_HOST}, {REDIS_PORT})")
+        self.redis_conn = Redis(host=REDIS_HOST, port=REDIS_PORT)
+        self.task_q = Queue(name="tasks", connection=self.redis_conn)
+
+    def get_queue(self):
+        return self.task_q
+
+
+work_queue = WorkQueue()
+
+
+def is_up_to_date(output):
+    """
+    A commit can fail if there are no changes!
+    """
+    return "branch is up to date" in output
+
+
+async def fix_style_task(user, email, fork_url, branch):
+    # In the worker container, the code in __main__.py will not have
+    # run to set the logging level.
+    logging.basicConfig(level=logging.INFO)
+    style_logger = logging.getLogger("fix_style_task")
+
+    # At this point, we can clone the repository and make the change
+    with helpers.temp_dir() as cwd:
+
+        # Clone a fresh spack develop to use for spack style
+        git("clone", helpers.spack_upstream, "spack-develop")
+
+        spack = sh.Command(f"{cwd}/spack-develop/bin/spack")
+
+        # clone the develop repository to another folder for our PR
+        git("clone", "spack-develop", "spack")
+
+        os.chdir("spack")
+        git("config", "user.name", user)
+        git("config", "user.email", email)
+
+        # This will authenticate the push with the added ssh credentials
+        git("remote", "add", "upstream", helpers.spack_upstream)
+        git("remote", "set-url", "origin", fork_url)
+
+        # we're on upstream/develop. Fetch and check out just the PR branch
+        git("fetch", "origin", f"{branch}:{branch}")
+        git("checkout", branch)
+
+        # Save the message for the user
+        res, err = helpers.run_command(spack, ["--color", "never", "style", "--fix"])
+        message = f"fix_style_task, user = {user}, email = {email}, fork = {fork_url}, branch = {branch}\n"
+        message += comments.get_style_message(res)
+
+        # Commit (allow for no changes)
+        res, err = helpers.run_command(
+            git,
+            ["commit", "-a", "-m", f"[spackbot] updating style on behalf of {user}"],
+        )
+
+        # Continue differently if the branch is up to date or not
+        if is_up_to_date(res):
+            message += "\nunable to make any further changes"
+            style_logger.log(message)
+            return
+
+        message += "\nupdated the branch with isort fixes."
+
+        # Finally, try to push, update the message if permission not allowed
+        try:
+            git("push", "origin", branch)
+        except Exception:
+            message += "\n\nunable to push to branch"
+
+        style_logger.log(message)

workers/Dockerfile

@@ -0,0 +1,11 @@
+FROM ghcr.io/spack/ubuntu-bionic:latest
+
+COPY workers/requirements.txt /source/requirements.txt
+COPY workers/entrypoint.sh /source/entrypoint.sh
+COPY spackbot /source/spackbot
+
+RUN pip3 install --upgrade pip setuptools wheel && \
+    pip3 install -r /source/requirements.txt
+
+WORKDIR /source
+ENTRYPOINT ["/bin/bash", "/source/entrypoint.sh"]

workers/entrypoint.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Make sure spack can be found by worker jobs
+. $SPACK_ROOT/share/spack/setup-env.sh
+
+# Define REDIS_HOST and REDIS_PORT in .env file or k8s deployment.  Workers
+# always take jobs from the "copy" queue first and then the "index" queue
+# when passed in this order.
+rq worker -u redis://${REDIS_HOST}:${REDIS_PORT} --with-scheduler tasks

workers/requirements.txt

@@ -0,0 +1,2 @@
+aiohttp
+rq