bug fixes, updates, and modernization

.agent/rules/env.md

@@ -0,0 +1,7 @@
+---
+trigger: always_on
+---
+
+Always execute Python commands, tests, and scripts using the interpreter located at: `/Users/harmsm/miniconda3/envs/topiary/bin/python`. Do not use the global python command. Please highlight the changes in each file you modify in the Antigravity IDE and allow me to approve them. 
+
+

.github/workflows/python-app.yml

@@ -1,9 +1,6 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-# NOTE: this does not run with --run-raxml or --run-generax flags, so these
-# tests will be skipped.
-
 name: topiary
 
 on:
@@ -12,84 +9,96 @@ on:
   pull_request:
     branches: [ main   ]
 
+concurrency:
+  group: topiary-ci-group
+  cancel-in-progress: false
+
 jobs:
   build-nix:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
-      max-parallel: 3
+      max-parallel: 1
       matrix:
         os: [ubuntu-latest,macos-latest]
-        python-version: ['3.8', '3.9', '3.10' , '3.11']
+        python-version: ['3.11', '3.12', '3.13']
 
     steps:
     - name: checkout
-      uses: actions/checkout@v3.0.0
+      uses: actions/checkout@v4
 
     - name: set up miniconda
-      uses: conda-incubator/setup-miniconda@v2
+      uses: conda-incubator/setup-miniconda@v3
       with:
         activate-environment: topiary
         environment-file: environment.yml
         channel-priority: strict
         python-version: ${{ matrix.python-version }}
         auto-activate-base: false
+        miniconda-version: "latest"
 
-    - name: install topiary
-      shell: bash -l {0}
+    - name: get dependency hashes
+      id: get-hashes
+      shell: bash
       run: |
-        # platform specific packages
-        conda install -c conda-forge -c bioconda "mpi4py" "muscle>=5.0" "raxml-ng>=1.1" "generax>=2.0" "blast>=2.2"
-        python -m pip install . -vv
+        GENERAX_HASH=$(git ls-remote https://github.com/harmslab/generax.git HEAD | awk '{print $1}')
+        RAXML_HASH=$(git ls-remote https://github.com/harmslab/raxml-ng.git HEAD | awk '{print $1}')
+        echo "generax_hash=$GENERAX_HASH" >> $GITHUB_OUTPUT
+        echo "raxml_hash=$RAXML_HASH" >> $GITHUB_OUTPUT
 
-    - name: run flake8
-      shell: bash -l {0}
-      run: |
-        conda install flake8
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+    - name: cache generax
+      id: cache-generax
+      uses: actions/cache@v4
+      with:
+        path: bin-cache/generax
+        key: ${{ runner.os }}-generax-${{ steps.get-hashes.outputs.generax_hash }}
 
-    - name: run pytest
+    - name: cache raxml-ng
+      id: cache-raxml
+      uses: actions/cache@v4
+      with:
+        path: bin-cache/raxml-ng
+        key: ${{ runner.os }}-raxml-${{ steps.get-hashes.outputs.raxml_hash }}
+
+    - name: install topiary
       shell: bash -l {0}
       run: |
-        conda install pytest
-        export PATH=`which conda | xargs dirname`/envs/topiary/bin/:$PATH
-        pytest
-
-  build-windows:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      max-parallel: 3
-      matrix:
-        os: [windows-latest]
-        python-version: ['3.8', '3.9', '3.10' , '3.11']
+        # platform specific packages
+        conda install -c conda-forge cmake
+        python -m pip install .[dev] -vv
+        python -m pip install coverage flake8 pytest genbadge[tests] pytest-mock
 
-    steps:
-    - name: checkout
-      uses: actions/checkout@v3.0.0
+        # Ensure bin-cache exists
+        mkdir -p bin-cache
 
-    - name: set up miniconda
-      uses: conda-incubator/setup-miniconda@v2
-      with:
-        activate-environment: topiary
-        environment-file: environment.yml
-        python-version: ${{ matrix.python-version }}
-        auto-activate-base: false
+        # compile generax if not cached
+        if [ ! -f bin-cache/generax ]; then
+            cd dependencies
+            conda run -n topiary bash compile-generax.sh
+            cp $CONDA_PREFIX/bin/generax ../bin-cache/
+            cd ..
+        else
+            cp bin-cache/generax $CONDA_PREFIX/bin/
+        fi
 
-    - name: install topiary
-      shell: pwsh
-      run: |
-        python -m pip install . -vv
+        # compile raxml if not cached
+        if [ ! -f bin-cache/raxml-ng ]; then
+            cd dependencies
+            conda run -n topiary bash compile-raxml-ng.sh
+            cp $CONDA_PREFIX/bin/raxml-ng ../bin-cache/
+            cd ..
+        else
+            cp bin-cache/raxml-ng $CONDA_PREFIX/bin/
+        fi
 
     - name: run flake8
-      shell: pwsh
+      shell: bash -l {0}
       run: |
-        conda install flake8
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 src/topiary tests --count --select=E9,F63,F7,F82 --show-source --statistics
 
     - name: run pytest
-      shell: pwsh
+      shell: bash -l {0}
+      env:
+        TOPIARY_MPI_OVERSUBSCRIBE: "1"
       run: |
-        conda install pytest
-        $Env:PATH = "C:\Miniconda3\Scripts\;$Env:PATH"
-        pytest
+        pytest tests/ --run-raxml --run-generax --run-blast 

dependencies/README

@@ -0,0 +1,10 @@
+You can use these scripts to download and compile the Harmslab-specific forks
+of generax and raxml-ng. We recommend using our pre-compiled binaries. 
+
+NOTE: these should be run inside the conda environment holding topiary. 
+
+Example usage:
+
+conda activate topiary
+bash compile-generax.sh
+bash compile-raxml-ng.sh

dependencies/compile-generax.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# NOTE: this script should generally be run by "install.sh", not run by itself.
+
+# -----------------------------------------------------------------------------
+# COMPILING ON A CLUSTER
+# 
+# To run generax on a cluster, you need to make sure that it is compiled
+# using your cluster's compilers and MPI. To do this, you can add a line like
+# this one to the top of this script:
+
+# module load mpi/gcc/13.1.0/openmpi/4.1.6
+
+# This makes sure that generax compiles against the correct openmpi library, 
+# which allows it to run across multiple cores/nodes on a cluster. The exact
+# command depends on your cluster. If your cluster uses SLURM, you can usually
+# find the correct command by running `module spider mpi`. If you are unsure
+# which command to use, please contact your cluster administrator.
+
+# -----------------------------------------------------------------------------
+# No user modifications should be required below this line
+
+# Check for conda environment
+if [ -z "$CONDA_PREFIX" ]; then
+    echo "ERROR: \$CONDA_PREFIX is not set. This script should be run via "
+    echo "'conda run -n <env_name> bash compile-generax.sh'"
+    exit 1
+fi
+
+# Clean up environment
+rm -rf generax
+
+# Download repo and submodules
+git clone https://github.com/harmslab/generax.git
+cd generax
+git submodule update --init --recursive
+
+# Build binary
+mkdir build
+cd build
+cmake -DUSE_MPI=ON -DCORAX_BUILD_PORTABLE=ON ..
+make -j 4
+
+# Copy binary to final location
+cp bin/generax $CONDA_PREFIX/bin/
+

dependencies/compile-raxml-ng.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# This script should generally be run by "install.sh", not run by itself. It 
+# should not require any user edits. 
+
+# Check for conda environment
+if [ -z "$CONDA_PREFIX" ]; then
+    echo "ERROR: \$CONDA_PREFIX is not set. This script should be run via "
+    echo "'conda run -n <env_name> bash compile-raxml-ng.sh'"
+    exit 1
+fi
+
+# Clean up environment
+rm -rf raxml-ng
+
+# Download repo and submodules
+git clone https://github.com/harmslab/raxml-ng.git
+cd raxml-ng
+git submodule update --init --recursive
+
+# Build binary
+mkdir build
+cd build
+cmake -DTERRAPHAST_ARCH_NATIVE=OFF -DENABLE_RAXML_SIMD=OFF -DENABLE_PLLMOD_SIMD=OFF ..
+make -j 4
+
+# Copy binary to final location
+cp bin/raxml-ng $CONDA_PREFIX/bin/
+