diff --git a/.github/ci-fpm-pool.conf b/.github/ci-fpm-pool.conf new file mode 100644 index 0000000..8108d79 --- /dev/null +++ b/.github/ci-fpm-pool.conf @@ -0,0 +1,10 @@ +[nextcloud] +user = runner +group = runner +listen = /run/php/nc-fpm.sock +listen.owner = www-data +listen.group = www-data +pm = static +pm.max_children = 8 +php_admin_value[memory_limit] = 512M +php_admin_value[apc.enable_cli] = 1 diff --git a/.github/ci-nginx.conf b/.github/ci-nginx.conf new file mode 100644 index 0000000..fd6547c --- /dev/null +++ b/.github/ci-nginx.conf @@ -0,0 +1,60 @@ +upstream php-handler { + server unix:/run/php/nc-fpm.sock; +} + +server { + listen 8080; + # NC_ROOT is replaced at deploy time by sed + root NC_ROOT; + client_max_body_size 512M; + + location = /robots.txt { + allow all; + log_not_found off; + access_log off; + } + + location ^~ /.well-known { + location = /.well-known/carddav { return 301 /remote.php/dav/; } + location = /.well-known/caldav { return 301 /remote.php/dav/; } + return 301 /index.php$request_uri; + } + + location ~ ^/(?:build|tests|config|lib|3rdparty|templates|data)(?:$|/) { + return 404; + } + + location ~ ^/(?:\.|autotest|occ|issue|indie|db_|console) { + return 404; + } + + location ~ \.php(?:$|/) { + rewrite ^/(?!index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|ocs-provider\/.+|.+\/richdocumentscode(_arm64)?\/proxy) /index.php$request_uri; + + fastcgi_split_path_info ^(.+?\.php)(/.*)$; + set $path_info $fastcgi_path_info; + + try_files $fastcgi_script_name =404; + + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_param PATH_INFO $path_info; + fastcgi_param HTTPS off; + fastcgi_param modHeadersAvailable true; + fastcgi_param front_controller_active true; + fastcgi_pass php-handler; + fastcgi_intercept_errors on; + fastcgi_request_buffering off; + fastcgi_read_timeout 600; + } + + location ~ \.(?:css|js|svg|gif|png|jpg|ico|wasm|tflite|map|ogg|flac)$ { + try_files $uri /index.php$request_uri; + expires 6M; + access_log off; + } + + location / { + try_files $uri $uri/ /index.php$request_uri; + } +} diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml index 4dc86a6..daf14d6 100644 --- a/.github/workflows/tests-integration.yml +++ b/.github/workflows/tests-integration.yml @@ -8,27 +8,37 @@ on: jobs: test-integration: - name: NC ${{ matrix.nextcloud-version }} - runs-on: ubuntu-latest + name: ${{ matrix.nc-branch }} + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: - nextcloud-version: ["32", "33"] + nc-branch: ["stable32", "stable33", "master"] + include: + - nc-branch: stable32 + php-version: "8.2" + apps-branch: stable32 + - nc-branch: stable33 + php-version: "8.3" + apps-branch: stable33 + - nc-branch: master + php-version: "8.3" + apps-branch: main + services: - nextcloud: - image: nextcloud:${{ matrix.nextcloud-version }} + postgres: + image: postgres:17 env: - SQLITE_DATABASE: nextcloud - NEXTCLOUD_ADMIN_USER: admin - NEXTCLOUD_ADMIN_PASSWORD: admin - ports: - - 8080:80 + POSTGRES_USER: nextcloud + POSTGRES_PASSWORD: nextcloud + POSTGRES_DB: nextcloud options: >- - --health-cmd "curl -f http://localhost/status.php || exit 1" + --health-cmd pg_isready --health-interval 10s --health-timeout 5s - --health-retries 30 - --health-start-period 60s + --health-retries 5 + ports: + - 5432:5432 smtp4dev: image: rnwood/smtp4dev:latest env: @@ -37,46 +47,140 @@ jobs: - 9025:25 - 9143:143 - 9080:80 + steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@v4 - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - name: Set up PHP ${{ matrix.php-version }} + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php-version }} + extensions: apcu, bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, mbstring, \ + pdo_pgsql, posix, session, simplexml, xmlreader, xmlwriter, zip, zlib + ini-values: >- + apc.enable_cli=1, + apc.shm_size=128M, + opcache.enable=1, + opcache.enable_cli=1, + opcache.memory_consumption=256, + opcache.interned_strings_buffer=32, + opcache.max_accelerated_files=20000, + opcache.validate_timestamps=0, + opcache.save_comments=1, + opcache.jit=1255, + opcache.jit_buffer_size=128M, + memory_limit=512M + coverage: none + + - uses: actions/setup-python@v5 with: python-version: "3.12" + cache: pip - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e ".[dev]" + - name: Install test dependencies + run: pip install -e ".[dev]" - - name: Wait for Nextcloud + - name: Checkout Nextcloud server + uses: actions/checkout@v4 + with: + submodules: true + repository: nextcloud/server + ref: ${{ matrix.nc-branch }} + path: nc-server + + - name: Checkout notifications + uses: actions/checkout@v4 + with: + repository: nextcloud/notifications + ref: ${{ matrix.nc-branch }} + path: nc-server/apps/notifications + + - name: Checkout activity + uses: actions/checkout@v4 + with: + repository: nextcloud/activity + ref: ${{ matrix.nc-branch }} + path: nc-server/apps/activity + + - name: Checkout spreed + uses: actions/checkout@v4 + with: + repository: nextcloud/spreed + ref: ${{ matrix.apps-branch }} + path: nc-server/apps/spreed + + - name: Checkout announcementcenter + uses: actions/checkout@v4 + with: + repository: nextcloud/announcementcenter + ref: main + path: nc-server/apps/announcementcenter + + - name: Checkout mail + uses: actions/checkout@v4 + with: + repository: nextcloud/mail + ref: main + path: nc-server/apps/mail + + - name: Install app dependencies + working-directory: nc-server run: | - for i in $(seq 1 60); do - if curl -sf http://localhost:8080/status.php | python3 -c "import sys,json; sys.exit(0 if json.load(sys.stdin)['installed'] else 1)" 2>/dev/null; then - echo "Nextcloud is ready" - break + for app in apps/notifications apps/activity apps/spreed apps/announcementcenter apps/mail; do + if [ -f "$app/composer.json" ]; then + echo "::group::composer install $app" + composer install --no-dev --working-dir="$app" + echo "::endgroup::" fi - echo "Waiting... ($i)" - sleep 3 done - - name: Configure Nextcloud for testing + - name: Set up Nextcloud + working-directory: nc-server run: | - NC_CONTAINER=${{ job.services.nextcloud.id }} - docker exec $NC_CONTAINER su -s /bin/bash www-data -c "php occ config:system:set ratelimit_protection_enabled --value=false --type=boolean" - docker exec $NC_CONTAINER su -s /bin/bash www-data -c "php occ config:system:set auth.bruteforce.protection.enabled --value=false --type=boolean" - docker exec $NC_CONTAINER su -s /bin/bash www-data -c "php occ config:system:set loglevel --value=0 --type=integer" - docker exec $NC_CONTAINER su -s /bin/bash www-data -c "php occ config:system:set ratelimit_overwrite files_sharing.shareapi.createshare user limit --value=1000 --type=integer" - docker exec $NC_CONTAINER su -s /bin/bash www-data -c "php occ config:system:set ratelimit_overwrite files_sharing.shareapi.createshare user period --value=60 --type=integer" - docker exec $NC_CONTAINER su -s /bin/bash www-data -c "php occ app:install spreed" || echo "spreed already installed" - docker exec $NC_CONTAINER su -s /bin/bash www-data -c "php occ app:install admin_notifications" || echo "admin_notifications already installed" - docker exec $NC_CONTAINER su -s /bin/bash www-data -c "php occ app:install announcementcenter" || echo "announcementcenter already installed" - docker exec $NC_CONTAINER su -s /bin/bash www-data -c "php occ app:install mail" + mkdir data + php occ maintenance:install --verbose --database=pgsql \ + --database-name=nextcloud --database-host=127.0.0.1 --database-port=5432 \ + --database-user=nextcloud --database-pass=nextcloud \ + --admin-user admin --admin-pass admin + php occ config:system:set loglevel --value=2 --type=integer + php occ config:system:set ratelimit.protection.enabled --value=false --type=boolean + php occ config:system:set auth.bruteforce.protection.enabled --value=false --type=boolean + php occ config:system:set memcache.local --value='\OC\Memcache\APCu' + php occ config:system:set filelocking.enabled --value=false --type=boolean + php occ config:system:set overwrite.cli.url --value="http://localhost:8080" + php occ app:enable notifications + php occ app:enable activity + php occ app:enable spreed + php occ app:enable announcementcenter + php occ app:enable mail SMTP4DEV_IP=$(docker inspect ${{ job.services.smtp4dev.id }} --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}') echo "smtp4dev IP: $SMTP4DEV_IP" - docker exec $NC_CONTAINER su -s /bin/bash www-data -c "php occ mail:account:create admin 'Test Mail' test@localhost $SMTP4DEV_IP 143 none test test $SMTP4DEV_IP 25 none test test" - docker exec $NC_CONTAINER su -s /bin/bash www-data -c "php occ mail:account:sync 1" - docker exec $NC_CONTAINER su -s /bin/bash www-data -c "php occ app:list" | head -40 + php occ mail:account:create admin 'Test Mail' test@localhost $SMTP4DEV_IP 143 none test test $SMTP4DEV_IP 25 none test test + php occ mail:account:sync 1 + php occ app:list | head -40 + + - name: Set up PHP-FPM and nginx + run: | + sudo cp .github/ci-fpm-pool.conf /etc/php/${{ matrix.php-version }}/fpm/pool.d/nextcloud.conf + sudo systemctl restart php${{ matrix.php-version }}-fpm + sudo apt-get -y install nginx > /dev/null 2>&1 + sed "s|NC_ROOT|${{ github.workspace }}/nc-server|" .github/ci-nginx.conf \ + | sudo tee /etc/nginx/sites-enabled/nextcloud.conf > /dev/null + sudo rm -f /etc/nginx/sites-enabled/default + chmod o+x $HOME ${{ github.workspace }} + sudo nginx -t + sudo systemctl restart nginx + + - name: Verify Nextcloud is accessible + run: | + for i in $(seq 1 10); do + if curl -sf http://localhost:8080/status.php > /dev/null 2>&1; then + curl -sf http://localhost:8080/status.php | python3 -c "import sys,json; d=json.load(sys.stdin); print(f'NC {d[\"versionstring\"]} installed={d[\"installed\"]}')" + break + fi + echo "Waiting for nginx+fpm... ($i)" + sleep 1 + done - name: Run integration tests env: @@ -87,8 +191,8 @@ jobs: SMTP4DEV_HTTP_PORT: "9080" SMTP4DEV_SMTP_PORT: "9025" MAIL_RECIPIENT: test@localhost - NC_CONTAINER: ${{ job.services.nextcloud.id }} MAIL_ACCOUNT_ID: "1" + NC_SERVER_DIR: ${{ github.workspace }}/nc-server run: pytest tests/integration/ -v -m integration --ignore=tests/integration/test_session_cache.py --cov=nc_mcp_server --cov-report=xml:coverage-integration.xml - name: Run session cache tests @@ -97,18 +201,30 @@ jobs: NEXTCLOUD_URL: http://localhost:8080 NEXTCLOUD_USER: admin NEXTCLOUD_PASSWORD: admin - NC_CONTAINER: ${{ job.services.nextcloud.id }} + NC_SERVER_DIR: ${{ github.workspace }}/nc-server run: pytest tests/integration/test_session_cache.py -v -m integration --cov=nc_mcp_server --cov-append --cov-report=xml:coverage-integration.xml - name: Upload coverage - uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5 + uses: codecov/codecov-action@v5 with: files: coverage-integration.xml - flags: integration,nc${{ matrix.nextcloud-version }} + flags: integration,${{ matrix.nc-branch }} token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: false - - name: Dump Nextcloud logs on failure + - name: Dump logs on failure if: failure() run: | - docker exec ${{ job.services.nextcloud.id }} cat /var/www/html/data/nextcloud.log 2>/dev/null | tail -50 || echo "No logs found" + echo "=== Nextcloud log ===" + cat nc-server/data/nextcloud.log 2>/dev/null | python3 -c "import sys,json;[print(json.loads(l).get('message','')[:200]) for l in sys.stdin]" 2>/dev/null | tail -20 || echo "No NC logs" + echo "=== nginx error log ===" + sudo cat /var/log/nginx/error.log 2>/dev/null | tail -20 || echo "No nginx logs" + echo "=== PHP-FPM log ===" + sudo journalctl -u php${{ matrix.php-version }}-fpm --no-pager -n 20 2>/dev/null || echo "No FPM journal" + echo "=== File permissions ===" + ls -la nc-server/data/ 2>/dev/null | head -5 + ls -la /run/php/ 2>/dev/null | head -5 + echo "=== FPM pool status ===" + sudo systemctl status php${{ matrix.php-version }}-fpm --no-pager 2>/dev/null | tail -10 + echo "=== Quick WebDAV test ===" + curl -v -u admin:admin -X PUT -d "test" http://localhost:8080/remote.php/dav/files/admin/debug-test.txt 2>&1 | tail -20 diff --git a/tests/integration/test_mail.py b/tests/integration/test_mail.py index 76209a4..b433cdc 100644 --- a/tests/integration/test_mail.py +++ b/tests/integration/test_mail.py @@ -50,15 +50,20 @@ def _send_test_email(subject: str, body: str = "test body", to: str = MAIL_RECIP def _sync_mail_account(account_id: int) -> None: """Trigger a mailbox sync so new messages appear in the NC database.""" - container = os.environ.get("NC_CONTAINER", "ncmcp-nextcloud-1") - cmd = f"php occ mail:account:sync {account_id}" - result = subprocess.run( - ["docker", "exec", container, "su", "-s", "/bin/bash", "www-data", "-c", cmd], - capture_output=True, - text=True, - timeout=30, - check=False, - ) + nc_server_dir = os.environ.get("NC_SERVER_DIR", "") + if nc_server_dir: + args = ["php", "occ", "mail:account:sync", str(account_id)] + result = subprocess.run(args, capture_output=True, text=True, timeout=30, check=False, cwd=nc_server_dir) + else: + container = os.environ.get("NC_CONTAINER", "ncmcp-nextcloud-1") + cmd = f"php occ mail:account:sync {account_id}" + result = subprocess.run( + ["docker", "exec", container, "su", "-s", "/bin/bash", "www-data", "-c", cmd], + capture_output=True, + text=True, + timeout=30, + check=False, + ) if result.returncode != 0: raise AssertionError(f"mail:account:sync {account_id} failed: {result.stderr}") diff --git a/tests/integration/test_session_cache.py b/tests/integration/test_session_cache.py index 1aeab50..e3a3d12 100644 --- a/tests/integration/test_session_cache.py +++ b/tests/integration/test_session_cache.py @@ -24,50 +24,45 @@ def _make_config(password: str = "admin") -> Config: return config -def _create_app_password() -> str: - """Create a fresh app password via occ CLI.""" +def _run_occ(command: str) -> subprocess.CompletedProcess[str]: + """Run a Nextcloud occ command. Supports bare-metal (NC_SERVER_DIR) and Docker (NC_CONTAINER).""" + nc_server_dir = os.environ.get("NC_SERVER_DIR", "") nc_container = os.environ.get("NC_CONTAINER", "") + if nc_server_dir: + args = ["php", "occ", *command.split()] + return subprocess.run(args, capture_output=True, text=True, timeout=15, check=False, cwd=nc_server_dir) if nc_container: - result = subprocess.run( - [ - "docker", - "exec", - nc_container, - "su", - "-s", - "/bin/bash", - "www-data", - "-c", - "php -d xdebug.mode=off occ user:auth-tokens:add --name pytest-session-test admin", - ], - capture_output=True, - text=True, - timeout=15, - check=False, - ) + args = [ + "docker", + "exec", + nc_container, + "su", + "-s", + "/bin/bash", + "www-data", + "-c", + f"php -d xdebug.mode=off occ {command}", + ] else: - result = subprocess.run( - [ - "docker", - "exec", - "ncmcp-nextcloud-1", - "sudo", - "-u", - "www-data", - "php", - "-d", - "xdebug.mode=off", - "occ", - "user:auth-tokens:add", - "--name", - "pytest-session-test", - "admin", - ], - capture_output=True, - text=True, - timeout=15, - check=False, - ) + args = [ + "docker", + "exec", + "ncmcp-nextcloud-1", + "sudo", + "-u", + "www-data", + "php", + "-d", + "xdebug.mode=off", + "occ", + *command.split(), + ] + return subprocess.run(args, capture_output=True, text=True, timeout=15, check=False) + + +def _create_app_password() -> str: + """Create a fresh app password via occ CLI.""" + result = _run_occ("user:auth-tokens:add --name pytest-session-test admin") for line in result.stdout.splitlines(): token = line.strip() if len(token) == 72 and token.isalnum(): @@ -212,35 +207,7 @@ async def test_app_password_ocs_write_works(self) -> None: def _occ(command: str) -> str: - nc_container = os.environ.get("NC_CONTAINER", "") - if nc_container: - args = [ - "docker", - "exec", - nc_container, - "su", - "-s", - "/bin/bash", - "www-data", - "-c", - f"php -d xdebug.mode=off occ {command}", - ] - else: - args = [ - "docker", - "exec", - "ncmcp-nextcloud-1", - "sudo", - "-u", - "www-data", - "php", - "-d", - "xdebug.mode=off", - "occ", - *command.split(), - ] - result = subprocess.run(args, capture_output=True, text=True, timeout=15, check=False) - return result.stdout.strip() + return _run_occ(command).stdout.strip() class TestSessionExpiryRecovery: