diff --git a/.github/workflows/backend_checks.yml b/.github/workflows/backend_checks.yml index 22e6c508179..a6824fa5c76 100644 --- a/.github/workflows/backend_checks.yml +++ b/.github/workflows/backend_checks.yml @@ -39,7 +39,7 @@ jobs: strategy: matrix: # NOTE: These are the currently supported/tested Python Versions - python_version: ["3.8.14", "3.9.14", "3.10.7"] + python_version: ["3.10.7"] runs-on: ubuntu-latest steps: - name: Checkout @@ -67,211 +67,17 @@ jobs: path: /tmp/python-${{ matrix.python_version }}.tar retention-days: 1 - ################### - ## Static Checks ## - ################### - Static-Checks: - strategy: - matrix: - session_name: - ["isort", "black", "mypy", "pylint", "xenon", "check_install"] - runs-on: ubuntu-latest - continue-on-error: true - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set Up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - cache: "pip" - - - name: Install Nox - run: pip install nox>=2022 - - - name: Run Static Check - run: nox -s ${{ matrix.session_name }} - - ################# - ## Misc Checks ## - ################# - Misc-Tests: - needs: Build - strategy: - matrix: - test_selection: - - "check_fides_annotations" - - "fides_db_scan" - - "docs_check" - - "minimal_config_startup" - - runs-on: ubuntu-latest - timeout-minutes: 15 - continue-on-error: true - steps: - - name: Download container - uses: actions/download-artifact@v3 - with: - name: python-${{ env.DEFAULT_PYTHON_VERSION }} - path: /tmp/ - - - name: Load image - run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar - - - name: Checkout - uses: actions/checkout@v3 - - - name: Set Up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - cache: "pip" - - - name: Install Nox - run: pip install nox>=2022 - - - name: Run test suite - run: nox -s "${{ matrix.test_selection }}" - - ################ - ## Safe Tests ## - ################ - Safe-Tests: - needs: Build - strategy: - matrix: - python_version: ["3.8.14", "3.9.14", "3.10.7"] - test_selection: - - "ctl-not-external" - - "ops-unit" - - "ops-integration" - - "lib" - - runs-on: ubuntu-latest - timeout-minutes: 15 - continue-on-error: true - steps: - - name: Download container - uses: actions/download-artifact@v3 - with: - name: python-${{ matrix.python_version }} - path: /tmp/ - - - name: Load image - run: docker load --input /tmp/python-${{ matrix.python_version }}.tar - - - name: Checkout - uses: actions/checkout@v3 - - - name: Set Up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - cache: "pip" - - - name: Install Nox - run: pip install nox>=2022 - - - name: Run test suite - run: nox -s "pytest(${{ matrix.test_selection }})" - - - name: Upload coverage - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: true - - ################## - ## Unsafe Tests ## - ################## - # NOTE: Matrixes aren't used here due to the danger of race conditions for external resources - - Pytest-Ctl-External: - needs: Build - strategy: - max-parallel: 1 # This prevents collisions in shared external resources - matrix: - python_version: ["3.8.14", "3.9.14", "3.10.7"] - runs-on: ubuntu-latest - timeout-minutes: 20 - # In PRs run with the "unsafe" label, or run on a "push" event to main - if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push' - steps: - - name: Download container - uses: actions/download-artifact@v3 - with: - name: python-${{ matrix.python_version }} - path: /tmp/ - - - name: Load image - run: docker load --input /tmp/python-${{ matrix.python_version }}.tar - - - name: Checkout - uses: actions/checkout@v3 - - - name: Install Nox - run: pip install nox>=2022 - - - name: Run external test suite - run: nox -s "pytest(ctl-external)" - env: - SNOWFLAKE_FIDESCTL_PASSWORD: ${{ secrets.SNOWFLAKE_FIDESCTL_PASSWORD }} - REDSHIFT_FIDESCTL_PASSWORD: ${{ secrets.REDSHIFT_FIDESCTL_PASSWORD }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_FIDESCTL_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_FIDESCTL_ACCESS_KEY }} - OKTA_CLIENT_TOKEN: ${{ secrets.OKTA_FIDESCTL_CLIENT_TOKEN }} - AWS_DEFAULT_REGION: us-east-1 - BIGQUERY_CONFIG: ${{ secrets.BIGQUERY_CONFIG }} - - External-Datastores: - needs: Build - strategy: - max-parallel: 1 # This prevents collisions in shared external resources - matrix: - python_version: ["3.8.14", "3.9.14", "3.10.7"] - runs-on: ubuntu-latest - timeout-minutes: 20 - # In PRs run with the "unsafe" label, or run on a "push" event to main - if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push' - steps: - - name: Download container - uses: actions/download-artifact@v3 - with: - name: python-${{ matrix.python_version }} - path: /tmp/ - - - name: Load image - run: docker load --input /tmp/python-${{ matrix.python_version }}.tar - - - name: Checkout - uses: actions/checkout@v3 - - - name: Install Nox - run: pip install nox>=2022 - - - name: Integration Tests (External) - env: - REDSHIFT_TEST_URI: ${{ secrets.REDSHIFT_TEST_URI }} - REDSHIFT_TEST_DB_SCHEMA: fidesops_test - BIGQUERY_KEYFILE_CREDS: ${{ secrets.BIGQUERY_KEYFILE_CREDS }} - BIGQUERY_DATASET: fidesopstest - SNOWFLAKE_TEST_URI: ${{ secrets.SNOWFLAKE_TEST_URI }} - run: nox -s "pytest(ops-external-datastores)" - External-SaaS-Connectors: needs: Build runs-on: ubuntu-latest timeout-minutes: 20 - # In PRs run with the "unsafe" label, or run on a "push" event to main - if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push' permissions: contents: read id-token: write strategy: max-parallel: 1 # This prevents collisions in shared external resources matrix: - python_version: ["3.8.14", "3.9.14", "3.10.7"] + python_version: ["3.10.7"] steps: - name: Download container uses: actions/download-artifact@v3 diff --git a/.github/workflows/cli_checks.yml b/.github/workflows/cli_checks.yml deleted file mode 100644 index 571e5c3e52a..00000000000 --- a/.github/workflows/cli_checks.yml +++ /dev/null @@ -1,57 +0,0 @@ -# This file is used for testing various CLI commands, -# either for development or users, that may otherwise be -# difficult to test in a programmatic way or require -# significant amounts of setup -name: CLI Command Checks - -on: - pull_request: - paths-ignore: - - "**.md" - push: - branches: - - "main" - -env: - DEFAULT_PYTHON_VERSION: "3.10.7" - -jobs: - Test-Envs: - runs-on: ubuntu-latest - timeout-minutes: 25 - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install Nox - run: pip install nox>=2022 - - - name: Initialize the test environment - run: nox -s "fides_env(test)" -- test - - Fides-Deploy: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - - - name: Install Nox - run: pip install nox>=2022 - - - name: Build the sample image - run: nox -s "build(sample)" - - - name: Install fides - run: pip install . - - - name: Start the sample application - run: fides deploy up --no-pull --no-init - - - name: Stop the sample application - run: fides deploy down diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index ec6cd000d90..00000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,70 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ main ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ main ] - schedule: - - cron: '00 6 * * 1' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'javascript', 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # â„šī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # âœī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/cypress_e2e.yml b/.github/workflows/cypress_e2e.yml deleted file mode 100644 index 74fb784f0b2..00000000000 --- a/.github/workflows/cypress_e2e.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Cypress E2E Tests - -on: - pull_request: - paths-ignore: - - "**.md" - push: - branches: - - "main" - -env: - CI: true - -jobs: - Cypress-E2E: - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [16.x] - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install Nox - run: pip install nox>=2022 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - - name: Start test environment in the background - run: nox -s "fides_env(test)" -- keep_alive - - - name: Install dependencies - run: | - cd clients/cypress-e2e - npm install - - - name: Cypress E2E tests - uses: cypress-io/github-action@v5 - with: - working-directory: clients/cypress-e2e - install: false - wait-on: "http://localhost:8080, http://localhost:3001" - record: true - env: - # pass the Cypress Cloud record key as an environment variable - CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} - # pass GitHub token to allow accurately detecting a build vs a re-run build - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Teardown - run: nox -s teardown diff --git a/.github/workflows/frontend_checks.yml b/.github/workflows/frontend_checks.yml deleted file mode 100644 index 99a433f0a02..00000000000 --- a/.github/workflows/frontend_checks.yml +++ /dev/null @@ -1,140 +0,0 @@ -name: Frontend Code Checks - -on: - pull_request: - paths: - - "clients/**" - - ".github/workflows/frontend_pr_checks.yml" - push: - branches: - - "main" - -env: - CI: true - -jobs: - Admin-UI-Unit: - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [16.x] - defaults: - run: - working-directory: clients/admin-ui - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - - name: Install dependencies - run: npm install - - - name: Lint - run: npm run lint - - - name: Format - run: npm run format:ci - - - name: Jest test - run: npm run test:ci - - - name: Build - run: npm run build - - Admin-UI-Cypress: - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [16.x] - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - - name: Install dependencies - run: | - cd clients/admin-ui - npm install - - - name: Cypress E2E tests - uses: cypress-io/github-action@v4 - with: - working-directory: clients/admin-ui - install: false - start: npm run cy:start - wait-on: "http://localhost:3000" - wait-on-timeout: 120 - - - uses: actions/upload-artifact@v3 - if: failure() - with: - name: cypress-videos - path: /home/runner/work/fides/fides/clients/admin-ui/cypress/videos/*.mp4 - - Privacy-Center-Unit: - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [16.x] - defaults: - run: - working-directory: clients/privacy-center - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - - name: Install dependencies - run: npm install - - - name: Lint - run: npm run lint - - - name: Format - run: npm run format:ci - - - name: Build - run: npm run build - - - name: Jest test - run: npm run test:ci - - Privacy-Center-Cypress: - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [16.x] - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - - - name: Install dependencies - run: | - cd clients/privacy-center - npm install - - - name: Cypress E2E tests - uses: cypress-io/github-action@v4 - with: - working-directory: clients/privacy-center - install: false - start: npm run cy:start - wait-on: "http://localhost:3000" - wait-on-timeout: 120 diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml deleted file mode 100644 index 594d695fdbe..00000000000 --- a/.github/workflows/publish_docker.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: Docker Build & Push - -on: - push: - branches: - - main - tags: - - "*" - -env: - DOCKER_USER: ethycaci - DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} - -jobs: - push-fides: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # This is required to properly tag images - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ env.DOCKER_USER }} - password: ${{ env.DOCKER_TOKEN }} - - - name: Install Dev Requirements - run: pip install -r dev-requirements.txt - - - name: Build Fides Image - run: nox -s "build(prod)" - - - name: Push Fides Dev Tag - run: nox -s "push(dev)" - - - name: Check Prod Tag - id: check-tag - run: | - if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo ::set-output name=match::true - fi - - name: Push Fides Prod Tags - if: steps.check-tag.outputs.match == 'true' - run: nox -s "push(prod)" diff --git a/.github/workflows/publish_docs.yaml b/.github/workflows/publish_docs.yaml deleted file mode 100644 index 4d90ad26df7..00000000000 --- a/.github/workflows/publish_docs.yaml +++ /dev/null @@ -1,66 +0,0 @@ -name: Publish Docs - -on: - push: - branches: - - main - release: - types: [published] - -env: - TAG: ${{ github.event.release.tag_name }} - PROD_PUBLISH: true - -jobs: - publish_docs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Echo the Release Tag - run: echo ${{ env.TAG }} - - - name: Install Nox - run: pip install nox>=2022 - - - name: Install Docs Requirements - run: pip install -r docs/fides/requirements.txt - - - name: Install fides - run: pip install -e ./[all] - - - name: Checkout the gh-pages branches - run: git fetch origin gh-pages --depth=1 - - - name: Build docs - run: nox -s generate_docs - - # This is required by "mike" - - name: Set the Git Username & Email - run: | - git config user.name github-actions - git config user.email github-actions@github.com - - # Deploys docs without a version in the URL for legacy links - # This will match the "stable" version of the versioned docs - - - name: Deploy the Legacy Docs if a Release is Published - if: ${{ env.TAG }} - run: mkdocs gh-deploy -v -f docs/fides/mkdocs.yml --force - - - name: Deploy Stable Docs if a Release is Published - if: ${{ env.TAG }} - run: mike deploy --config-file docs/fides/mkdocs.yml --push --update-aliases ${{ env.TAG }} stable - - # This will match "stable" when a new release is cut - - name: Deploy Dev Docs - run: mike deploy --config-file docs/fides/mkdocs.yml --push dev - - - name: Set Default Version - run: mike set-default --config-file docs/fides/mkdocs.yml --push stable diff --git a/.github/workflows/publish_package.yaml b/.github/workflows/publish_package.yaml deleted file mode 100644 index 7b04c21c457..00000000000 --- a/.github/workflows/publish_package.yaml +++ /dev/null @@ -1,72 +0,0 @@ -name: Publish Fides - -on: - push: - branches: - - main - tags: - - "*" - -jobs: - upload_to_pypi: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # This is required to properly tag packages - - - name: Setup Python 3.9 - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - - name: Use Node.js 16 - uses: actions/setup-node@v3 - with: - node-version: 16 - - - name: Install node modules - run: | - cd clients/admin-ui - npm install - - - name: Build and export frontend files - run: | - cd clients/admin-ui - npm run prod-export - - - name: Install Twine and wheel - run: pip install twine wheel - - # The git reset is required here because the build modifies - # egg-info and the wheel becomes a dirty version - - name: Build the sdist - run: | - python setup.py sdist - git reset --hard - - - name: Build the wheel - run: python setup.py bdist_wheel - - - name: Check Prod Tag - id: check-tag - run: | - if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo ::set-output name=match::true - fi - - # We don't upload to Test PyPI if it is a real release - - name: Upload to test pypi - if: steps.check-tag.outputs.match != 'true' - run: twine upload --repository testpypi dist/* - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }} - - - - name: Upload to pypi - if: steps.check-tag.outputs.match == 'true' - run: twine upload dist/* - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/release_event.yml b/.github/workflows/release_event.yml deleted file mode 100644 index a803e7c8375..00000000000 --- a/.github/workflows/release_event.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Notify Downstream Projects of Release - -on: - release: - types: [released] - -jobs: - fidesctl-plus: - runs-on: ubuntu-latest - steps: - - name: Send Repository Dispatch Event - uses: peter-evans/repository-dispatch@v2 - with: - client-payload: '{"tag": "${{ github.event.release.tag_name }}", "url": "${{ github.event.release.html_url }}"}' - event-type: new-fides-release - repository: ethyca/fidesctl-plus - token: ${{ secrets.DISPATCH_ACCESS_TOKEN }} - - fidesops-plus: - runs-on: ubuntu-latest - steps: - - name: Send Repository Dispatch Event - uses: peter-evans/repository-dispatch@v2 - with: - client-payload: '{"tag": "${{ github.event.release.tag_name }}", "url": "${{ github.event.release.html_url }}"}' - event-type: new-fides-release - repository: ethyca/fidesops-plus - token: ${{ secrets.DISPATCH_ACCESS_TOKEN }} - - fides-helm: - runs-on: ubuntu-latest - steps: - - name: Send Repository Dispatch Event - uses: peter-evans/repository-dispatch@v2 - with: - client-payload: '{"tag": "${{ github.event.release.tag_name }}", "url": "${{ github.event.release.html_url }}"}' - event-type: new-fides-release - repository: ethyca/fides-helm - token: ${{ secrets.DISPATCH_ACCESS_TOKEN }} diff --git a/data/saas/config/zendesk_config.yml b/data/saas/config/zendesk_config.yml index 38d35a38965..668778b48b0 100644 --- a/data/saas/config/zendesk_config.yml +++ b/data/saas/config/zendesk_config.yml @@ -21,7 +21,7 @@ saas_config: test_request: method: GET - path: /api/v2/users/search.json + path: /api/v2/userz/search.json query_params: - name: query value: test@ethyca diff --git a/noxfiles/test_setup_nox.py b/noxfiles/test_setup_nox.py index 7c6b8126e73..464ff5e25cf 100644 --- a/noxfiles/test_setup_nox.py +++ b/noxfiles/test_setup_nox.py @@ -153,12 +153,15 @@ def pytest_ops(session: Session, mark: str, coverage_arg: str) -> None: "VAULT_NAMESPACE", "-e", "VAULT_TOKEN", + "-e", + "FIDES__DEV_MODE=false", CI_ARGS_EXEC, CONTAINER_NAME, "pytest", coverage_arg, OPS_TEST_DIR, "-m", - "integration_saas", + "integration_saas_test", + "--tb=no", ) session.run(*run_command, external=True) diff --git a/pyproject.toml b/pyproject.toml index ec8fc11360b..937bc277b74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -183,6 +183,7 @@ markers = [ "integration_mariadb", "integration_bigquery", "integration_saas", + "integration_saas_test", "integration_saas_override", "integration_mailchimp", "integration_zendesk", diff --git a/tests/ops/integration_tests/saas/test_zendesk_task.py b/tests/ops/integration_tests/saas/test_zendesk_task.py index 156cb1a8b51..ab29e973c9a 100644 --- a/tests/ops/integration_tests/saas/test_zendesk_task.py +++ b/tests/ops/integration_tests/saas/test_zendesk_task.py @@ -4,7 +4,7 @@ from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner -@pytest.mark.integration_saas +@pytest.mark.integration_saas_test class TestZendeskConnector: def test_connection(self, zendesk_runner: ConnectorRunner): zendesk_runner.test_connection()