diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..3ce2cc82 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,41 @@ +# Version control +.git/ +.github/ + +# Frontend source — pre-built assets are already committed to web/ +web-src/ + +# Dev / build tooling +tools/build.py +tools/deploy_docker.sh +tools/init.py +tools/run_e2e_tests.sh +tools/start_in_virtualenv.sh +samples/ + +# Python cache +__pycache__/ +*.pyc +*.pyo +*.pyd +*.egg-info/ + +# Tests (not needed at runtime) +src/tests/ +src/e2e_tests/ + +# Docs +*.md +LICENSE + +# macOS +.DS_Store + +# IDE / local tooling +.vscode/ +.idea/ +.claude/ + +# Runtime directories (mounted as volumes) +conf/runners/ +logs/ diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..75100203 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,24 @@ +version: 2 +updates: + # Frontend (Vite/Vue) dependencies + - package-ecosystem: "npm" + directory: "/web-src" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 + groups: + dev-dependencies: + dependency-type: "development" + + # Python backend dependencies + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 + + # GitHub Actions workflow versions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..e6979e36 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,101 @@ +name: CI + +on: + push: + branches: [ master, stable ] + pull_request: + branches: [ master, stable ] + +jobs: + python-tests: + name: Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install ldap3 bcrypt parameterized pytest + + - name: Run unit tests + working-directory: src + env: + PYTHONPATH: . + run: pytest tests/ -q --tb=short + + frontend-tests: + name: Frontend (Node 22) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: "22" + cache: npm + cache-dependency-path: web-src/package-lock.json + + - name: Install dependencies + working-directory: web-src + run: npm ci + + - name: Run unit tests + working-directory: web-src + run: npm run test:unit-ci + + e2e-tests: + name: E2E (Playwright) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + cache: pip + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: "22" + cache: npm + cache-dependency-path: web-src/package-lock.json + + - name: Install frontend dependencies + working-directory: web-src + run: npm ci + + - name: Install Playwright browser + working-directory: web-src + run: npx playwright install --with-deps chromium + + - name: Run e2e tests + working-directory: web-src + # Builds the frontend, then starts the backend with the isolated + # e2e config (tests/e2e/server.sh creates its own venv) and runs + # the Playwright suite against it. + run: npm run test:e2e + + - name: Upload Playwright report on failure + uses: actions/upload-artifact@v4 + if: failure() + with: + name: playwright-report + path: web-src/playwright-report/ + retention-days: 7 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..b45cf6df --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,68 @@ +name: Docker + +on: + push: + branches: [ master, stable ] + tags: [ 'v*' ] + +jobs: + build-and-push: + name: Build & push (${{ matrix.platform }}) + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v6 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: "22" + cache: npm + cache-dependency-path: web-src/package-lock.json + + - name: Build frontend + working-directory: web-src + run: npm ci && npm run build + # Outputs to ../web/ (see build.outDir in web-src/vite.config.js) + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract image metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: ghcr.io/${{ github.repository }} + tags: | + # master branch → :latest + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }} + # stable branch → :stable + type=raw,value=stable,enable=${{ github.ref == 'refs/heads/stable' }} + # git tag v1.2.3 → :1.2.3 and :1.2 + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Build and push + uses: docker/build-push-action@v7 + with: + context: . + file: tools/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index f43d393d..5d7a6eea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# macOS +.DS_Store + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -63,6 +66,7 @@ target/ .ipynb_checkpoints conf/runners +conf/scripts conf/theme conf/conf.json conf/.htpasswd @@ -80,3 +84,9 @@ web-src/geckodriver.log venv/ /venv2/ e2e_venv/ + +# e2e (Playwright) +.e2e_venv/ +web-src/tests/e2e/.run/ +web-src/playwright-report/ +web-src/test-results/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index cfb1fcfd..00000000 --- a/.travis.yml +++ /dev/null @@ -1,85 +0,0 @@ -if: tag IS blank -sudo: required -dist: focal -language: node_js -node_js: - - '15' -cache: - directories: - - web-src/node_modules -addons: - chrome: stable - apt: - update: true - packages: - - libu2f-udev -env: - global: - - OWNER=${TRAVIS_REPO_SLUG%/*} - # GITHUB_TOKEN - - secure: GtWCiMPNz3MDDTvXqaVsgZBvlxYRZuTY6sUEhWTL37zJZHgRLlxTCTD27hHZp2P4B6G6KGV/0iEvMYxqzo6jMrzcEnt7TlGetteqI18dhV1dIQGH6uy8Y/PktkK2g2FuJeGV3FRK4+a21v6zzSuUaFa26k97mPapa4LS83XXj7rc13ll23HhhtObVF/a1n3U0Xwe4FkdoxbKlecJUnNujESxFk4xQl4N1tFv15fldDlFq0XWs26eEp3LU/n7wkMzbg9WEqPvDeaYlvir9VpvcYWpVvf8Uz+wpvW1jnE2R6bK1TDv5BNzQwbf2JtN284yZ8I1nwZtOJkc1RUr5wUFxCD0p8tc30sAlKemI385v2t1ccoBYMwH8LHFIUoSXoolHZBsnZeADqpo3a0d8hVFWv4AcxC27Q6SfMCltl0+ogAoQ7wVfkRT++044p415Ar4raEqIkqTm64FaRNMS0v2y5mS2634PMSGMRnK+NBrWz1yFKxsiuPKypIWygtrJ4pyiL2yPBZupnCluEgqva3q6AmMDlNuSUmcTEnCRGB01U2if6/oSEgISH1VK2lsRSxuoG5dFFuezwv90YENh/7pw0/hgge7EOse6OzDsU3uNRWcTuXF7eEBhjM1wBHWHlaAwV9vfHMZX4sHWP4R4CZhPjVEPRz6HPEg1tFCs2EvkuI= - # DOCKER_USER - - secure: 0AUrT+5xg+JGjAeZQfeCyVFHYw04YDSzCpeQILtt7Ca00rNJ4yshvCx+zEsyh0aMEO6+fLmGo4KCWC1tHKqKdO9ByIBBO1vbsQ83kDqC7GR1xorK8abN708NArxLdqylPRrrYK9Gr0VLk8t17DIfQdoP2QryJ2mDdsthzpliZOJ7c58LxIkBlog0uLabrX/d/m8ZEpgpqQalCUBmImc31tKBDprl5CVLk8ONLRVwdQ8WcYQaTNpOiIfx0OWp5iX+P9gMUyBTw8aFMlmwfZpXbDyGlwckDdIkKfWTUPq8FXNPLjyPbJ2zraID/kkNEw6J+x91w/F7VydhOoU/Gc4IAlL6TKN5xQquSclMz05kBHthSzZf7g6KUuQ0TgzK46zArcV4ZEItLU1h2IcLsPLi3+/O6TtUdUSQOIaCQX6YbQsnGDgEMWjtfpNJKpTyd+7SR+BuoQmtihr+Utl8rfq7vFTzkz+AiCvNnGQJzQoZKs83hgC57BRSD+LIkI0BrZ54ijNYoKCqvKHqmamkrXQEdiImBlTg8NpmBNHJgHdL6PRqi3NxLzJzdqKtz5pkI4MVVLYFXsUApD0AWvEOPejLnfVEGber5cA/Hm3HhqB7M9ja2BFml+oYBLc3mnzjKd/FT3VwWMiijfTJVvr6feFrhSQGiRyBLmaoRNmUSjUsfR0= - # DOCKER_PASSWORD - - secure: S22ffhnZOs1yFdBwJO9+uzy9DB3e7ehLWbj47U1zavsMKMDyDw0lkOIy8PMERFh4roBoM5dG95RIVbfNbrPXQxnY5Og2w7RTTv3eeHGdYzh+34Dppfk8mhEAhn3NL98sOe6is/5sEDvZ2ykPFLvoJmyV15V7Wvtuy1Zx+0lyZ0R0tX6sVJUWDlClHspCuSIKK+iptL5yLu4TtvX9Wks/c3kH6GIXYIJIeC63D+RRhuetbtGKND/RtFkq5IDP9qMZNXUAT/Mb8hrsk9HntFgl79dG2ChvBpDE8/LqjYDBiFTiUAtJfBhC0pVB3WaEwGTU/hWe8WTjl29JIkGcoaeT7+wncJ72lEPJoO60YWSdtWfTlNlUiN27AcxGqk39MDhB5NAbuJpKvcFLMmWFY2uJefrR6XVEXBZ+9yAwzuZmj0GYFOQTuczAqncyj/3BuOEqfIkkQ5BLAS5BUuzSEbHOjwasqbTVcWM1H3cv2ZYATXQQN8KhcZ5c5lxy8eD0NXHKvBFlS3HOXOXn3P6PqGgFHzjL+yyHMvzIXBJY4jEr8FIH16dwbXDqb4gi4lrrCZHDeIhVKsmLSUJjhmiKeP7dWcfUOGMxzLRmqA8r58TXcN3OvBrqNN63nUSG+Wb6XxmzLwE4PrlBy0fTRymG8WXrdE/Z2lglBhc8J3A8ER9c46s= - # AWS_ACCESS_KEY_ID - - secure: UUEboIaoKJD1vMGYrHzdr54QyWwOtY0XWGMrvCWwN76vHa2EOFl4OSO5MaSnNweoBfmw1HwVIeDuda1/cFVW2JPCSihVKdt9thK4dbXvXGkUOoUbHAPU9L6s2VXzFfuxTMhEew6sL8XY9K4RdVC+fgSGL03oQeJPSJgzNlAiOXEp8bNMELTEWQX7vIUaRv6vBLZ60UPkcq4SWfKb7uhLrKMGwmPVxSd7Jrp3JzXD7Tgj7WNE+KowpJqDgjuoXBhtFrqp0DCUj9HsPPSOQxuwFcQr/3u6TzpctYYv2qNor0h3ugmMJgiLNbn6VW+KbnLXGax1+YVdMGe+QAt/6yZjMbGioD1008bWeSpw2M6n7643yA9q8AeHUxMnP1VNsKit6z6YXskxsPpV9OIas/5KfmSGOhlrpeHpWonBWcdOlktVepOQ1bySxbDYH7bGx02cn+p/0P26NC+30OvejWNcUHq1n5hgT1fujxt2jGCcc0mHN+Pp4goN/9nwPExE1d55olLgFfA4Gd8gUc4bKowiBgczTYNF1Oms/8FzFWWgSwQBdPujWaEmtMkgw9bZKv8UEPeIeeyk7wZWDB61lYQ7X5X9kbE6CwTfTS4thpDlqmggRYOoHIIFKD9+QdgJcGSQCPYRnJ3Rc8XP2CbX2FcrjjDKB4VnBoK+/04Lmv1q6YM= - # AWS_SECRET_ACCESS_KEY - - secure: edY/+tpJOfRuLq6SN3pBnsKnaJux49niN+iDezEo+dsEFt0CchPQAOQ09iqIBfbF3WHrVy1jqJLwVQVbt0bZR6h2d1eGEysGF2saSEL4wMlziZfKP2aEMuVwVPgLJzt3AeoMib2tCAcXOfV1kF/lsbib3gCR0Amgb1GG3MUulAPmrwGxlwIQJuNcErMrWvl4QpTuK8/uOHBMqURCQLJLHrrYKTLzgrmbUs+7+ugqt35/hYdEZEV4lqs65Ty35eytA7zOOHR31x7k3gPx/MZyuuBHMXL8vrw1VQpDCgClp0TiT3FPJGgjf2BmkJADaAgIn+9DdxGuZrlYDorbLEvtSkGkilobL+m7WS3STqyzM7i1+MpYNMlJO+7KyXPC+RxLBsC8j4neo8SXnFZxWC83tVvMFnWXXnHb8i2wnW3E5PpB3I3ptmEsaWMb18uQa0j/G4wAKzjrxGZjWiu7goEIFKOfMPrFRj3F2uifJnTO3e/TnoKJhkAPxOkNhF6AACkhNZ9Z41Gkvav/upZxBoz7ojOoTH4Tn+KyVCMpa90wzQGvdaMuSw/+0Uf80piPO/GYriiF0pAI4yHfm9+SqtmjRtItgp7Nk3fXjYOY3tKL8jBlEUYMItMped6mYTW4SC/d2ib0xyxSNISyCZKRWKyV3rbAr6F15TVZUT+kFvBTtc0= - - PATH=$HOME/.local/bin:$PATH -before_install: - - sudo apt-get -y install python3-pip python3-setuptools apache2-utils python3-venv - - wget https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/116.0.5845.96/linux64/chromedriver-linux64.zip - - unzip chromedriver-linux64.zip -d $HOME/.local/bin -install: - - pip3 install -r requirements.txt - - pip3 install pyasn1 --upgrade - - pip3 install ldap3 parameterized bcrypt - - pip3 install requests --upgrade - - pip3 install awscli - - cd web-src - - npm install - - npm install allure-commandline --save-dev - - cd .. - - python3 -m venv e2e_venv - - e2e_venv/bin/pip install -r src/e2e_tests/requirements.txt -before_script: - - cd src - - python3 -m unittest discover -s tests -p "*.py" -t . - - cd ../web-src - - npm run test:unit-ci - - cd .. -script: - - python3 tools/build.py - - tools/run_e2e_tests.sh -after_script: - - tools/report_allure.sh -before_deploy: - - |- - if ! [ "$BEFORE_DEPLOY_RUN" ]; then export BEFORE_DEPLOY_RUN=1; - . tools/add_git_tag.sh - fi -deploy: - - provider: releases - name: dev - api_key: "$GITHUB_TOKEN" - file: build/script-server.zip - prerelease: true - overwrite: true - skip_cleanup: true - on: - branch: master - - provider: releases - name: "$(unzip -qc build/script-server.zip version.txt)" - api_key: "$GITHUB_TOKEN" - file: build/script-server.zip - skip_cleanup: true - on: - branch: stable - - provider: script - script: tools/deploy_docker.sh - skip_cleanup: true - on: - tags: false - all_branches: true - condition: "$TRAVIS_BRANCH =~ ^stable|master$" diff --git a/README.md b/README.md index 182390ca..e6d7f808 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,211 @@ -[](https://travis-ci.com/bugy/script-server) [](https://gitter.im/script-server/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +> **Fork** — This is a community-maintained fork of [bugy/script-server](https://github.com/bugy/script-server) (original author: [@bugy](https://github.com/bugy)). The upstream project is no longer actively maintained. + +[](https://github.com/knep/script-server/actions/workflows/ci.yml) + +## What's new in this fork + +### 2026-06-17 — Code quality and Python 3.14 compatibility + +**Vue 3 `emits` declarations:** 9 components were missing the `emits` option, causing Vue 3 to treat event listeners as DOM fallthrough attributes. This produced silent misbehaviour in the script-edit dialog (radio buttons and text inputs fired the wrong handler) and the schedule panel. All affected components now declare their emitted events explicitly. + +**Dead frontend dependencies removed:** `brace` and `codemirror` were listed in `package.json` but never imported in the source — only `ace-builds` is used. Both packages have been removed. + +**Python dependency bounds tightened:** `tornado>=6.1` and `requests>=2.28` now use the compatible-release operator (`~=`), allowing minor/patch updates while blocking breaking major changes. + +**DES-crypt htpasswd support on Python 3.13+:** the stdlib `crypt` module was removed in Python 3.13. DES-crypt password verification in the built-in htpasswd verifier now calls the system `crypt(3)` C function directly via ctypes (available as `libcrypt` on Linux, `libc` on macOS), with no new dependency. The note in the Python 3.12/3.13 section below still applies for Windows deployments. + +**`asyncio.set_event_loop_policy` replaced:** this function is deprecated since Python 3.14 and scheduled for removal in Python 3.16. The Windows Tornado workaround in `server.py` now uses `asyncio.set_event_loop(asyncio.SelectorEventLoop())` instead, which has the same effect without touching the global event-loop policy. + +**6 unit tests un-skipped:** 4 tests in `ExecutionInstanceTabs` were skipped because the old materialize tab component required layout measurements (`offsetWidth`) that jsdom cannot simulate. After the Vuetify migration the add-tab button is always rendered, so no layout computation is needed and the tests pass as-is. 2 tests in `test_auth_htpasswd` were skipped on Python ≥ 3.13 — now fixed via the ctypes crypt call above. + +### 2026-06-12 — State management migrated from Vuex to Pinia + +The frontend store layer has been fully migrated from Vuex 4 to [Pinia](https://pinia.vuejs.org/) +(the official Vue 3 store). Vuex has been removed from the dependency tree. + +**What changed:** + +- All Vuex modules (`src/*/store/`) replaced by Pinia stores (`src/*/stores/`). +- `defineStore` with Options API style (state / getters / actions) — no `context.commit`, + no `dispatch`, just `this.field` and direct action calls. +- Dynamic Vuex modules (the per-execution `scriptExecutor` module) replaced by a + `createExecutor()` factory that returns a `reactive()` object. +- Circular-import workarounds using runtime `require()` inside action bodies replaced by + straightforward top-level ES module imports (no actual circular imports existed). +- All 14 unit test files updated: `createVuexStore` → `createPinia` + `setActivePinia`, + store mutations replaced by direct property assignment on the Pinia store instance, + `vi.spyOn` used to stub actions with side-effects. +- `execution-details` date assertion made locale-independent. +- `ExecutionInstanceTabs` tests updated for Vuetify 4 DOM structure (buttons instead of + list items, `v-tab--selected` instead of `active`). + +### 2026-06-12 — UI migration to Vuetify 4 — complete + +The frontend has been fully migrated from the unmaintained materialize-css to +[Vuetify 4](https://vuetifyjs.com/). materialize-css has been removed from the +dependency tree; the build no longer bundles it. + +**What changed:** + +- **Foundation**: shared Vuetify instance (`src/common/vuetifyPlugin.js`) registered in + the main and admin apps, with a `scriptServer` theme mirroring the existing palette and + the `md` iconset reusing the Material Icons font already shipped (no new icon dependency). +- **Migrated components**: `checkbox` (`v-checkbox`), `textfield` (`v-text-field` / `v-combobox` + for `editable_list`), `TextArea` (`v-textarea` with auto-grow), `RadioGroup` + (`v-radio-group`), `Combobox` (`v-select` or `v-autocomplete` with type-to-filter when + the list has more than 10 options), `ChipsList` (`v-combobox` with chips, keeping the + CSV typing behaviour), `PromisableButton` (`v-btn` with built-in loading spinner; the + standalone `CircleSpinner` component is gone), `DatePicker` (`v-date-input`), `TimePicker` + (`v-text-field` with HH:MM validation), `server_file_field` (`v-text-field` + `v-dialog`). +- **Migrated views**: script list sidebar, script-view panel (schedule button, schedule + panel, parameter history modal), admin tabs, parameter list, script-edit dialog, login page. +- **materialize-css removed**: `src/common/materializecss/` shims, `src/assets/css/materializecss/` + overrides, `style_imports.js`, and the two Vite plugins that patched materialize internals + are all gone. The build and test setup no longer reference the package. +- **3 latent Vue 3 bugs fixed**: the script-edit dialog rendered as `[object Promise]` + (Vue 2 async-component syntax); `RadioGroup` used the Vue 2 v-model contract so switching + edit modes never reached the dialog; the schedule panel used the removed `$children` API + to collect field errors, so its Schedule button was never disabled on error. +- **One deliberate behaviour change**: reopening an autocomplete with a value already set + shows all options (Vuetify standard) instead of filtering on the current value; filtering + while typing is unchanged. +- Test setup gained jsdom stubs required by Vuetify overlays (`visualViewport`, + browser-accurate `offsetParent` for `
`/``). + +### 2026-05-28 — Frontend migrated to Vue 3 + Vite + Vitest + +The web frontend was upgraded from Vue 2 (Vue CLI + Webpack + Karma) to a modern toolchain: + +- **Vue 2 → Vue 3**: v-model refactor (`value`/`input` → `modelValue`/`update:modelValue`), + `emits` declarations, lifecycle hook renames (`beforeDestroy`/`destroyed` → + `beforeUnmount`/`unmounted`), removal of `Vue.set`/`Vue.delete`/`$set`/`$delete` (Vue 3 + Proxy reactivity), `:deep()` CSS selectors, and Vue Router 4 / Vuex 4. +- **Vue CLI + Webpack → Vite**: faster builds; assets are emitted into a single hashed + `web/assets/` folder (the server's web-build check was updated accordingly). +- **Karma + Mocha → Vitest** (jsdom): the unit suite runs headless without a browser + (`npm run test:unit-ci`). A few materialize-CSS browser-only behaviours (dropdown/modal + animations, layout measurements) are skipped under jsdom. +- Removed the obsolete `vue.config.js`, `babel.config.js`, and Karma entry point. + +Dev commands are unchanged: `npm run serve` (dev server) and `npm run build` (production build). + +### 2026-05-28 — docker-compose.yml for easy deployment + +A `docker-compose.yml` is now included at the root of the repository for quick local deployments: + +```bash +# Start +docker compose up -d + +# Stop +docker compose down +``` + +Mount your script configs in `./conf/runners/` and logs will be written to `./logs/`. +See the full [docker-compose instructions](#with-docker-compose) below. + +### 2026-05-28 — Frontend unit tests for `date` and `time` components + +Unit tests added for the `DateField` and `TimeField` Vue components (42 tests total), covering label/input rendering, two-way value binding, and required-field validation. + +### 2026-05-28 — Docker image on GitHub Container Registry + +A Docker image for this fork is now published automatically on every commit to `master`: + +```bash +docker run -d \ + -p 5000:5000 \ + -v /path/to/your/conf/runners:/app/conf/runners \ + -v /path/to/your/logs:/app/logs \ + ghcr.io/knep/script-server:latest +``` + +Available tags: `latest` (master), `stable`, and semver tags (e.g. `1.19.0`) on git releases. +See the full [installation instructions](#as-a-docker-container) below. + +### 2026-05-28 — New `time` parameter type + +A new `time` parameter type shows a native time picker in the UI and passes the selected time to the script in a configurable format. + +**Configuration example:** +```json +{ + "name": "start_time", + "type": "time", + "time_format": "%H:%M" +} +``` + +- `time_format` is a Python [strftime](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) format string. Default: `%H:%M` (24-hour HH:MM). +- The UI always shows a native time picker. The script receives the time in the configured format. +- An invalid `time_format` (e.g. `"HH:MM"` instead of `"%H:%M"`) is now detected at startup with a clear error message. + +### 2026-05-28 — HTTP security headers + +All responses (including WebSocket upgrade responses) now include the following security headers: + +| Header | Value | Condition | +|--------|-------|-----------| +| `X-Content-Type-Options` | `nosniff` | Always | +| `Referrer-Policy` | `strict-origin-when-cross-origin` | Always | +| `Content-Security-Policy` | `default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; connect-src 'self' ws: wss:; frame-ancestors 'none'; object-src 'none'` | Always | +| `Permissions-Policy` | `camera=(), microphone=(), geolocation=()` | Always | +| `Strict-Transport-Security` | `max-age=31536000; includeSubDomains` | HTTPS only (`cookie_secure: true`) | + +`X-Frame-Options: DENY` was already present; `frame-ancestors 'none'` in the CSP provides equivalent coverage for modern browsers. + +### 2025-05-27 — New `date` parameter type + +A new `date` parameter type shows a native date picker in the UI and passes the selected date to the script in a configurable format. + +**Configuration example:** +```json +{ + "name": "start_date", + "type": "date", + "date_format": "%d/%m/%Y" +} +``` + +- `date_format` is a Python [strftime](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) format string. Default: `%Y-%m-%d` (ISO 8601). +- The UI always shows a calendar date picker. The script receives the date in the configured format. +- An invalid `date_format` (e.g. `"DD/MM/YYYY"` instead of `"%d/%m/%Y"`) is now detected at startup with a clear error message. + +### 2025-05-27 — GitHub Actions CI + secure cookies + +- GitHub Actions CI added ([view workflows](https://github.com/knep/script-server/actions)): Python 3.10/3.11/3.12/3.13 matrix + Node 22 frontend tests on every push and pull request. +- Cookies (`username`, `token`, XSRF) are now set with `HttpOnly`, `SameSite=Lax`, and `Secure` flags. The `Secure` flag can be disabled in `conf.json` via `"cookie_secure": false` for HTTP-only deployments. + +### 2025-05-27 — Python 3.12/3.13 compatibility + +**Python version support:** updated minimum from Python 3.7 (end-of-life since June 2023) to **Python 3.9+** (Python 3.13 recommended). + +**Python 3.13 note:** the `crypt` standard-library module was removed in Python 3.13. If your `htpasswd` file contains DES-crypt passwords (entries that do not start with `$2y$`, `$apr1$`, or `{SHA}`), the server will refuse to start with a clear error message. Regenerate those passwords using bcrypt (`htpasswd -B`) or SHA-1 (`htpasswd -s`). + +**Fixes:** +- Replaced invalid string escape sequences (`\d`, `\w`, `\/`, `\ `, `\|`, `\p`, `\[`, `\.`) with raw strings (`r'...'`) in test files — these would become `SyntaxError` in Python 3.14 +- Replaced deprecated `thread.setDaemon(True)` with `thread.daemon = True` in `user_file_storage.py` and `auth_abstract_oauth.py` + +**Dependencies (`requirements.txt`):** +- Raised Tornado floor from `>=4` to `>=6.1` (Tornado 4/5 are incompatible with Python 3.12) +- Added `requests>=2.28` as an explicit dependency (used by HTTP notification destinations) +- Documented optional dependencies (`ldap3`, `bcrypt`) with install instructions + +--- # script-server -Script-server is a Web UI for scripts. + +Script-server is a Web UI for scripts. As an administrator, you add your existing scripts into Script server and other users would be able to execute them via a web interface. The UI is very straightforward and can be used by non-tech people. -No script modifications are needed - you configure each script in Script server and it creates the corresponding UI with parameters and takes care of validation, execution, etc. - -[DEMO server](https://script-server.net/) +No script modifications are needed - you configure each script in Script server and it creates the corresponding UI with parameters and takes care of validation, execution, etc. [Admin interface screenshots](https://github.com/bugy/script-server/wiki/Admin-interface) ## Features -- Different types of script parameters (text, flag, dropdown, file upload, etc.) +- Different types of script parameters (text, integer, date, time, flag, dropdown, file upload, etc.) - Real-time script output - Users can send input during script execution - Auth (optional): LDAP, Google OAuth, htpasswd file @@ -32,15 +224,31 @@ or [how to configure the server](https://github.com/bugy/script-server/wiki/Serv ### Server-side -Python 3.7 or higher with the following modules: +Python 3.9 or higher (Python 3.12 recommended) with the following modules: -* Tornado 5 / 6 +* tornado >= 6.1 +* requests >= 2.28 -Some features can require additional modules. Such requirements are specified in a corresponding feature description. +Optional modules (required only for specific features): + +| Module | Feature | +|--------|---------| +| `ldap3 >= 2.9` | LDAP authentication | +| `bcrypt >= 4.0` | bcrypt password support in htpasswd auth | + +Install all dependencies at once: +```bash +pip install -r requirements.txt +``` + +Install with optional dependencies: +```bash +pip install -r requirements.txt ldap3>=2.9 bcrypt>=4.0 +``` OS support: -- Linux (main). Tested and working on Debian 10,11 +- Linux (main). Tested and working on Debian 10, 11 - Windows (additional). Light testing - macOS (additional). Light testing @@ -52,104 +260,161 @@ Internet connection is **not** needed. All the files are loaded from the server. ## Installation ### For production -1. Download script-server.zip file from [Latest release](https://github.com/bugy/script-server/releases/latest) or [Dev release](https://github.com/bugy/script-server/releases/tag/dev) -2. Create script-server folder anywhere on your PC and extract zip content to this folder +1. Download script-server.zip from [Latest release](https://github.com/bugy/script-server/releases/latest) (last upstream release: v1.18.0) +2. Create a script-server folder anywhere on your machine and extract the zip content into it +3. Install dependencies: `pip install -r requirements.txt` + +For detailed steps on Linux with virtualenv, see the [Installation guide](https://github.com/bugy/script-server/wiki/Installing-on-virtualenv-(linux)). -(For detailed steps on linux with virtualenv, please see [Installation guide](https://github.com/bugy/script-server/wiki/Installing-on-virtualenv-(linux))) +##### As a Docker container -##### As a docker container -Please find pre-built images here: https://hub.docker.com/r/bugy/script-server/tags -For the usage please check [this ticket](https://github.com/bugy/script-server/issues/171#issuecomment-461620836) +Images for this fork are published on [GitHub Container Registry](https://github.com/knep/script-server/pkgs/container/script-server): + +```bash +# Pull the latest image (built from master) +docker pull ghcr.io/knep/script-server:latest + +# Run with your script configs and logs persisted +docker run -d \ + -p 5000:5000 \ + -v /path/to/your/conf/runners:/app/conf/runners \ + -v /path/to/your/logs:/app/logs \ + ghcr.io/knep/script-server:latest +``` + +Available tags: +| Tag | Source | +|-----|--------| +| `latest` | `master` branch — most recent changes | +| `stable` | `stable` branch | +| `1.2.3` / `1.2` | Git tag `v1.2.3` | + +##### With docker-compose + +A ready-to-use `docker-compose.yml` is included at the root of the repository: + +```bash +# Clone or download docker-compose.yml, then: +docker compose up -d +``` + +Place your script runner configs in `./conf/runners/` (created automatically on first run). +Execution logs are written to `./logs/`. + +To customise the server (auth, SSL, port…), uncomment the optional volume lines in `docker-compose.yml`: + +```yaml +volumes: + - ./conf/runners:/app/conf/runners + - ./logs:/app/logs + # - ./conf/conf.json:/app/conf/conf.json:ro # server config + # - ./conf/.htpasswd:/app/conf/.htpasswd:ro # htpasswd auth + # - ./conf/theme:/app/conf/theme:ro # custom CSS/images +``` ### For development -1. Clone/download the repository -2. Run 'tools/init.py --no-npm' script +1. Clone this repository +2. Run `tools/init.py --no-npm` -`init.py` script should be run after pulling any new changes +`init.py` should be run after pulling any new changes. -If you are making changes to web files, use `npm run build` or `npm run serve` +**Frontend** (`web-src/`, Vue 3 + Vite). If you are making changes to web files: +- `npm install` once to install dependencies +- `npm run serve` — Vite dev server (proxies API calls to the backend on port 5000) +- `npm run build` — production build into `web/` +- `npm run test:unit-ci` — run the Vitest unit suite (headless, jsdom) -### A issue running on OpenBSD and maybe other UNIX systems -See [A issue running on OpenBSD and maybe other UNIX systems](https://github.com/bugy/script-server/wiki/OpenBSD-process-termination-issues). +### A note on OpenBSD and some other UNIX systems +See [OpenBSD process termination issues](https://github.com/bugy/script-server/wiki/OpenBSD-process-termination-issues). ## Setup and run -1. Create configurations for your scripts in *conf/runners/* folder (see [script config page](https://github.com/bugy/script-server/wiki/Script-config) for details) -2. Launch launcher.py from script-server folder - * Windows command: launcher.py - * Linux command: ./launcher.py +1. Create configurations for your scripts in the *conf/runners/* folder (see [script config page](https://github.com/bugy/script-server/wiki/Script-config) for details) +2. Launch launcher.py from the script-server folder + * Windows: `launcher.py` + * Linux/macOS: `./launcher.py` 3. Add/edit scripts on the admin page By default, the server will run on http://localhost:5000 ### Server config -All the features listed above and some other minor features can be configured in *conf/conf.json* file. -It is allowed not to create this file. In this case, default values will be used. -See [server config page](https://github.com/bugy/script-server/wiki/Server-configuration) for details +All the features listed above and some other minor features can be configured in *conf/conf.json*. +It is allowed not to create this file — default values will be used in that case. +See [server config page](https://github.com/bugy/script-server/wiki/Server-configuration) for details. + +#### Running over plain HTTP (e.g. local dev) + +Cookies (including the XSRF token) are sent with the `Secure` flag by default, so +they are **not stored by the browser over plain HTTP**. Combined with the default +`token` XSRF protection — which requires the browser to read the `_xsrf` cookie and +echo it back — this makes every `POST` (e.g. *starting an execution*) fail with +`403` and an *"XSRF token missing or invalid"* message. + +When serving over HTTP (not HTTPS), set `cookie_secure` to `false` in `conf/conf.json`: + +```json +{ + "security": { + "cookie_secure": false + } +} +``` + +Alternatively, switch XSRF protection to header mode (`"xsrf_protection": "header"`), +which validates the `X-Requested-With` header instead of a cookie token and so does +not depend on cookies at all. Keep the secure defaults for any HTTPS deployment. + +> Tip: if you previously ran with the secure defaults, clear the stale `_xsrf` cookie +> for the site (or use a fresh browser profile) after changing these settings. ### Admin panel -Admin panel is accessible on admin.html page (e.g. http://localhost:5000/admin.html) +Admin panel is accessible at admin.html (e.g. http://localhost:5000/admin.html) ## Logging -All web/operating logs are written to the *logs/server.log* -Additionally each script logs are written to separate file in *logs/processes*. File name format is -{script\_name}\_{client\_address}\_{date}\_{time}.log. +All web/operating logs are written to *logs/server.log*. +Each script's logs are written to a separate file in *logs/processes*. File name format: +`{script_name}_{client_address}_{date}_{time}.log` ## Testing/demo -Script-server has bundled configs/scripts for testing/demo purposes, which are located in samples folder. You can -link/copy these config files (samples/configs/\*.json) to server config folder (conf/runners). +Script-server has bundled configs/scripts for testing/demo purposes, located in the samples folder. You can +link/copy these config files (`samples/configs/*.json`) to the server config folder (`conf/runners`). ## Security -I do my best to make script-server secure and invulnerable to attacks, injections or user data security. However to be -on the safe side, it's better to run Script server only on a trusted network. -Any security leaks report or recommendations are greatly appreciated! +Script-server is designed to be secure and invulnerable to attacks, injections or user data leaks. However, to be +on the safe side, it's better to run Script server only on a trusted network. ### Shell commands injection Script server guarantees that all user parameters are passed to an executable script as arguments and won't be executed -under any conditions. There is no way to inject fraud command from a client-side. However, user parameters are not -escaped, so scripts should take care of not executing them also (general recommendation for bash is at least to wrap all -arguments in double-quotes). It's recommended to use typed parameters when appropriate, because they are validated for -proper values and so they are harder to be subject of commands injection. Such attempts would be easier to detect also. +under any conditions. There is no way to inject a fraudulent command from the client side. However, user parameters are not +escaped, so scripts should take care of not executing them directly (the general recommendation for bash is to wrap all +arguments in double-quotes). Using typed parameters is recommended when appropriate, as they are validated for +proper values and are harder to exploit. -_Important!_ Command injection protection is fully supported for Linux, but _only_ for .bat and .exe files on Windows +_Important!_ Command injection protection is fully supported for Linux, but _only_ for .bat and .exe files on Windows. ### XSS and CSRF -_(v1.0 - v1.16)_ +_(v1.0 - v1.16)_ Script server _is_ vulnerable to these attacks. -_(v1.17+)_ -Script server is protected against XSRF attacks via a special token. +_(v1.17+)_ +Script server is protected against XSRF attacks via a special token. XSS protection: the code is written according to [OWASP Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/DOM_based_XSS_Prevention_Cheat_Sheet.html) and the only **known** vulnerabilities are: * `output_format`=`html_iframe`, see the reasoning in the - linked [Wiki page]((https://github.com/bugy/script-server/wiki/Script-config#output_format)) + linked [Wiki page](https://github.com/bugy/script-server/wiki/Script-config#output_format) ## Contribution -If you like the project and think you could help with making it better, there are many ways you can do it: - -- Create a new issue for new feature proposal or a bug -- Implement existing issues (there are quite some of them: frontend/backend, simple/complex, choose whatever you like) -- Help with improving the documentation -- Set up a demo server -- Spread a word about the project to your colleagues, friends, blogs or any other channels -- Any other things you could imagine - -Any contribution would be of great help and I will highly appreciate it! -If you have any questions, please create a new issue, or contact me via buggygm@gmail.com - -## Asking questions -If you have any questions, feel free to: -- Ask in gitter: https://gitter.im/script-server/community -- or [create a ticket](https://github.com/bugy/script-server/issues/new) -- or contact me via email: buggygm@gmail.com (for some non-shareable questions) +If you find a bug or want to propose a feature, please [open an issue](https://github.com/knep/script-server/issues) on this fork. -## Special thanks - +Contributions are welcome: +- Bug reports and feature proposals +- Pull requests (fixes, features, documentation) +- Any other improvements you can think of diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..9e08372a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,18 @@ +services: + script-server: + image: ghcr.io/knep/script-server:latest + container_name: script-server + restart: unless-stopped + ports: + - "5000:5000" + volumes: + # Script configurations (required) — put your .json runner files here + - ./conf/runners:/app/conf/runners + # Execution logs + - ./logs:/app/logs + # Optional: server configuration (auth, SSL, port, etc.) + # - ./conf/conf.json:/app/conf/conf.json:ro + # Optional: htpasswd file for basic auth + # - ./conf/.htpasswd:/app/conf/.htpasswd:ro + # Optional: custom theme (CSS/images) + # - ./conf/theme:/app/conf/theme:ro diff --git a/requirements.txt b/requirements.txt index 6ddf2bc6..6e40e042 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,15 @@ -tornado>=4 +# Python 3.9+ required (3.12+ recommended) + +# Core +tornado~=6.1 + +# HTTP notification destination (communications/destination_http.py) +requests~=2.28 + +# Optional: LDAP authentication backend +# Uncomment or install separately: pip install 'ldap3~=2.9' +# ldap3~=2.9 + +# Optional: bcrypt password verification for htpasswd auth +# Uncomment or install separately: pip install 'bcrypt~=4.0' +# bcrypt~=4.0 diff --git a/src/auth/auth_abstract_oauth.py b/src/auth/auth_abstract_oauth.py index c137f8eb..da133e9a 100644 --- a/src/auth/auth_abstract_oauth.py +++ b/src/auth/auth_abstract_oauth.py @@ -54,7 +54,7 @@ def __repr__(self) -> str: def _start_timer(callback): timer = threading.Timer(30, callback) - timer.setDaemon(True) + timer.daemon = True timer.start() return timer diff --git a/src/auth/auth_authentik_openid.py b/src/auth/auth_authentik_openid.py new file mode 100644 index 00000000..75256293 --- /dev/null +++ b/src/auth/auth_authentik_openid.py @@ -0,0 +1,52 @@ +import logging + +from tornado import escape + +from auth.auth_abstract_oauth import AbstractOauthAuthenticator, _OauthUserInfo +from model import model_helper + +LOGGER = logging.getLogger('script_server.GoogleOauthAuthorizer') + + +# noinspection PyProtectedMember +class AuthentikOpenidAuthenticator(AbstractOauthAuthenticator): + def __init__(self, params_dict): + authenitk_url = model_helper.read_obligatory( + params_dict, + 'authenitk_url', + ': should contain openid url, e.g. http://localhost:9001/') + if not authenitk_url.endswith('/'): + authenitk_url = authenitk_url + '/' + self._authenitk_url = authenitk_url + + super().__init__(authenitk_url + 'application/o/authorize/', + authenitk_url + 'application/o/token/', + 'email openid profile', + params_dict) + + async def fetch_user_info(self, access_token) -> _OauthUserInfo: + user_future = self.http_client.fetch( + self._authenitk_url + 'application/o/userinfo/', + headers={'Authorization': 'Bearer ' + access_token}) + + user_response = await user_future + + if not user_response: + raise Exception('No response during loading userinfo') + + response_values = {} + if user_response.body: + response_values = escape.json_decode(user_response.body) + + eager_groups = None + if self.group_support: + eager_groups = response_values.get('groups') + if eager_groups is None: + eager_groups = [] + LOGGER.warning('Failed to load user groups. Most probably groups mapping is not enabled. ' + 'Check the corresponding wiki section') + + return _OauthUserInfo(response_values.get('preferred_username'), True, response_values, eager_groups) + + async def fetch_user_groups(self, access_token): + raise Exception('This shouldn\'t be used, all the groups should be fetched with user info.') diff --git a/src/auth/auth_azure_ad_oauth.py b/src/auth/auth_azure_ad_oauth.py new file mode 100644 index 00000000..5b1a940f --- /dev/null +++ b/src/auth/auth_azure_ad_oauth.py @@ -0,0 +1,34 @@ +import logging + +import tornado.auth + +from auth.auth_abstract_oauth import AbstractOauthAuthenticator, _OauthUserInfo +from model import model_helper + +LOGGER = logging.getLogger('script_server.AzureADOauthAuthenticator') + + +class AzureAdOAuthAuthenticator(AbstractOauthAuthenticator): + def __init__(self, params_dict): + params_dict['group_support'] = False + self.auth_url = model_helper.read_obligatory(params_dict, 'auth_url', ' for OAuth') + self.token_url = model_helper.read_obligatory(params_dict, 'token_url', ' for OAuth') + + super().__init__( + self.auth_url, + self.token_url, + 'openid email profile User.Read', + params_dict, + ) + + async def fetch_user_info(self, access_token) -> _OauthUserInfo: + headers = {'Authorization': f'Bearer {access_token}'} + user_response = await self.http_client.fetch('https://graph.microsoft.com/v1.0/me', headers=headers) + if not user_response: + return None + + user_data = tornado.escape.json_decode(user_response.body) + return _OauthUserInfo(user_data.get('userPrincipalName'), True, user_data) + + async def fetch_user_groups(self, access_token): + return [] diff --git a/src/auth/auth_htpasswd.py b/src/auth/auth_htpasswd.py index ffaa26e5..ccf56164 100644 --- a/src/auth/auth_htpasswd.py +++ b/src/auth/auth_htpasswd.py @@ -1,3 +1,5 @@ +import ctypes +import ctypes.util import logging import os @@ -10,6 +12,19 @@ LOGGER = logging.getLogger('script_server.HtpasswdAuthenticator') +def _crypt_des(password: str, salt: str): + """DES-crypt via ctypes, replacing the stdlib crypt module removed in Python 3.13.""" + lib_name = ctypes.util.find_library('crypt') or ctypes.util.find_library('c') + if not lib_name: + return None + try: + lib = ctypes.CDLL(lib_name) + lib.crypt.restype = ctypes.c_char_p + result = lib.crypt(password.encode(), salt.encode()) + return result.decode() if result else None + except (OSError, AttributeError): + return None + def _select_verifier(htpasswd_path, process_invoker: ProcessInvoker): if _HtpasswdVerifier.is_installed(process_invoker): @@ -124,8 +139,11 @@ def verify(self, username, password): return hashed_password == expected elif not os_utils.is_win(): - import crypt - hashed_password = crypt.crypt(password, existing_password[:2]) + hashed_password = _crypt_des(password, existing_password[:2]) + if hashed_password is None: + raise InvalidServerConfigException( + 'htpasswd contains DES-crypt passwords but the system crypt library is unavailable. ' + 'Please regenerate passwords using bcrypt (htpasswd -B) or SHA-1 (htpasswd -s).') return hashed_password == existing_password else: diff --git a/src/auth/auth_ldap.py b/src/auth/auth_ldap.py index a0d42731..e5fea120 100644 --- a/src/auth/auth_ldap.py +++ b/src/auth/auth_ldap.py @@ -3,7 +3,7 @@ import os from string import Template -from ldap3 import Connection, SIMPLE +from ldap3 import Connection, SIMPLE, Server from ldap3.core.exceptions import LDAPAttributeError from ldap3.utils.conv import escape_filter_chars @@ -39,7 +39,7 @@ def _resolve_base_dn(full_username): return '' -def _search(dn, search_request, attributes, connection): +def _ldap_search(dn, search_request, attributes, connection): search_string = search_request.as_search_string() success = connection.search(dn, search_string, attributes=attributes) @@ -53,7 +53,7 @@ def _search(dn, search_request, attributes, connection): def _load_multiple_entries_values(dn, search_request, attribute_name, connection): - entries = _search(dn, search_request, [attribute_name], connection) + entries = _ldap_search(dn, search_request, [attribute_name], connection) if entries is None: return [] @@ -77,32 +77,25 @@ class LdapAuthenticator(auth_base.Authenticator): def __init__(self, params_dict, temp_folder): super().__init__() - self.url = model_helper.read_obligatory(params_dict, 'url', ' for LDAP auth') + self._ldap_connector = LdapConnector( + model_helper.read_obligatory(params_dict, 'url', ' for LDAP auth'), + params_dict.get('version') + ) - username_pattern = strip(params_dict.get('username_pattern')) - if username_pattern: - self.username_template = Template(username_pattern) - else: - self.username_template = None + self._ldap_user_resolver = LdapUserResolver( + params_dict.get('ldap_user_resolver'), + self._ldap_connector) base_dn = params_dict.get('base_dn') if base_dn: self._base_dn = base_dn.strip() else: - resolved_base_dn = _resolve_base_dn(username_pattern) - - if resolved_base_dn: - LOGGER.info('Resolved base dn: ' + resolved_base_dn) - self._base_dn = resolved_base_dn - else: + self._base_dn = self._ldap_user_resolver.auto_resolve_base_dn() + if not self._base_dn: LOGGER.warning( 'Cannot resolve LDAP base dn, so using empty. Please specify it using "base_dn" attribute') self._base_dn = '' - self.version = params_dict.get("version") - if not self.version: - self.version = 3 - self._groups_file = os.path.join(temp_folder, 'ldap_groups.json') self._user_groups = self._load_groups(self._groups_file) @@ -119,13 +112,10 @@ def perform_basic_auth(self, user, password): def _authenticate_internal(self, username, password): LOGGER.info('Logging in user ' + username) - if self.username_template: - full_username = self.username_template.substitute(username=username) - else: - full_username = username + full_username = self._ldap_user_resolver.resolve_ldap_username(username, self._base_dn) try: - connection = self._connect(full_username, password) + connection = self._ldap_connector.connect(full_username, password) if connection.bound: try: @@ -155,18 +145,6 @@ def _authenticate_internal(self, username, password): raise auth_base.AuthFailureError(error) - def _connect(self, full_username, password): - connection = Connection( - self.url, - user=full_username, - password=password, - authentication=SIMPLE, - read_only=True, - version=self.version - ) - connection.bind() - return connection - def _get_groups(self, user): groups = self._user_groups.get(user) if groups is not None: @@ -213,15 +191,8 @@ def _get_user_ids(self, full_username, connection): LOGGER.warning('Unsupported username pattern for ' + full_username) return full_username, None - entries = _search(base_dn, search_request, ['uid'], connection) - if not entries: - return full_username, None + entry = LdapConnector.find_user(base_dn, search_request, connection) - if len(entries) > 1: - LOGGER.warning('More than one user found by filter: ' + str(search_request)) - return full_username, None - - entry = entries[0] return get_entry_dn(entry), entry.uid.value def _load_groups(self, groups_file): @@ -248,3 +219,123 @@ def as_search_string(self): def __str__(self) -> str: return self.as_search_string() + + +class LdapConnector: + def __init__(self, url, version): + self.url = url + self.version = version + if not self.version: + self.version = 3 + + def connect(self, full_username, password): + server = Server(self.url, connect_timeout=10) + connection = Connection( + server, + user=full_username, + password=password, + authentication=SIMPLE, + read_only=True, + version=self.version, + ) + connection.bind() + return connection + + @staticmethod + def find_user(base_dn, search_request, connection, attributes=None): + if attributes is None: + attributes = ['uid'] + + entries = _ldap_search(base_dn, search_request, attributes, connection) + if not entries: + return None + + if len(entries) > 1: + LOGGER.warning('More than one user found by filter: ' + str(search_request)) + return None + + return entries[0] + + +class LdapUserResolver: + def __init__(self, config, ldap_connector: LdapConnector) -> None: + self.username_template = None + self.username_pattern = None + self.search_by_attribute = None + self.admin_user = None + self.admin_password = None + self.ldap_connector = ldap_connector + + if config: + username_pattern = strip(config.get('username_pattern')) + search_by_attribute = strip(config.get('search_by_attribute')) + + # Validate that either username_pattern or search_by_attribute is specified + if not username_pattern and not search_by_attribute: + raise ValueError( + 'Either username_pattern or search_by_attribute must be specified in ldap_user_resolver.') + + if username_pattern and search_by_attribute: + raise ValueError( + 'Cannot specify both username_pattern and search_by_attribute in ldap_user_resolver. Choose one method.') + + if username_pattern: + self.username_template = Template(username_pattern) + self.username_pattern = username_pattern + + if search_by_attribute: + self.search_by_attribute = search_by_attribute + self.admin_user = model_helper.read_obligatory( + config, + 'admin_user', + ' for ldap_user_resolver with search_by_attribute' + ) + self.admin_password = model_helper.read_obligatory( + config, + 'admin_password', + ' for ldap_user_resolver with search_by_attribute' + ) + + def resolve_ldap_username(self, username, base_dn): + if self.username_template: + return self.username_template.substitute(username=username) + elif self.search_by_attribute: + resolved_dn = self._find_user_dn_by_attribute(username, base_dn) + return resolved_dn + else: + return username + + def auto_resolve_base_dn(self): + if self.username_pattern: + resolved_base_dn = _resolve_base_dn(self.username_pattern) + if resolved_base_dn: + LOGGER.info('Resolved base dn: ' + resolved_base_dn) + return resolved_base_dn + + if self.search_by_attribute: + resolved_base_dn = _resolve_base_dn(self.admin_user) + if not resolved_base_dn: + raise Exception('"base_dn" is required for search_by_attribute user resolution') + return resolved_base_dn + + return None + + def _find_user_dn_by_attribute(self, username, base_dn): + admin_connection = self.ldap_connector.connect(self.admin_user, self.admin_password) + + try: + if not admin_connection.bound: + error_msg = f'Failed to bind with admin LDAP user: {admin_connection.last_error}' + LOGGER.error(error_msg) + raise auth_base.AuthFailureError(error_msg) + + search_request = SearchRequest(f'({self.search_by_attribute}=%s)', username) + + user = self.ldap_connector.find_user(base_dn, search_request, admin_connection) + if user is None: + raise auth_base.AuthRejectedError('Invalid credentials') + + return get_entry_dn(user) + + finally: + admin_connection.unbind() diff --git a/src/auth/authorization.py b/src/auth/authorization.py index 5d1ab415..ee9bff3a 100644 --- a/src/auth/authorization.py +++ b/src/auth/authorization.py @@ -10,6 +10,13 @@ def _normalize_user(user): return user.lower().strip() return user +def _matches_email_domain_pattern(user, pattern): + if not user or not pattern or not (isinstance(pattern, str) and pattern.startswith('*@')): + return False + + domain = pattern[1:] # remove the '*' character + return user.endswith(domain) + def _normalize_users(allowed_users): if isinstance(allowed_users, list): @@ -54,9 +61,16 @@ def _is_allowed_internal(self, user_id, normalized_allowed_users): if normalized_allowed_users == ANY_USER: return True - if _normalize_user(user_id) in normalized_allowed_users: + normalized_user = _normalize_user(user_id) + + if normalized_user in normalized_allowed_users: return True + # Check for domain patterns (e.g., "*@mydomain.com") + for pattern in normalized_allowed_users: + if _matches_email_domain_pattern(normalized_user, pattern): + return True + user_groups = self._groups_provider.get_groups(user_id) if not user_groups: return False diff --git a/src/auth/identification.py b/src/auth/identification.py index ab265525..5b3e2851 100644 --- a/src/auth/identification.py +++ b/src/auth/identification.py @@ -116,7 +116,8 @@ def _read_client_token(self, request_handler): def _write_client_token(self, client_id, request_handler): expiry_time = date_utils.get_current_millis() + days_to_ms(self.EXPIRES_DAYS) new_token = client_id + '&' + str(expiry_time) - request_handler.set_secure_cookie(self.COOKIE_KEY, new_token, expires_days=self.EXPIRES_DAYS) + server_config = request_handler.application.server_config + request_handler.set_secure_cookie(self.COOKIE_KEY, new_token, expires_days=self.EXPIRES_DAYS, secure=server_config.cookie_secure, httponly=True) def _can_write(self, request_handler): return can_write_secure_cookie(request_handler) diff --git a/src/auth/oauth_token_manager.py b/src/auth/oauth_token_manager.py index cc937292..313744d4 100644 --- a/src/auth/oauth_token_manager.py +++ b/src/auth/oauth_token_manager.py @@ -1,5 +1,7 @@ +import asyncio import datetime import logging +from collections import defaultdict from typing import Dict, Optional import tornado.ioloop @@ -15,6 +17,7 @@ class OAuthTokenManager: def __init__(self, enabled, fetch_token_callback) -> None: self._refresh_tokens = {} # type: Dict[str, str] self._pending_access_tokens = {} # type: Dict[str, OAuthTokenResponse] + self._refresh_locks = defaultdict(asyncio.Lock) # type: Dict[str, asyncio.Lock] self._scheduler = None self._enabled = enabled @@ -24,7 +27,8 @@ def update_tokens(self, token_response: OAuthTokenResponse, username, request_ha if not self._enabled: return - request_handler.set_secure_cookie('token', token_response.access_token) + server_config = request_handler.application.server_config + request_handler.set_secure_cookie('token', token_response.access_token, httponly=True, secure=server_config.cookie_secure) if token_response.should_refresh(): refresh_token = token_response.refresh_token @@ -33,7 +37,7 @@ def update_tokens(self, token_response: OAuthTokenResponse, username, request_ha self._refresh_tokens[username] = refresh_token self._schedule_token_refresh(username, refresh_token, token_response.resolve_next_refresh_datetime()) - request_handler.set_secure_cookie('token_details', token_response.serialize_details()) + request_handler.set_secure_cookie('token_details', token_response.serialize_details(), httponly=True, secure=server_config.cookie_secure) def can_restore_state(self, request_handler): if not self._enabled: @@ -109,7 +113,7 @@ def remove_user(self, username): def _schedule_token_refresh(self, username, refresh_token, next_refresh_datetime): if not self._scheduler: - self.scheduler = Scheduler() + self._scheduler = Scheduler() token_expires_in = next_refresh_datetime - datetime.datetime.now() if token_expires_in < datetime.timedelta(seconds=30): @@ -119,31 +123,34 @@ def _schedule_token_refresh(self, username, refresh_token, next_refresh_datetime else: next_refresh_datetime_adjusted = next_refresh_datetime - datetime.timedelta(minutes=1) - self.scheduler.schedule( + self._scheduler.schedule( next_refresh_datetime_adjusted, tornado.ioloop.IOLoop.current().add_callback, (self._refresh_token, username, refresh_token)) async def _refresh_token(self, username, refresh_token, force=False): - if not force: - if (username not in self._refresh_tokens) or (self._refresh_tokens[username] != refresh_token): - return + # serialize refreshes per user: a concurrent refresh with the same (rotated) + # refresh token would get a 401 from the provider and log the user out + async with self._refresh_locks[username]: + if not force: + if (username not in self._refresh_tokens) or (self._refresh_tokens[username] != refresh_token): + return - token_response = await self._fetch_token_callback(refresh_token, username) + token_response = await self._fetch_token_callback(refresh_token, username) - if token_response is None: - return + if token_response is None: + return - LOGGER.info(f'Refreshed token for {username}') + LOGGER.info(f'Refreshed token for {username}') - self._refresh_tokens[username] = token_response.refresh_token - self._pending_access_tokens[username] = token_response + self._refresh_tokens[username] = token_response.refresh_token + self._pending_access_tokens[username] = token_response - if token_response.should_refresh(): - self._schedule_token_refresh( - username, - token_response.refresh_token, - token_response.resolve_next_refresh_datetime()) + if token_response.should_refresh(): + self._schedule_token_refresh( + username, + token_response.refresh_token, + token_response.resolve_next_refresh_datetime()) @staticmethod def _restore_token_response_from_cookies(request_handler) -> Optional[OAuthTokenResponse]: diff --git a/src/auth/tornado_auth.py b/src/auth/tornado_auth.py index 05b741a0..64f22138 100644 --- a/src/auth/tornado_auth.py +++ b/src/auth/tornado_auth.py @@ -88,7 +88,8 @@ def authenticate(self, request_handler): LOGGER.info('Authenticated user ' + username) - request_handler.set_secure_cookie('username', username, expires_days=self.authenticator.auth_expiration_days) + server_config = request_handler.application.server_config + request_handler.set_secure_cookie('username', username, expires_days=self.authenticator.auth_expiration_days, httponly=True, secure=server_config.cookie_secure) path = tornado.escape.url_unescape(request_handler.get_argument('next', '/')) diff --git a/src/communications/destination_email.py b/src/communications/destination_email.py index 13731296..a53ca465 100644 --- a/src/communications/destination_email.py +++ b/src/communications/destination_email.py @@ -56,6 +56,7 @@ def __init__(self, params_dict): self.auth_enabled = read_bool_from_config('auth_enabled', params_dict) self.login = params_dict.get('login') self.tls = read_bool_from_config('tls', params_dict) + self.attach_files = read_bool_from_config('attach_files', params_dict, default=True) self.password = self.read_password(params_dict) self.to_addresses = split_addresses(self.to_addresses) @@ -103,7 +104,7 @@ def send(self, title, body, files=None): if self.auth_enabled: server.login(self.login, self.password) - if files: + if self.attach_files and files: for file in files: filename = file.filename part = MIMEApplication(file.content, Name=filename) diff --git a/src/config/config_service.py b/src/config/config_service.py index 9e5c1324..bcc4ebaa 100644 --- a/src/config/config_service.py +++ b/src/config/config_service.py @@ -3,6 +3,7 @@ import os import re import shutil +from datetime import datetime from typing import NamedTuple, Optional from auth.authorization import Authorizer @@ -14,8 +15,6 @@ from utils.file_utils import to_filename from utils.process_utils import ProcessInvoker from utils.string_utils import is_blank, strip -from datetime import datetime - SCRIPT_EDIT_CODE_MODE = 'new_code' SCRIPT_EDIT_UPLOAD_MODE = 'upload_script' @@ -56,12 +55,19 @@ def _create_archive_filename(filename): class ConfigService: - def __init__(self, authorizer, conf_folder, process_invoker: ProcessInvoker) -> None: + def __init__( + self, + authorizer, + conf_folder, + group_scripts_by_folder: bool, + process_invoker: ProcessInvoker) -> None: + self._authorizer = authorizer # type: Authorizer self._script_configs_folder = os.path.join(conf_folder, 'runners') self._scripts_folder = os.path.join(conf_folder, 'scripts') self._scripts_deleted_folder = os.path.join(conf_folder, 'deleted') self._process_invoker = process_invoker + self._group_scripts_by_folder = group_scripts_by_folder file_utils.prepare_folder(self._script_configs_folder) file_utils.prepare_folder(self._scripts_deleted_folder) @@ -117,7 +123,7 @@ def update_config(self, user, config, filename, uploaded_script): with open(original_file_path, 'r') as f: original_config_json = json.load(f) - short_original_config = script_config.read_short(original_file_path, original_config_json) + short_original_config = self.read_short_config(original_config_json, original_file_path) name = config['name'] @@ -133,6 +139,12 @@ def update_config(self, user, config, filename, uploaded_script): LOGGER.info('Updating script config "' + name + '" in ' + original_file_path) self._save_config(config, original_file_path) + def read_short_config(self, config_json, file_path): + return script_config.read_short( + file_path, + config_json, + self._group_scripts_by_folder, + self._script_configs_folder) def delete_config(self, user, name): self._check_admin_access(user) @@ -220,7 +232,7 @@ def list_configs(self, user, mode=None): def load_script(path, content) -> Optional[ShortConfig]: try: config_object = self.load_config_file(path, content) - short_config = script_config.read_short(path, config_object) + short_config = self.read_short_config(config_object, path) if short_config is None: return None @@ -257,7 +269,9 @@ def load_config_model(self, name, user, parameter_values=None, skip_invalid_para user, parameter_values, skip_invalid_parameters, - self._process_invoker) + self._process_invoker, + self._group_scripts_by_folder, + self._script_configs_folder) def _visit_script_configs(self, visitor): configs_dir = self._script_configs_folder @@ -296,7 +310,7 @@ def _find_config(self, name, user) -> Optional[ConfigSearchResult]: def find_and_load(path: str, content): try: config_object = self.load_config_file(path, content) - short_config = script_config.read_short(path, config_object) + short_config = self.read_short_config(config_object, path) if short_config is None: return None @@ -331,7 +345,9 @@ def _load_script_config( user, parameter_values, skip_invalid_parameters, - process_invoker): + process_invoker, + group_scripts_by_folder, + script_configs_folder): if isinstance(content_or_json_dict, str): json_object = custom_json.loads(content_or_json_dict) @@ -342,6 +358,8 @@ def _load_script_config( path, user.get_username(), user.get_audit_name(), + group_scripts_by_folder, + script_configs_folder, process_invoker, pty_enabled_default=os_utils.is_pty_supported()) diff --git a/src/config/script/list_values.py b/src/config/script/list_values.py index e4c4d0c6..5c783519 100644 --- a/src/config/script/list_values.py +++ b/src/config/script/list_values.py @@ -54,7 +54,7 @@ def get_values(self, parameter_values): class DependantScriptValuesProvider(ValuesProvider): def __init__(self, script, parameters_supplier, shell, process_invoker: ProcessInvoker) -> None: - pattern = re.compile('\${([^}]+)\}') + pattern = re.compile(r'\${([^}]+)\}') search_start = 0 script_template = '' diff --git a/src/e2e_tests/conftest.py b/src/e2e_tests/conftest.py index b337905e..567b2ad4 100644 --- a/src/e2e_tests/conftest.py +++ b/src/e2e_tests/conftest.py @@ -1,8 +1,7 @@ -import json -import time - import allure +import json import pytest +import time from selenium.webdriver import Chrome, Firefox, Ie from selenium.webdriver.chrome.options import Options as ChromeOptions from selenium.webdriver.firefox.options import Options as FirefoxOptions @@ -50,15 +49,15 @@ def scripts(): def browser(config_browser, config_headless_mode, request): if config_browser == 'chrome': options = ChromeOptions() - options.headless = config_headless_mode + if config_headless_mode: + options.add_argument('--headless=new') options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') - # mobile_emulation = {"deviceName": "Nexus 5"} - # options.add_experimental_option("mobileEmulation", mobile_emulation) driver = Chrome(options=options) elif config_browser == 'firefox': options = FirefoxOptions() - options.headless = config_headless_mode + if config_headless_mode: + options.add_argument('--headless=new') options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') driver = Firefox(options=options) diff --git a/src/execution/executor.py b/src/execution/executor.py index 03f6007f..7af3f7d2 100644 --- a/src/execution/executor.py +++ b/src/execution/executor.py @@ -114,7 +114,7 @@ def __init_secure_replacements(self): if not element_string.strip(): continue - value_pattern = '((? None: self.allowed_users = None self.alerts_config = None self.logging_config = None + self.groups_config = ScriptGroupsConfig() # type: ScriptGroupsConfig self.admin_config = None self.title = None self.enable_script_titles = None @@ -42,6 +43,7 @@ def __init__(self) -> None: self.user_header_name = None self.secret_storage_file = None self.xsrf_protection = None + self.cookie_secure = True # noinspection PyTypeChecker self.env_vars: EnvVariables = None @@ -59,9 +61,10 @@ def get_ssl_cert_path(self): class LoggingConfig: - def __init__(self, filename_pattern=None, date_format=None) -> None: + def __init__(self, filename_pattern=None, date_format=None, enabled=True) -> None: self.filename_pattern = filename_pattern self.date_format = date_format + self.enabled = enabled @classmethod def from_json(cls, json_config): @@ -71,6 +74,25 @@ def from_json(cls, json_config): json_logging_config = json_config config.filename_pattern = json_logging_config.get('execution_file') config.date_format = json_logging_config.get('execution_date_format') + config.enabled = model_helper.read_bool_from_config('enabled', json_logging_config, default=True) + + return config + + +class ScriptGroupsConfig: + + def __init__(self) -> None: + self.group_by_folders = True + + @classmethod + def from_json(cls, json_config): + config = ScriptGroupsConfig() + + if json_config: + config.group_by_folders = model_helper.read_bool_from_config( + 'group_by_folders', + json_config, + default=config.group_by_folders) return config @@ -180,10 +202,12 @@ def from_json(conf_path, temp_folder): security = model_helper.read_dict(json_object, 'security') + config.cookie_secure = model_helper.read_bool_from_config('cookie_secure', security, default=True) config.allowed_users = _prepare_allowed_users(allowed_users, admin_users, user_groups) config.alerts_config = json_object.get('alerts') config.callbacks_config = json_object.get('callbacks') config.logging_config = LoggingConfig.from_json(json_object.get('logging')) + config.groups_config = ScriptGroupsConfig.from_json(json_object.get('script_groups')) config.user_groups = user_groups config.admin_users = admin_users config.full_history_users = full_history_users @@ -212,12 +236,18 @@ def create_authenticator(auth_object, temp_folder, process_invoker: ProcessInvok elif auth_type == 'google_oauth': from auth.auth_google_oauth import GoogleOauthAuthenticator authenticator = GoogleOauthAuthenticator(auth_object) + elif auth_type == 'azure_ad_oauth': + from auth.auth_azure_ad_oauth import AzureAdOAuthAuthenticator + authenticator = AzureAdOAuthAuthenticator(auth_object) elif auth_type == 'gitlab': from auth.auth_gitlab import GitlabOAuthAuthenticator authenticator = GitlabOAuthAuthenticator(auth_object) elif auth_type == 'keycloak_openid': from auth.auth_keycloak_openid import KeycloakOpenidAuthenticator authenticator = KeycloakOpenidAuthenticator(auth_object) + elif auth_type == 'authentik': + from auth.auth_authentik_openid import AuthentikOpenidAuthenticator + authenticator = AuthentikOpenidAuthenticator(auth_object) elif auth_type == 'htpasswd': from auth.auth_htpasswd import HtpasswdAuthenticator authenticator = HtpasswdAuthenticator(auth_object, process_invoker) diff --git a/src/model/template_property.py b/src/model/template_property.py index 6b151008..64601094 100644 --- a/src/model/template_property.py +++ b/src/model/template_property.py @@ -7,20 +7,26 @@ class TemplateProperty: def __init__(self, template_config, parameters: ObservableList, value_wrappers: ObservableDict, empty=None) -> None: self._value_property = Property(None) - self._template_config = template_config self._values = value_wrappers self._empty = empty self._parameters = parameters - pattern = re.compile('\${([^}]+)\}') + pattern = re.compile(r'\${([^}]+)\}') search_start = 0 script_template = '' - required_parameters = set() - templates = template_config if isinstance(template_config, list) else [template_config] + self._multiple_templates = isinstance(template_config, list) + if template_config: + self._templates = template_config if isinstance(template_config, list) else [template_config] + else: + self._templates = [] + + self._template_required_parameters = {} + + for template in self._templates: + required_parameters = set() - for template in templates: if template: while search_start < len(template): match = pattern.search(template, search_start) @@ -36,7 +42,9 @@ def __init__(self, template_config, parameters: ObservableList, value_wrappers: search_start = match.end() + 1 - self.required_parameters = tuple(required_parameters) + self._template_required_parameters[template] = required_parameters + + self.required_parameters = set().union(*self._template_required_parameters.values()) self._reload() @@ -57,25 +65,31 @@ def on_remove(self, parameter): self._reload() def _reload(self): - values_filled = True - for param_name in self.required_parameters: - value_wrapper = self._values.get(param_name) - if value_wrapper is None or is_empty(value_wrapper.mapped_script_value): - values_filled = False - break - - if self._template_config is None: + if not self._templates: self.value = None - elif values_filled: - if isinstance(self._template_config, list): - values = [] - for single_template in self._template_config: - values.append(fill_parameter_values(self._parameters, single_template, self._values)) - self.value = values - else: - self.value = fill_parameter_values(self._parameters, self._template_config, self._values) else: - self.value = self._empty + any_values_filled = False + values = [] + + for template in self._templates: + template_values_filled = True + for param_name in self._template_required_parameters[template]: + value_wrapper = self._values.get(param_name) + if value_wrapper is None or is_empty(value_wrapper.mapped_script_value): + template_values_filled = False + break + + if template_values_filled: + values.append(fill_parameter_values(self._parameters, template, self._values)) + any_values_filled = True + + if any_values_filled: + if not self._multiple_templates: + self.value = values[0] + else: + self.value = values + else: + self.value = self._empty self._value_property.set(self.value) diff --git a/src/tests/auth/test_auth_abstract_oauth.py b/src/tests/auth/test_auth_abstract_oauth.py index fb5c6401..b82994fb 100644 --- a/src/tests/auth/test_auth_abstract_oauth.py +++ b/src/tests/auth/test_auth_abstract_oauth.py @@ -157,13 +157,18 @@ def mock_request_handler(code): handler_mock.get_secure_cookie = lambda cookie: secure_cookies.get(cookie) - def set_secure_cookie(cookie, value): + def set_secure_cookie(cookie, value, **kwargs): secure_cookies[cookie] = value.encode('utf-8') def clear_secure_cookie(cookie): if cookie in secure_cookies: del secure_cookies[cookie] + server_config = mock_object() + server_config.cookie_secure = False + + handler_mock.application = mock_object() + handler_mock.application.server_config = server_config handler_mock.set_secure_cookie = set_secure_cookie handler_mock.clear_cookie = clear_secure_cookie @@ -592,7 +597,7 @@ def start_quick_timer(callback): return timer = threading.Timer(0.01, callback) - timer.setDaemon(True) + timer.daemon = True timer.start() self.timer_invocations += 1 diff --git a/src/tests/auth/test_auth_htpasswd.py b/src/tests/auth/test_auth_htpasswd.py index 21cdda4c..8bc97943 100644 --- a/src/tests/auth/test_auth_htpasswd.py +++ b/src/tests/auth/test_auth_htpasswd.py @@ -1,4 +1,5 @@ import sys +import unittest from unittest import TestCase, mock from unittest.mock import patch diff --git a/src/tests/auth/test_auth_keycloak_openid.py b/src/tests/auth/test_auth_keycloak_openid.py index 7698b32f..fca9ef71 100644 --- a/src/tests/auth/test_auth_keycloak_openid.py +++ b/src/tests/auth/test_auth_keycloak_openid.py @@ -16,8 +16,9 @@ REALM_URL = 'http://my-keycloak.net/realms/master' -access_expiration_duration = 0.1 -refresh_expiration_duration = 0.6 +access_expiration_duration = 0.5 +refresh_expiration_duration = 2.0 +auth_info_ttl = 1.0 class OauthServerMock: @@ -118,7 +119,8 @@ def send_tokens(self, token_prefix, request_handler): 'refresh_expires_in': refresh_expiration_duration }) - self.cleanup_old_tokens(self.access_token_expiration_times, token_prefix) + # Real Keycloak rotates refresh tokens, but old access tokens are stateless JWTs + # and stay valid until their expiration, even after a refresh self.cleanup_old_tokens(self.refresh_token_expiration_times, token_prefix) self.access_token_expiration_times[access_token] = time.time() + access_expiration_duration @@ -187,18 +189,61 @@ async def test_success_validate_after_refresh(self): self.oauth_server.set_groups('bugy', ['g3']) - await gen.sleep(0.4 + 0.1) + await gen.sleep(auth_info_ttl + 0.5) valid_1 = await self.authenticator.validate_user(username, mock_request_handler(previous_request=request_1)) self.assertTrue(valid_1) - for i in range(1, 8): - await gen.sleep(0.05) + await self.wait_for_groups('bugy', ['g3']) - if self.authenticator.get_groups('bugy') == ['g3']: + @gen_test + async def test_success_validate_when_refresh_races_with_validation(self): + # Regression test for a flaky failure of test_success_validate_after_refresh: + # the scheduler-driven token refresh fires on the IOLoop right before validate_user, + # so the userinfo request is sent with an access token from before the refresh + username, request_1 = await self.authenticate('qwerty123') + + self.oauth_server.set_groups('bugy', ['g3']) + + await gen.sleep(auth_info_ttl + 0.5) + + token_manager = self.authenticator._token_manager + current_refresh_token = token_manager._refresh_tokens[username] + self.io_loop.add_callback(token_manager._refresh_token, username, current_refresh_token) + + valid_1 = await self.authenticator.validate_user(username, mock_request_handler(previous_request=request_1)) + self.assertTrue(valid_1) + + await self.wait_for_groups('bugy', ['g3']) + + @gen_test + async def test_success_validate_when_concurrent_refreshes(self): + # Two refreshes in flight with the same refresh token: without per-user + # serialization, the second one gets 401 (token rotated) and logs the user out + username, request_1 = await self.authenticate('qwerty123') + + self.oauth_server.set_groups('bugy', ['g3']) + + await gen.sleep(auth_info_ttl + 0.5) + + token_manager = self.authenticator._token_manager + current_refresh_token = token_manager._refresh_tokens[username] + self.io_loop.add_callback(token_manager._refresh_token, username, current_refresh_token) + self.io_loop.add_callback(token_manager._refresh_token, username, current_refresh_token) + + valid_1 = await self.authenticator.validate_user(username, mock_request_handler(previous_request=request_1)) + self.assertTrue(valid_1) + + await self.wait_for_groups('bugy', ['g3']) + + async def wait_for_groups(self, username, expected_groups): + for i in range(1, 20): + if self.authenticator.get_groups(username) == expected_groups: break - self.assertEqual(['g3'], self.authenticator.get_groups('bugy')) + await gen.sleep(0.1) + + self.assertEqual(expected_groups, self.authenticator.get_groups(username)) @gen_test async def test_failed_validate_after_deactivate(self): @@ -296,7 +341,7 @@ def create_authenticator(self, dump_file=None): 'client_id': 'my-client', 'secret': 'top_secret', 'group_support': True, - 'auth_info_ttl': 0.4, + 'auth_info_ttl': auth_info_ttl, 'state_dump_file': dump_file }) diff --git a/src/tests/auth_ldap_test.py b/src/tests/auth_ldap_test.py index f49f779b..77bb8fee 100644 --- a/src/tests/auth_ldap_test.py +++ b/src/tests/auth_ldap_test.py @@ -1,21 +1,40 @@ import unittest +from typing import Dict from ldap3 import Connection, SIMPLE, MOCK_SYNC, OFFLINE_AD_2012_R2, Server from ldap3.utils.dn import safe_dn -from auth.auth_base import AuthRejectedError +from auth.auth_base import AuthRejectedError, AuthFailureError from auth.auth_ldap import LdapAuthenticator from tests import test_utils from tests.test_utils import mock_request_handler class _LdapAuthenticatorMockWrapper: - def __init__(self, username_pattern, base_dn): - authenticator = LdapAuthenticator({ - 'url': 'unused', - 'username_pattern': username_pattern, - 'base_dn': base_dn}, - test_utils.temp_folder) + def __init__(self, + username_pattern=None, + base_dn=None, + search_user_by_attribute=None, + admin_user=None, + admin_password=None): + config = {'url': 'unused'} # type: Dict[str, object] + + if username_pattern or search_user_by_attribute: + user_resolver_config = {} + if username_pattern: + user_resolver_config['username_pattern'] = username_pattern + if search_user_by_attribute: + user_resolver_config['search_by_attribute'] = search_user_by_attribute + if admin_user: + user_resolver_config['admin_user'] = admin_user + if admin_password: + user_resolver_config['admin_password'] = admin_password + config['ldap_user_resolver'] = user_resolver_config + + if base_dn: + config['base_dn'] = base_dn + + authenticator = LdapAuthenticator(config, test_utils.temp_folder) def connect(username, password): server = Server('mock_server', get_info=OFFLINE_AD_2012_R2) @@ -51,11 +70,12 @@ def connect(username, password): connection.bind() return connection - authenticator._connect = connect + authenticator._ldap_connector.connect = connect self.base_dn = base_dn self._entries = {} self.authenticator = authenticator + self.add_user('Admin', 'admin_pass') def authenticate(self, username, password): return self.authenticator.authenticate(_mock_request_handler(username, password)) @@ -342,5 +362,156 @@ def tearDown(self): test_utils.cleanup() +class TestLdapUserResolver(unittest.TestCase): + + def test_authenticate_with_uid_resolver(self): + auth_wrapper = _LdapAuthenticatorMockWrapper( + base_dn='dc=ldap,dc=test', + search_user_by_attribute='uid', + admin_user='cn=admin,cn=users,dc=ldap,dc=test', + admin_password='admin_pass' + ) + + auth_wrapper.add_user('John Doe', 'user_pass', uid='johndoe') + + user = auth_wrapper.authenticate('johndoe', 'user_pass') + self.assertEqual(user, 'johndoe') + + def test_authenticate_with_different_attribute(self): + auth_wrapper = _LdapAuthenticatorMockWrapper( + base_dn='dc=ldap,dc=test', + search_user_by_attribute='sAMAccountName', + admin_user='cn=admin,cn=users,dc=ldap,dc=test', + admin_password='admin_pass' + ) + + auth_wrapper.add_user('Jane Smith', 'user_pass', sAMAccountName='jsmith') + + user = auth_wrapper.authenticate('jsmith', 'user_pass') + self.assertEqual(user, 'jsmith') + + def test_authenticate_fails_with_wrong_uid(self): + auth_wrapper = _LdapAuthenticatorMockWrapper( + base_dn='dc=ldap,dc=test', + search_user_by_attribute='uid', + admin_user='cn=admin,cn=users,dc=ldap,dc=test', + admin_password='admin_pass' + ) + + auth_wrapper.add_user('John Doe', 'user_pass', uid='johndoe') + + self.assertRaisesRegex( + AuthRejectedError, + 'Invalid credentials', + auth_wrapper.authenticate, + 'nonexistent', + 'user_pass' + ) + + def test_authenticate_fails_with_wrong_password(self): + auth_wrapper = _LdapAuthenticatorMockWrapper( + base_dn='dc=ldap,dc=test', + search_user_by_attribute='uid', + admin_user='cn=admin,cn=users,dc=ldap,dc=test', + admin_password='admin_pass' + ) + + auth_wrapper.add_user('John Doe', 'user_pass', uid='johndoe') + + self.assertRaisesRegex( + AuthRejectedError, + 'Invalid credentials', + auth_wrapper.authenticate, + 'johndoe', + 'wrong_pass' + ) + + def test_authenticate_fails_with_wrong_admin_password(self): + auth_wrapper = _LdapAuthenticatorMockWrapper( + base_dn='dc=ldap,dc=test', + search_user_by_attribute='uid', + admin_user='cn=admin,cn=users,dc=ldap,dc=test', + admin_password='wrong_pass' + ) + + auth_wrapper.add_user('John Doe', 'user_pass', uid='johndoe') + + self.assertRaisesRegex( + AuthFailureError, + 'Failed to bind with admin LDAP user: invalidCredentials', + auth_wrapper.authenticate, + 'johndoe', + 'user_pass' + ) + + def test_load_groups_with_uid_resolver(self): + auth_wrapper = _LdapAuthenticatorMockWrapper( + base_dn='dc=ldap,dc=test', + search_user_by_attribute='uid', + admin_user='cn=admin,cn=users,dc=ldap,dc=test', + admin_password='admin_pass' + ) + + auth_wrapper.add_user('John Doe', 'user_pass', uid='johndoe') + auth_wrapper.add_group('admin_group', ['John Doe']) + + user = auth_wrapper.authenticate('johndoe', 'user_pass') + groups = auth_wrapper.get_groups('johndoe') + self.assertEqual(['admin_group'], groups) + + def setUp(self): + test_utils.setup() + + def tearDown(self): + test_utils.cleanup() + + +class TestConfigValidation(unittest.TestCase): + + def test_reject_both_username_pattern_and_search_by_attribute(self): + config = { + 'url': 'ldap://localhost', + 'ldap_user_resolver': { + 'username_pattern': 'cn=$username,dc=test', + 'search_by_attribute': 'uid', + 'admin_user': 'admin', + 'admin_password': 'pass' + } + } + + with self.assertRaisesRegex(ValueError, 'Cannot specify both username_pattern and search_by_attribute'): + LdapAuthenticator(config, test_utils.temp_folder) + + def test_reject_neither_username_pattern_nor_search_by_attribute(self): + config = { + 'url': 'ldap://localhost', + 'ldap_user_resolver': { + 'admin_user': 'admin' + } + } + + with self.assertRaisesRegex(ValueError, 'Either username_pattern or search_by_attribute must be specified'): + LdapAuthenticator(config, test_utils.temp_folder) + + def test_reject_incomplete_search_by_attribute_config(self): + config = { + 'url': 'ldap://localhost', + 'ldap_user_resolver': { + 'search_by_attribute': 'uid', + 'admin_user': 'admin' + # Missing admin_password + } + } + + with self.assertRaisesRegex(Exception, 'admin_password.*for ldap_user_resolver'): + LdapAuthenticator(config, test_utils.temp_folder) + + def setUp(self): + test_utils.setup() + + def tearDown(self): + test_utils.cleanup() + + def _mock_request_handler(username, password): return mock_request_handler(arguments={'username': username, 'password': password}) diff --git a/src/tests/config_service_test.py b/src/tests/config_service_test.py index a2753df0..c1430bfd 100644 --- a/src/tests/config_service_test.py +++ b/src/tests/config_service_test.py @@ -1,6 +1,7 @@ import json import os import sys +import tempfile import unittest from collections import OrderedDict from shutil import copyfile @@ -30,6 +31,14 @@ def test_list_configs_when_one(self): self.assertEqual(1, len(configs)) self.assertEqual('conf_x', configs[0].name) + def test_list_configs_when_one_and_symlink(self): + conf_path = os.path.join(test_utils.temp_folder, 'runners', 'sub', 'x.json') + with self._temporary_file_symlink(conf_path, {'name': 'test X'}): + configs = self.config_service.list_configs(self.user) + self.assertEqual(1, len(configs)) + self.assertEqual('test X', configs[0].name) + self.assertEqual('sub', configs[0].group) + def test_list_configs_when_multiple(self): _create_script_config_file('conf_x') _create_script_config_file('conf_y') @@ -40,9 +49,9 @@ def test_list_configs_when_multiple(self): self.assertCountEqual(['conf_x', 'conf_y', 'A B C'], conf_names) def test_list_configs_when_multiple_and_subfolders(self): - _create_script_config_file('conf_x', subfolder = 's1') - _create_script_config_file('conf_y', subfolder = 's2') - _create_script_config_file('ABC', subfolder = os.path.join('s1', 'inner')) + _create_script_config_file('conf_x', subfolder='s1') + _create_script_config_file('conf_y', subfolder='s2') + _create_script_config_file('ABC', subfolder=os.path.join('s1', 'inner')) configs = self.config_service.list_configs(self.user) conf_names = [config.name for config in configs] @@ -114,6 +123,41 @@ def test_load_config_with_slash_in_name(self): config = self.config_service.load_config_model('Name with slash /', self.user) self.assertEqual('Name with slash /', config.name) + def test_list_configs_when_multiple_subfolders_and_symlink(self): + def create_config_file(name, relative_path, group=None): + filename = os.path.basename(relative_path) + config = {'name': name} + if group is not None: + config['group'] = group + test_utils.write_script_config( + config, + filename, + config_folder=os.path.join(test_utils.temp_folder, 'runners', os.path.dirname(relative_path))) + + subfolder = os.path.join(test_utils.temp_folder, 'runners', 'sub') + symlink_path = os.path.join(subfolder, 'x.json') + with self._temporary_file_symlink(symlink_path, {'name': 'test X'}): + create_config_file('conf Y', os.path.join('sub', 'y', 'conf_y.json')) + create_config_file('conf Z', os.path.join('sub', 'z', 'conf_z.json')) + create_config_file('conf A', 'conf_a.json') + create_config_file('conf B', os.path.join('b', 'conf_b.json')) + create_config_file('conf C', os.path.join('c', 'conf_c.json'), group='test group') + create_config_file('conf D', os.path.join('d', 'conf_d.json'), group='') + + configs = self.config_service.list_configs(self.user) + actual_name_group_map = {c.name: c.group for c in configs} + + self.assertEqual( + actual_name_group_map, + {'test X': 'sub', + 'conf Y': 'sub', + 'conf Z': 'sub', + 'conf A': None, + 'conf B': 'b', + 'conf C': 'test group', + 'conf D': None}, + ) + def tearDown(self): super().tearDown() test_utils.cleanup() @@ -125,7 +169,19 @@ def setUp(self): self.user = User('ConfigServiceTest', {AUTH_USERNAME: 'ConfigServiceTest'}) self.admin_user = User('admin_user', {AUTH_USERNAME: 'The Admin'}) authorizer = Authorizer(ANY_USER, ['admin_user'], [], [], EmptyGroupProvider()) - self.config_service = ConfigService(authorizer, test_utils.temp_folder, test_utils.process_invoker) + self.config_service = ConfigService(authorizer, test_utils.temp_folder, True, test_utils.process_invoker) + + @staticmethod + def _temporary_file_symlink(symlink_path, file_content: dict): + f = tempfile.NamedTemporaryFile() + + f.write(json.dumps(file_content).encode('utf-8')) + f.flush() + subdir = os.path.dirname(symlink_path) + os.makedirs(subdir) + os.symlink(f.name, symlink_path) + + return f class ConfigServiceAuthTest(unittest.TestCase): @@ -209,7 +265,11 @@ def setUp(self): authorizer = Authorizer([], ['adm_user'], [], [], EmptyGroupProvider()) self.user1 = User('user1', {}) self.admin_user = User('adm_user', {}) - self.config_service = ConfigService(authorizer, test_utils.temp_folder, test_utils.process_invoker) + self.config_service = ConfigService( + authorizer, + test_utils.temp_folder, + True, + test_utils.process_invoker) def script_path(path): @@ -242,7 +302,7 @@ def setUp(self): authorizer = Authorizer([], ['admin_user', 'admin_non_editor'], [], ['admin_user'], EmptyGroupProvider()) self.admin_user = User('admin_user', {}) - self.config_service = ConfigService(authorizer, test_utils.temp_folder, test_utils.process_invoker) + self.config_service = ConfigService(authorizer, test_utils.temp_folder, True, test_utils.process_invoker) def tearDown(self): super().tearDown() @@ -416,7 +476,7 @@ def setUp(self): authorizer = Authorizer([], ['admin_user', 'admin_non_editor'], [], ['admin_user'], EmptyGroupProvider()) self.admin_user = User('admin_user', {}) - self.config_service = ConfigService(authorizer, test_utils.temp_folder, test_utils.process_invoker) + self.config_service = ConfigService(authorizer, test_utils.temp_folder, True, test_utils.process_invoker) for suffix in 'XYZ': name = 'Conf ' + suffix @@ -669,7 +729,7 @@ def setUp(self): authorizer = Authorizer([], ['admin_user'], [], [], EmptyGroupProvider()) self.admin_user = User('admin_user', {}) - self.config_service = ConfigService(authorizer, test_utils.temp_folder, test_utils.process_invoker) + self.config_service = ConfigService(authorizer, test_utils.temp_folder, True, test_utils.process_invoker) def tearDown(self): super().tearDown() @@ -717,7 +777,7 @@ def setUp(self) -> None: authorizer = Authorizer([], ['admin_user', 'admin_non_editor'], [], ['admin_user'], EmptyGroupProvider()) self.admin_user = User('admin_user', {}) - self.config_service = ConfigService(authorizer, test_utils.temp_folder, test_utils.process_invoker) + self.config_service = ConfigService(authorizer, test_utils.temp_folder, True, test_utils.process_invoker) for pair in [('script.py', b'123'), ('another.py', b'xyz'), diff --git a/src/tests/env_utils_test.py b/src/tests/env_utils_test.py index dbc572db..1d9f0b73 100644 --- a/src/tests/env_utils_test.py +++ b/src/tests/env_utils_test.py @@ -67,27 +67,27 @@ def test_invalid_minor_version(self): class TestEnvVariables(unittest.TestCase): def test_default(self): env_vars = EnvVariables(os.environ) - self.assertEqual(os.getlogin(), env_vars.build_env_vars()['USER']) + self.assertEqual(os.environ['USER'], env_vars.build_env_vars()['USER']) def test_extra_variables(self): env_vars = EnvVariables(os.environ) all_env_vars = env_vars.build_env_vars(extra_variables={'my_var': 'abcd'}) - self.assertEqual(os.getlogin(), all_env_vars['USER']) + self.assertEqual(os.environ['USER'], all_env_vars['USER']) self.assertEqual('abcd', all_env_vars['my_var']) def test_default_extra_variables(self): env_vars = EnvVariables(os.environ, extra_variables={'my_var2': 'def'}) all_env_vars = env_vars.build_env_vars() - self.assertEqual(os.getlogin(), all_env_vars['USER']) + self.assertEqual(os.environ['USER'], all_env_vars['USER']) self.assertEqual('def', all_env_vars['my_var2']) def test_extra_variables_when_collission(self): env_vars = EnvVariables(os.environ, extra_variables={'my_var': 'def'}) all_env_vars = env_vars.build_env_vars(extra_variables={'my_var': 'abcd'}) - self.assertEqual(os.getlogin(), all_env_vars['USER']) + self.assertEqual(os.environ['USER'], all_env_vars['USER']) self.assertEqual('abcd', all_env_vars['my_var']) def test_hidden_variables(self): diff --git a/src/tests/execution_logging_test.py b/src/tests/execution_logging_test.py index 738b4da9..79de102a 100644 --- a/src/tests/execution_logging_test.py +++ b/src/tests/execution_logging_test.py @@ -280,7 +280,7 @@ def test_get_history_entries_only_for_current_user(self, user_id): self.simulate_logging(execution_id='id4', user_id='userA') entries = self._get_entries_sorted(user_id) - self.assertEquals(2, len(entries)) + self.assertEqual(2, len(entries)) self.validate_history_entry(entry=entries[0], id='id1', user_id='userA') self.validate_history_entry(entry=entries[1], id='id4', user_id='userA') @@ -292,7 +292,7 @@ def test_get_history_entries_for_power_user(self): self.simulate_logging(execution_id='id4', user_id='userA') entries = self._get_entries_sorted('power_user') - self.assertEquals(4, len(entries)) + self.assertEqual(4, len(entries)) self.validate_history_entry(entry=entries[0], id='id1', user_id='userA') self.validate_history_entry(entry=entries[1], id='id2', user_id='userB') @@ -306,7 +306,7 @@ def test_get_history_entries_for_system_call(self): self.simulate_logging(execution_id='id4', user_id='userA') entries = self._get_entries_sorted('some user', system_call=True) - self.assertEquals(4, len(entries)) + self.assertEqual(4, len(entries)) self.validate_history_entry(entry=entries[0], id='id1', user_id='userA') self.validate_history_entry(entry=entries[1], id='id2', user_id='userB') @@ -357,7 +357,7 @@ def test_entry_with_user_id_name_different(self): entry = self.logging_service.find_history_entry('id1', '192.168.2.12') self.validate_history_entry(entry, id='id1', user_name='userX', user_id='192.168.2.12') - def test_find_entry_when_windows_line_seperator(self): + def test_find_entry_when_windows_line_separator(self): self.simulate_logging(execution_id='id1', user_name='userX', user_id='192.168.2.12') _replace_line_separators(self.get_log_files(), '\n', '\r\n') @@ -381,7 +381,7 @@ def test_find_entry_when_another_user_and_no_entry(self): entry = self.logging_service.find_history_entry('id2', 'userA') self.assertIsNone(entry) - def test_find_log_when_windows_line_seperator(self): + def test_find_log_when_windows_line_separator(self): self.simulate_logging(execution_id='id1', log_lines=['hello', 'wonderful', 'world']) _replace_line_separators(self.get_log_files(), '\n', '\r\n') @@ -542,7 +542,8 @@ def test_logging_values(self): 'my_script', script_command='echo', parameters=[param1, param2, param3, param4], - logging_config=LoggingConfig('test-${SCRIPT}-${p1}')) + logging_config=LoggingConfig('test-${SCRIPT}-${p1}'), + path=os.path.join('conf', 'my_script.json')) config_model.set_all_param_values({'p1': 'abc', 'p3': True, 'p4': 987}) execution_id = self.executor_service.start_script( @@ -568,7 +569,7 @@ def test_logging_values(self): self.assertEqual('some text\nanother text', log) log_files = os.listdir(test_utils.temp_folder) - self.assertEqual(['test-my_script-abc.log'], log_files) + self.assertCountEqual(['test-my_script-abc.log', 'conf'], log_files) def test_exit_code(self): config_model = create_config_model( diff --git a/src/tests/execution_service_test.py b/src/tests/execution_service_test.py index 0e5346c2..9ffa4c23 100644 --- a/src/tests/execution_service_test.py +++ b/src/tests/execution_service_test.py @@ -10,7 +10,6 @@ from execution.execution_service import ExecutionService from execution.executor import create_process_wrapper from model.model_helper import AccessProhibitedException -from model.script_config import ConfigModel from tests import test_utils from tests.test_utils import mock_object, create_audit_names, _MockProcessWrapper, _IdGeneratorMock from utils import audit_utils @@ -441,10 +440,12 @@ def _start_with_config(execution_service, config, parameter_values=None, user_id def _create_script_config(parameter_configs): - config = ConfigModel( - {'name': 'script_x', - 'script_path': 'ls', - 'parameters': parameter_configs}, - 'script_x.json', 'user1', 'localhost', - test_utils.process_invoker) - return config + return test_utils.create_config_model( + 'script_x', + config={'name': 'script_x', + 'script_path': 'ls', + 'parameters': parameter_configs}, + username='user1', + audit_name='localhost', + + ) diff --git a/src/tests/executor_test.py b/src/tests/executor_test.py index 1245c910..21d29e9f 100644 --- a/src/tests/executor_test.py +++ b/src/tests/executor_test.py @@ -30,10 +30,12 @@ def test_start_without_values(self): self.assertEqual(None, process_wrapper.working_directory) self.assertEqual(['ls'], process_wrapper.command) - expected_values = {} - expected_values.update(os.environ) + expected_values = dict(os.environ) expected_values['EXECUTION_ID'] = '123' - self.assertEqual(expected_values, process_wrapper.all_env_variables) + expected_values.pop('PYTEST_CURRENT_TEST', None) + actual = dict(process_wrapper.all_env_variables) + actual.pop('PYTEST_CURRENT_TEST', None) + self.assertEqual(expected_values, actual) def test_start_with_one_value(self): config = create_config_model( diff --git a/src/tests/file_download_feature_test.py b/src/tests/file_download_feature_test.py index 80464d4c..3c9ee35e 100644 --- a/src/tests/file_download_feature_test.py +++ b/src/tests/file_download_feature_test.py @@ -86,22 +86,22 @@ def test_double_asterisk_match_multiple_files_when_complex(self): }) def test_regex_only_0_matches(self): - files = file_download_feature.find_matching_files('#\d+#', 'some text without numbers') + files = file_download_feature.find_matching_files(r'#\d+#', 'some text without numbers') self.assertEqual(files, []) def test_regex_only_1_match(self): - files = file_download_feature.find_matching_files('#(\/[^\/]+)+#', 'the text is in /home/username/text.txt') + files = file_download_feature.find_matching_files(r'#(\/[^\/]+)+#', 'the text is in /home/username/text.txt') self.assertEqual(files, ['/home/username/text.txt']) def test_regex_only_3_matches(self): - files = file_download_feature.find_matching_files('#(\/([\w.\-]|(\\\ ))+)+#', 'found files: ' - '/home/username/text.txt, ' - '/tmp/data.dat, ' - '/opt/software/script\ server/read_me.md') + files = file_download_feature.find_matching_files(r'#(\/([\w.\-]|(\\\ ))+)+#', 'found files: ' + '/home/username/text.txt, ' + '/tmp/data.dat, ' + '/opt/software/script\\ server/read_me.md') - self.assertEqual(files, ['/home/username/text.txt', '/tmp/data.dat', '/opt/software/script\ server/read_me.md']) + self.assertEqual(files, ['/home/username/text.txt', '/tmp/data.dat', '/opt/software/script\\ server/read_me.md']) def test_regex_only_any_path_linux_3_matches(self): test_utils.set_linux() @@ -109,9 +109,9 @@ def test_regex_only_any_path_linux_3_matches(self): files = file_download_feature.find_matching_files('##any_path#', 'found files: ' '/home/username/text.txt, ' '/tmp/data.dat, ' - '/opt/software/script\ server/read_me.md') + '/opt/software/script\\ server/read_me.md') - self.assertEqual(files, ['/home/username/text.txt', '/tmp/data.dat', '/opt/software/script\ server/read_me.md']) + self.assertEqual(files, ['/home/username/text.txt', '/tmp/data.dat', '/opt/software/script\\ server/read_me.md']) def test_regex_only_any_path_win_3_matches(self): test_utils.set_win() @@ -119,11 +119,11 @@ def test_regex_only_any_path_win_3_matches(self): files = file_download_feature.find_matching_files('##any_path#', 'found files: ' 'C:\\Users\\username\\text.txt, ' 'D:\\windows\\System32, ' - 'C:\\Program\ Files\\script\ server\\read_me.md') + 'C:\\Program\\ Files\\script\\ server\\read_me.md') self.assertEqual(files, ['C:\\Users\\username\\text.txt', 'D:\\windows\\System32', - 'C:\\Program\ Files\\script\ server\\read_me.md']) + 'C:\\Program\\ Files\\script\\ server\\read_me.md']) def test_regex_only_search_user_home_win(self): test_utils.set_win() @@ -134,21 +134,21 @@ def test_regex_only_search_user_home_win(self): self.assertEqual(files, ['~\\text.txt']) def test_1_regex_and_text_no_matches(self): - files = file_download_feature.find_matching_files('/home/username/#\d+#', 'username=some_name\n ' + files = file_download_feature.find_matching_files(r'/home/username/#\d+#', 'username=some_name\n ' 'folder=some_folder\n ' 'time=now') self.assertEqual(files, []) def test_1_regex_and_text_1_match(self): - files = file_download_feature.find_matching_files('/home/username/#\d+#', 'username=some_name\n ' + files = file_download_feature.find_matching_files(r'/home/username/#\d+#', 'username=some_name\n ' 'folder=some_folder\n ' 'time=153514') self.assertEqual(files, ['/home/username/153514']) def test_1_regex_and_text_3_matches(self): - files = file_download_feature.find_matching_files('/home/username/#\d+#', 'username=some_name\n ' + files = file_download_feature.find_matching_files(r'/home/username/#\d+#', 'username=some_name\n ' 'folder=some_folder\n ' 'time=153514\n ' 'age=18, size=256Mb') @@ -156,7 +156,7 @@ def test_1_regex_and_text_3_matches(self): self.assertEqual(files, ['/home/username/153514', '/home/username/18', '/home/username/256']) def test_1_regex_with_first_group_and_text_1_match(self): - files = file_download_feature.find_matching_files('/home/#1#username=(\w+)#/file.txt', 'username=some_name\n ' + files = file_download_feature.find_matching_files(r'/home/#1#username=(\w+)#/file.txt', 'username=some_name\n ' 'folder=some_folder\n ' 'time=153514\n ' 'age=18, size=256Mb') @@ -164,7 +164,7 @@ def test_1_regex_with_first_group_and_text_1_match(self): self.assertEqual(files, ['/home/some_name/file.txt']) def test_1_regex_with_second_group_and_text_2_matches(self): - files = file_download_feature.find_matching_files('/home/username/#2#=(some_(\w+))#.txt', + files = file_download_feature.find_matching_files(r'/home/username/#2#=(some_(\w+))#.txt', 'username=some_name\n ' 'folder=some_folder\n ' 'time=153514\n ' @@ -173,7 +173,7 @@ def test_1_regex_with_second_group_and_text_2_matches(self): self.assertEqual(files, ['/home/username/name.txt', '/home/username/folder.txt']) def test_2_regexes_1_match(self): - files = file_download_feature.find_matching_files('/home/#2#username=((\w+))#/#1#time=(\d+)#.txt', + files = file_download_feature.find_matching_files(r'/home/#2#username=((\w+))#/#1#time=(\d+)#.txt', 'username=some_name\n ' 'folder=some_folder\n ' 'time=153514\n ' @@ -184,7 +184,7 @@ def test_2_regexes_1_match(self): def test_1_regex_and_asterisk(self): test_utils.create_file(os.path.join('some_folder', 'file.txt')) - files = file_download_feature.find_matching_files('*/#1#folder=(\w+)#/*.txt', 'username=some_name\n ' + files = file_download_feature.find_matching_files(r'*/#1#folder=(\w+)#/*.txt', 'username=some_name\n ' 'folder=some_folder\n ' 'time=153514\n ' 'age=18, size=256Mb') @@ -507,7 +507,7 @@ def test_single_dynamic_image(self): def test_single_dynamic_image_when_unnormalized(self): test_utils.create_file('sub/test.png') - config = create_config_model('my_script', output_files=[inline_image('#([\.\w]+/)+\w+.png#')]) + config = create_config_model('my_script', output_files=[inline_image(r'#([\.\w]+/)+\w+.png#')]) execution_id = self.start_execution(config) @@ -526,10 +526,10 @@ def test_mixed_images_when_multiple_output(self): path5 = test_utils.create_file('some/long/path/me.jpg') config = create_config_model('my_script', output_files=[ - inline_image(test_utils.temp_folder + os_utils.path_sep() + '#test\d+.png#'), + inline_image(test_utils.temp_folder + os_utils.path_sep() + r'#test\d+.png#'), inline_image(path2), inline_image(path3), - inline_image('##any_path/path/\w+#.jpg') + inline_image(r'##any_path/path/\w+#.jpg') ]) execution_id = self.start_execution(config) diff --git a/src/tests/file_utils_test.py b/src/tests/file_utils_test.py index 5b5cac00..40695643 100644 --- a/src/tests/file_utils_test.py +++ b/src/tests/file_utils_test.py @@ -7,13 +7,13 @@ class TestToFilename(TestCase): def test_replace_special_characters_linux(self): os_utils.set_linux() - filename = file_utils.to_filename('!@#$%^&*()_+\|/?.<>,\'"') + filename = file_utils.to_filename('!@#$%^&*()_+\\|/?.<>,\'"') self.assertEqual('!@#$%^&*()_+\\|_?.<>,\'"', filename) def test_replace_special_characters_windows(self): os_utils.set_win() - filename = file_utils.to_filename('!@#$%^&*()_+\|/?.<>,\'"') + filename = file_utils.to_filename('!@#$%^&*()_+\\|/?.<>,\'"') self.assertEqual('!@#$%^&_()_+____.__,\'_', filename) def tearDown(self) -> None: diff --git a/src/tests/ip_idenfication_test.py b/src/tests/ip_idenfication_test.py index 52f66aed..c28620c5 100644 --- a/src/tests/ip_idenfication_test.py +++ b/src/tests/ip_idenfication_test.py @@ -15,6 +15,8 @@ def mock_request_handler(ip=None, x_forwarded_for=None, x_real_ip=None, saved_to handler_mock.application = mock_object() handler_mock.application.auth = TornadoAuth(None) handler_mock.application.identification = IpBasedIdentification(TrustedIpValidator(['127.0.0.1']), user_header_name) + handler_mock.application.server_config = mock_object() + handler_mock.application.server_config.cookie_secure = False handler_mock.request = mock_object() handler_mock.request.headers = {} @@ -38,7 +40,7 @@ def get_secure_cookie(name): return values.encode('utf8') return None - def set_secure_cookie(key, value, expires_days=30): + def set_secure_cookie(key, value, expires_days=30, **kwargs): cookies[key] = value def clear_cookie(key): diff --git a/src/tests/model_helper_test.py b/src/tests/model_helper_test.py index 94a22ce7..9def5ce2 100644 --- a/src/tests/model_helper_test.py +++ b/src/tests/model_helper_test.py @@ -552,7 +552,7 @@ def test_default_value_when_empty_string(self): class TestReadStrFromConfig(unittest.TestCase): def test_normal_text(self): value = read_str_from_config({'key1': 'xyz'}, 'key1') - self.assertEquals('xyz', value) + self.assertEqual('xyz', value) def test_none_value_no_default(self): value = read_str_from_config({'key1': None}, 'key1') @@ -560,7 +560,7 @@ def test_none_value_no_default(self): def test_none_value_with_default(self): value = read_str_from_config({'key1': None}, 'key1', default='abc') - self.assertEquals('abc', value) + self.assertEqual('abc', value) def test_no_key_no_default(self): value = read_str_from_config({'key1': 'xyz'}, 'key2') @@ -568,11 +568,11 @@ def test_no_key_no_default(self): def test_no_key_with_default(self): value = read_str_from_config({'key1': 'xyz'}, 'key2', default='abc') - self.assertEquals('abc', value) + self.assertEqual('abc', value) def test_text_with_whitespaces(self): value = read_str_from_config({'key1': ' xyz \n'}, 'key1') - self.assertEquals(' xyz \n', value) + self.assertEqual(' xyz \n', value) def test_text_when_blank_to_none_and_none(self): value = read_str_from_config({'key1': None}, 'key1', blank_to_none=True) @@ -588,7 +588,7 @@ def test_text_when_blank_to_none_and_blank(self): def test_text_when_blank_to_none_and_blank_and_default(self): value = read_str_from_config({'key1': ' \t \n'}, 'key1', blank_to_none=True, default='abc') - self.assertEquals('abc', value) + self.assertEqual('abc', value) def test_text_when_int(self): self.assertRaisesRegex(InvalidValueTypeException, 'Invalid key1 value: string expected, but was: 5', diff --git a/src/tests/observable_test.py b/src/tests/observable_test.py index 3c5a8897..9f1a5367 100644 --- a/src/tests/observable_test.py +++ b/src/tests/observable_test.py @@ -745,7 +745,14 @@ def test_time_buffer_read_until_closed(self): def test_time_buffer_aggregated_read_until_closed(self): observable = self.create_observable() - buffered_observable = observable.time_buffered(100, lambda chunks: ['|'.join(chunks)]) + # Use 30ms period (not 100ms) to avoid a timing race: the helper + # _test_read_until_closed sends late messages after 100ms sleep. + # With period=100ms both the flush thread and the async thread fire at + # ~t=100ms; the flush thread can close the buffered observable before + # read_until_closed subscribes (at t≈140ms), resulting in empty data. + # With period=30ms flush cycles land at t=30/60/90/120ms; the subscriber + # is set up at t≈49ms and the late-message flush happens at t≈120ms. + buffered_observable = observable.time_buffered(30, lambda chunks: ['|'.join(chunks)]) data, _, late_messages = self._test_read_until_closed( observable, diff --git a/src/tests/parameter_config_test.py b/src/tests/parameter_config_test.py index 0089346d..9900fcb0 100644 --- a/src/tests/parameter_config_test.py +++ b/src/tests/parameter_config_test.py @@ -230,6 +230,68 @@ def test_map_to_script_args_multiselect_single_arg(self): multiselect_separator='_') self.assertEqual('abc_456_def', parameter_model.to_script_args(['abc', '456', 'def'])) + def test_map_to_script_date_default_format(self): + parameter_model = create_parameter_model('param1', type='date') + self.assertEqual('2024-03-15', parameter_model.map_to_script('2024-03-15')) + + def test_map_to_script_date_custom_format(self): + parameter_model = create_parameter_model('param1', type='date', date_format='%d/%m/%Y') + self.assertEqual('15/03/2024', parameter_model.map_to_script('2024-03-15')) + + def test_map_to_script_date_compact_format(self): + parameter_model = create_parameter_model('param1', type='date', date_format='%Y%m%d') + self.assertEqual('20240315', parameter_model.map_to_script('2024-03-15')) + + def test_map_to_script_time_default_format(self): + parameter_model = create_parameter_model('param1', type='time') + self.assertEqual('14:30', parameter_model.map_to_script('14:30')) + + def test_map_to_script_time_custom_format(self): + parameter_model = create_parameter_model('param1', type='time', time_format='%H%M') + self.assertEqual('1430', parameter_model.map_to_script('14:30')) + + def test_map_to_script_time_with_seconds_format(self): + parameter_model = create_parameter_model('param1', type='time', time_format='%H:%M:%S') + self.assertEqual('14:30:00', parameter_model.map_to_script('14:30')) + + # --- date_format / time_format config validation --- + + def test_date_format_valid_custom(self): + # Should not raise — contains % directives + create_parameter_model('param1', type='date', date_format='%d/%m/%Y') + + def test_date_format_invalid_no_directives(self): + self.assertRaisesRegex( + Exception, + 'invalid date_format.*DD/MM/YYYY.*strftime directive', + create_parameter_model, 'param1', type='date', date_format='DD/MM/YYYY') + + def test_date_format_invalid_java_style(self): + self.assertRaisesRegex( + Exception, + 'invalid date_format', + create_parameter_model, 'param1', type='date', date_format='yyyy-MM-dd') + + def test_date_format_validation_only_for_date_type(self): + # Same string on a text parameter should never raise + create_parameter_model('param1', type='text', date_format='DD/MM/YYYY') + + def test_time_format_valid_custom(self): + # Should not raise — contains % directives + create_parameter_model('param1', type='time', time_format='%I:%M %p') + + def test_time_format_invalid_no_directives(self): + self.assertRaisesRegex( + Exception, + 'invalid time_format.*HH:MM.*strftime directive', + create_parameter_model, 'param1', type='time', time_format='HH:MM') + + def test_time_format_validation_only_for_time_type(self): + # Same string on a text parameter should never raise + create_parameter_model('param1', type='text', time_format='HH:MM') + + # --- end format validation tests --- + class TestDefaultValue(unittest.TestCase): @@ -678,6 +740,66 @@ def test_required_int_parameter_when_zero(self): error = validate_value(parameter, 0) self.assertIsNone(error) + def test_date_parameter_when_valid(self): + parameter = create_parameter_model('param', type='date') + + error = validate_value(parameter, '2024-03-15') + self.assertIsNone(error) + + def test_date_parameter_when_invalid_format(self): + parameter = create_parameter_model('param', type='date') + + error = validate_value(parameter, '15/03/2024') + self.assert_error(error) + + def test_date_parameter_when_not_a_date(self): + parameter = create_parameter_model('param', type='date') + + error = validate_value(parameter, 'hello') + self.assert_error(error) + + def test_date_parameter_when_invalid_day(self): + parameter = create_parameter_model('param', type='date') + + error = validate_value(parameter, '2024-02-30') + self.assert_error(error) + + def test_time_parameter_when_valid(self): + parameter = create_parameter_model('param', type='time') + + error = validate_value(parameter, '14:30') + self.assertIsNone(error) + + def test_time_parameter_when_midnight(self): + parameter = create_parameter_model('param', type='time') + + error = validate_value(parameter, '00:00') + self.assertIsNone(error) + + def test_time_parameter_when_end_of_day(self): + parameter = create_parameter_model('param', type='time') + + error = validate_value(parameter, '23:59') + self.assertIsNone(error) + + def test_time_parameter_when_invalid_format(self): + parameter = create_parameter_model('param', type='time') + + error = validate_value(parameter, '14:30:00') + self.assert_error(error) + + def test_time_parameter_when_not_a_time(self): + parameter = create_parameter_model('param', type='time') + + error = validate_value(parameter, 'hello') + self.assert_error(error) + + def test_time_parameter_when_invalid_hour(self): + parameter = create_parameter_model('param', type='time') + + error = validate_value(parameter, '25:00') + self.assert_error(error) + def test_file_upload_parameter_when_valid(self): parameter = create_parameter_model('param', type='file_upload') @@ -775,11 +897,11 @@ def test_list_with_script_when_matches_and_win_newline(self): self.assertIsNone(error) @parameterized.expand([ - ('a\d', 'ab', 'some desc', 'some desc'), - ('a\d', '12', 'desc 2', 'desc 2'), - ('a\d', 'a12', 'some long description', 'some long description'), - ('\d+\wa+', 'aaaa', 'some desc', 'some desc'), - ('\d+\wa+', 'aaaa', None, '\d+\wa+'), + (r'a\d', 'ab', 'some desc', 'some desc'), + (r'a\d', '12', 'desc 2', 'desc 2'), + (r'a\d', 'a12', 'some long description', 'some long description'), + (r'\d+\wa+', 'aaaa', 'some desc', 'some desc'), + (r'\d+\wa+', 'aaaa', None, r'\d+\wa+'), ]) def test_regex_validation_when_fail_with_description(self, regex, value, description, expected_description): parameter = create_parameter_model('param', regex={'pattern': regex, 'description': description}) @@ -789,10 +911,10 @@ def test_regex_validation_when_fail_with_description(self, regex, value, descrip self.assertEqual(error, "does not match regex pattern: " + expected_description) @parameterized.expand([ - ('a\d', 'a1',), - ('\da', '2a',), - ('a\d+', 'a12',), - ('\d+\wa+', '1Xaaaa'), + (r'a\d', 'a1',), + (r'\da', '2a',), + (r'a\d+', 'a12',), + (r'\d+\wa+', '1Xaaaa'), (None, '1Xaaaa'), ]) def test_regex_validation_when_success(self, regex, value): diff --git a/src/tests/parameter_server_file_test.py b/src/tests/parameter_server_file_test.py index c4b1173c..1081e9f3 100644 --- a/src/tests/parameter_server_file_test.py +++ b/src/tests/parameter_server_file_test.py @@ -126,12 +126,12 @@ def test_validate_failure_when_working_dir(self): create_files(['abc', 'def'], file_dir) working_dir_path = os.path.join(test_utils.temp_folder, 'work', 'dir') config = _create_parameter_model(recursive=False, file_dir=file_dir, working_dir=working_dir_path) - self.assertRegex(validate_value(config, 'def'), '.+ but should be in \[\]') + self.assertRegex(validate_value(config, 'def'), r'.+ but should be in \[\]') def test_validate_failure_when_excluded_file(self): create_files(['abc', 'def']) config = _create_parameter_model(recursive=False, file_dir=test_utils.temp_folder, excluded_files=['abc']) - self.assertRegex(validate_value(config, 'abc'), '.+ but should be in \[\'def\'\]') + self.assertRegex(validate_value(config, 'abc'), r".+ but should be in \['def'\]") def setUp(self): test_utils.setup() diff --git a/src/tests/process_utils_test.py b/src/tests/process_utils_test.py index 6b60e55b..74354083 100644 --- a/src/tests/process_utils_test.py +++ b/src/tests/process_utils_test.py @@ -28,9 +28,9 @@ def test_complex_command_linux(self): def test_complex_command_win(self): test_utils.set_win() - command_split = process_utils.split_command('"c:\program files\python\python.exe" test.py') + command_split = process_utils.split_command(r'"c:\program files\python\python.exe" test.py') - self.assertEqual(command_split, ['c:\program files\python\python.exe', 'test.py']) + self.assertEqual(command_split, [r'c:\program files\python\python.exe', 'test.py']) def test_unwrap_double_quotes_win(self): test_utils.set_win() diff --git a/src/tests/scheduling/schedule_service_test.py b/src/tests/scheduling/schedule_service_test.py index 4f944d2a..ef32a423 100644 --- a/src/tests/scheduling/schedule_service_test.py +++ b/src/tests/scheduling/schedule_service_test.py @@ -60,7 +60,11 @@ def setUp(self) -> None: scheduler._sleep = MagicMock() scheduler._sleep.side_effect = lambda x: time.sleep(0.001) - self.config_service = ConfigService(AnyUserAuthorizer(), test_utils.temp_folder, test_utils.process_invoker) + self.config_service = ConfigService( + AnyUserAuthorizer(), + test_utils.temp_folder, + True, + test_utils.process_invoker) self.create_config('my_script_A') self.create_config('unschedulable-script', scheduling_enabled=False) diff --git a/src/tests/script_config_test.py b/src/tests/script_config_test.py index 1acfa8c6..bf2b8424 100644 --- a/src/tests/script_config_test.py +++ b/src/tests/script_config_test.py @@ -6,7 +6,7 @@ from config.constants import PARAM_TYPE_SERVER_FILE, PARAM_TYPE_MULTISELECT from config.exceptions import InvalidConfigException -from model.script_config import ConfigModel, InvalidValueException, TemplateProperty, ParameterNotFoundException, \ +from model.script_config import InvalidValueException, TemplateProperty, ParameterNotFoundException, \ get_sorted_config from model.value_wrapper import ScriptValueWrapper from react.properties import ObservableDict, ObservableList @@ -259,10 +259,10 @@ def test_list_files_for_valid_param(self): param = create_script_param_config('recurs_file', type=PARAM_TYPE_SERVER_FILE, file_recursive=True, - file_dir=test_utils.temp_folder) + file_dir=self.subfolder) config_model = _create_config_model('my_conf', parameters=[param]) - create_files(['file1', 'file2']) + create_files(['file1', 'file2'], 'sub') file_names = [f['name'] for f in (config_model.list_files_for_param('recurs_file', []))] self.assertCountEqual(['file1', 'file2'], file_names) @@ -271,14 +271,14 @@ def test_list_files_when_working_dir(self): type=PARAM_TYPE_SERVER_FILE, file_recursive=True, file_dir='.') - config_model = _create_config_model('my_conf', parameters=[param], working_dir=test_utils.temp_folder) + config_model = _create_config_model('my_conf', parameters=[param], working_dir=self.subfolder) - create_files(['file1', 'file2']) + create_files(['file1', 'file2'], 'sub') file_names = [f['name'] for f in (config_model.list_files_for_param('recurs_file', []))] self.assertCountEqual(['file1', 'file2'], file_names) def test_list_files_when_unknown_param(self): - config_model = _create_config_model('my_conf', parameters=[], working_dir=test_utils.temp_folder) + config_model = _create_config_model('my_conf', parameters=[], working_dir=self.subfolder) self.assertRaises(ParameterNotFoundException, config_model.list_files_for_param, 'recurs_file', []) @@ -286,6 +286,9 @@ def setUp(self): super().setUp() test_utils.setup() + self.subfolder = os.path.join(test_utils.temp_folder, 'sub') + os.mkdir(self.subfolder) + def tearDown(self): super().tearDown() test_utils.cleanup() @@ -399,14 +402,17 @@ def test_get_required_parameters(self): class ConfigModelIncludeTest(unittest.TestCase): def test_static_include_simple(self): included_path = test_utils.write_script_config({'script_path': 'ping google.com'}, 'included') + included_path = file_utils.relative_path(included_path, test_utils.temp_folder) config_model = _create_config_model('main_conf', script_path=None, config={'include': included_path}) self.assertEqual('ping google.com', config_model.script_command) def test_static_include_multiple_inclusions(self): included_path_1 = test_utils.write_script_config({'script_path': 'ping google.com'}, 'included1') + included_path_1 = file_utils.relative_path(included_path_1, test_utils.temp_folder) included_path_2 = test_utils.write_script_config( {'script_path': 'echo 123', 'working_directory': '123'}, 'included2') + included_path_2 = file_utils.relative_path(included_path_2, test_utils.temp_folder) config_model = _create_config_model( 'main_conf', script_path=None, @@ -419,6 +425,7 @@ def test_static_include_precedence(self): 'script_path': 'ping google.com', 'working_directory': '123'}, 'included') + included_path = file_utils.relative_path(included_path, test_utils.temp_folder) config_model = _create_config_model('main_conf', config={ 'include': included_path, 'working_directory': 'abc'}) @@ -429,6 +436,7 @@ def test_static_include_single_parameter(self): included_path = test_utils.write_script_config({'parameters': [ create_script_param_config('param2', type='int') ]}, 'included1') + included_path = file_utils.relative_path(included_path, test_utils.temp_folder) config_model = _create_config_model('main_conf', config={ 'include': included_path, 'parameters': [create_script_param_config('param1', type='text')]}) @@ -448,12 +456,14 @@ def test_static_include_multiple_parameters_from_multiple_included(self): create_script_param_config('param2', type='int'), create_script_param_config('param3'), ]}, 'included1') + included_path_1 = file_utils.relative_path(included_path_1, test_utils.temp_folder) included_path_2 = test_utils.write_script_config({ 'parameters': [ create_script_param_config('param2', type='ip4'), create_script_param_config('param4'), create_script_param_config(None), ]}, 'included2') + included_path_2 = file_utils.relative_path(included_path_2, test_utils.temp_folder) config_model = _create_config_model('main_conf', config={ 'include': [included_path_1, included_path_2], @@ -482,6 +492,7 @@ def test_static_include_hidden_config(self): 'script_path': 'ping google.com', 'hidden': True}, 'included') + included_path = file_utils.relative_path(included_path, test_utils.temp_folder) config_model = _create_config_model('main_conf', script_path=None, config={'include': included_path}) self.assertEqual('ping google.com', config_model.script_command) @@ -546,6 +557,7 @@ def test_dynamic_include_relative_path(self): included_path = test_utils.write_script_config({'parameters': [ create_script_param_config('included_param') ]}, 'included', folder) + included_path = file_utils.relative_path(included_path, test_utils.temp_folder) included_folder = os.path.dirname(included_path) config_model = _create_config_model( 'main_conf', @@ -554,7 +566,7 @@ def test_dynamic_include_relative_path(self): 'include': '${p1}', 'working_directory': included_folder, 'parameters': [create_script_param_config('p1')]}) - config_model.set_param_value('p1', 'included.json') + config_model.set_param_value('p1', included_path) self.assertEqual(2, len(config_model.parameters)) @@ -566,6 +578,7 @@ def test_dynamic_include_replace(self): included_path2 = test_utils.write_script_config({'parameters': [ create_script_param_config('included_param_Y') ]}, 'included2') + included_path2 = file_utils.relative_path(included_path2, test_utils.temp_folder) config_model.set_param_value('p1', included_path2) @@ -588,6 +601,7 @@ def test_set_all_values_for_included(self): create_script_param_config('included_param1'), create_script_param_config('included_param2') ]}, 'included') + included_path = file_utils.relative_path(included_path, test_utils.temp_folder) config_model = _create_config_model( 'main_conf', config={ @@ -604,6 +618,7 @@ def test_set_all_values_for_dependant_on_constant(self): included_path = test_utils.write_script_config({'parameters': [ create_script_param_config('included_param1', values_script='echo ${p1}'), ]}, 'included') + included_path = file_utils.relative_path(included_path, test_utils.temp_folder) config_model = _create_config_model( 'main_conf', config={ @@ -652,19 +667,21 @@ def test_dynamic_include_add_2_parameters_with_default_when_one_dependant(self): @parameterized.expand([ (2, 'test desc', [('param3', 'int'), ('param4', 'text')]), (3, None, [('param3', 'int')]), - (None, None, []), + (None, None, [('param3', 'int')]), ]) def test_dynamic_include_when_multiple_includes(self, param2_value, expected_description, additional_parameters): included_path_1 = test_utils.write_script_config({ 'parameters': [ create_script_param_config('param3', type='int'), ]}, 'included1') + included_path_1 = file_utils.relative_path(included_path_1, test_utils.temp_folder) included_path_2 = test_utils.write_script_config({ 'description': 'test desc', 'parameters': [ create_script_param_config('param3', type='ip4'), create_script_param_config('param4') ]}, 'included2') + included_path_2 = file_utils.relative_path(included_path_2, test_utils.temp_folder) config_model = _create_config_model('main_conf', config={ 'include': ['${param1}', included_path_2[:-6] + '${param2}.json'], @@ -686,6 +703,7 @@ def test_dynamic_include_when_multiple_includes(self, param2_value, expected_des def prepare_config_model_with_included(self, included_params, static_param_name): included_path = test_utils.write_script_config({'parameters': included_params}, 'included') + included_path = file_utils.relative_path(included_path, test_utils.temp_folder) config_model = _create_config_model('main_conf', config={ 'include': '${' + static_param_name + '}', 'parameters': [create_script_param_config(static_param_name)]}) @@ -1079,6 +1097,7 @@ def test_create_with_schedulable_true_and_included_secure_parameter(self): another_path = test_utils.write_script_config( {'parameters': [{'name': 'p2', 'secure': True}]}, 'another_config') + another_path = file_utils.relative_path(another_path, test_utils.temp_folder) self.assertTrue(config_model.schedulable) @@ -1177,28 +1196,31 @@ def _create_config_model(name, *, parameters=None, parameter_values=None, working_dir=None, - script_path='echo 123', + script_path='DEFAULT', skip_invalid_parameters=False): result_config = {} - if script_path is not None: - result_config['script_path'] = script_path - if config: result_config.update(config) - result_config['name'] = name - - if parameters is not None: - result_config['parameters'] = parameters - - if path is None: - path = name - if working_dir is not None: result_config['working_directory'] = working_dir - model = ConfigModel(result_config, path, username, audit_name, test_utils.process_invoker) + if script_path == 'DEFAULT': + if config and 'script_path' in config: + script_path = None + else: + script_path = 'echo 123' + + model = test_utils.create_config_model( + name, + script_command=script_path, + config=result_config, + username=username, + audit_name=audit_name, + path=path, + parameters=parameters) + if parameter_values is not None: model.set_all_param_values(parameter_values, skip_invalid_parameters=skip_invalid_parameters) diff --git a/src/tests/server_conf_test.py b/src/tests/server_conf_test.py index 5e63f389..dc227865 100644 --- a/src/tests/server_conf_test.py +++ b/src/tests/server_conf_test.py @@ -4,6 +4,7 @@ from parameterized import parameterized +from auth.auth_azure_ad_oauth import AzureAdOAuthAuthenticator from auth.auth_gitlab import GitlabOAuthAuthenticator from auth.auth_google_oauth import GoogleOauthAuthenticator from auth.auth_htpasswd import HtpasswdAuthenticator @@ -255,8 +256,8 @@ def test_google_oauth(self): 'allowed_users': [] }}) self.assertIsInstance(config.authenticator, GoogleOauthAuthenticator) - self.assertEquals('1234', config.authenticator.client_id) - self.assertEquals('abcd', config.authenticator.secret) + self.assertEqual('1234', config.authenticator.client_id) + self.assertEqual('abcd', config.authenticator.secret) def test_google_oauth_without_allowed_users(self): with self.assertRaisesRegex(Exception, 'access.allowed_users field is mandatory for google_oauth'): @@ -264,6 +265,18 @@ def test_google_oauth_without_allowed_users(self): 'client_id': '1234', 'secret': 'abcd'}}) + def test_azure_ad_oauth(self): + config = _from_json({'auth': {'type': 'azure_ad_oauth', + 'auth_url': 'https://test.com/authorize', + 'token_url': 'https://test.com/token', + 'client_id': '1234', + 'secret': 'abcd'}}) + self.assertIsInstance(config.authenticator, AzureAdOAuthAuthenticator) + self.assertEqual('https://test.com/authorize', config.authenticator.auth_url) + self.assertEqual('https://test.com/token', config.authenticator.token_url) + self.assertEqual('1234', config.authenticator.client_id) + self.assertEqual('abcd', config.authenticator.secret) + def test_gitlab_oauth(self): config = _from_json({ 'auth': { @@ -280,22 +293,29 @@ def test_gitlab_oauth(self): def test_ldap(self): config = _from_json({'auth': {'type': 'ldap', 'url': 'http://test-ldap.net', - 'username_pattern': '|$username|', + 'ldap_user_resolver': { + 'username_pattern': '|$username|' + }, 'base_dn': 'dc=test', 'version': 3}}) self.assertIsInstance(config.authenticator, LdapAuthenticator) - self.assertEquals('http://test-ldap.net', config.authenticator.url) - self.assertEquals('|xyz|', config.authenticator.username_template.substitute(username='xyz')) - self.assertEquals('dc=test', config.authenticator._base_dn) - self.assertEquals(3, config.authenticator.version) + ldap_connector = config.authenticator._ldap_connector + self.assertEqual('|xyz|', config.authenticator._ldap_user_resolver.username_template.substitute(username='xyz')) + self.assertEqual('dc=test', config.authenticator._base_dn) + self.assertEqual('http://test-ldap.net', ldap_connector.url) + self.assertEqual(3, ldap_connector.version) def test_ldap_multiple_urls(self): config = _from_json({'auth': {'type': 'ldap', 'url': ['http://test-ldap-1.net', 'http://test-ldap-2.net'], - 'username_pattern': '|$username|'}}) + 'ldap_user_resolver': { + 'username_pattern': '|$username|' + }}}) self.assertIsInstance(config.authenticator, LdapAuthenticator) - self.assertEquals(['http://test-ldap-1.net', 'http://test-ldap-2.net'], config.authenticator.url) - self.assertEquals('|xyz|', config.authenticator.username_template.substitute(username='xyz')) + self.assertEqual(['http://test-ldap-1.net', 'http://test-ldap-2.net'], + config.authenticator._ldap_connector.url) + self.assertEqual('|xyz|', + config.authenticator._ldap_user_resolver.username_template.substitute(username='xyz')) def test_htpasswd_auth(self): file = test_utils.create_file('some-path', text='user1:1yL79Q78yczsM') @@ -319,7 +339,7 @@ class TestSecurityConfig(unittest.TestCase): def test_default_config(self): config = _from_json({}) - self.assertEquals('token', config.xsrf_protection) + self.assertEqual('token', config.xsrf_protection) @parameterized.expand([ ('token',), @@ -331,7 +351,7 @@ def test_xsrf_protection(self, xsrf_protection): 'xsrf_protection': xsrf_protection }}) - self.assertEquals(xsrf_protection, config.xsrf_protection) + self.assertEqual(xsrf_protection, config.xsrf_protection) def test_xsrf_protection_when_unsupported(self): self.assertRaises(InvalidValueException, _from_json, {'security': { @@ -362,18 +382,18 @@ def tearDown(self): def test_default_config(self): config = _from_json({}) env_vars = config.env_vars.build_env_vars() - self.assertEquals(env_vars, os.environ) + self.assertEqual(env_vars, os.environ) def test_config_when_safe_env_variables_used(self): config = _from_json({'title': '$$VAR1', 'auth': {'type': 'ldap', 'url': '$$MY_SECRET'}}) env_vars = config.env_vars.build_env_vars() - self.assertEquals(env_vars, os.environ) + self.assertEqual(env_vars, os.environ) self.assertEqual('abcd', env_vars['VAR1']) self.assertEqual('qwerty', env_vars['MY_SECRET']) - self.assertEquals(config.title, '$$VAR1') - self.assertEquals(config.authenticator.url, '$$MY_SECRET') + self.assertEqual(config.title, '$$VAR1') + self.assertEqual(config.authenticator._ldap_connector.url, '$$MY_SECRET') def test_config_when_unsafe_env_variables_used(self): config = _from_json({ @@ -397,18 +417,18 @@ def test_config_when_unsafe_env_variables_used(self): self.assertNotIn('EMAIL_PWD', env_vars) self.assertNotIn('EMAIL_PWD_2', env_vars) - self.assertEquals(config.title, '$$VAR1') - self.assertEquals(config.authenticator.secret, 'qwerty') + self.assertEqual(config.title, '$$VAR1') + self.assertEqual(config.authenticator.secret, 'qwerty') alert_destinations = AlertsService(config.alerts_config)._communication_service._destinations - self.assertEquals(alert_destinations[0]._communicator.password, '1234509') - self.assertEquals(alert_destinations[1]._communicator.password, '$VAR2') + self.assertEqual(alert_destinations[0]._communicator.password, '1234509') + self.assertEqual(alert_destinations[1]._communicator.password, '$VAR2') # noinspection PyTypeChecker callback_feature = ExecutionsCallbackFeature(None, config.callbacks_config, None) callback_destinations = callback_feature._communication_service._destinations - self.assertEquals(callback_destinations[0]._communicator.password, '007') - self.assertEquals(callback_destinations[1]._communicator.password, 'VAR1') + self.assertEqual(callback_destinations[0]._communicator.password, '007') + self.assertEqual(callback_destinations[1]._communicator.password, 'VAR1') def create_email_destination(self, password): return {'type': 'email', diff --git a/src/tests/test_utils.py b/src/tests/test_utils.py index 45c6ca15..1568da5a 100644 --- a/src/tests/test_utils.py +++ b/src/tests/test_utils.py @@ -121,10 +121,15 @@ def mock_object(): def write_script_config(conf_object, filename, config_folder=None): if config_folder is None: config_folder = os.path.join(temp_folder, 'runners') - file_path = os.path.join(config_folder, filename + '.json') + + if not filename.endswith('.json'): + filename = filename + '.json' + + file_path = os.path.join(config_folder, filename) config_json = json.dumps(conf_object) file_utils.write_file(file_path, config_json) + return file_path @@ -158,7 +163,9 @@ def create_script_param_config( stdin_expected_text=None, ui_separator_type=None, ui_separator_title=None, - values_ui_mapping=None): + values_ui_mapping=None, + date_format=None, + time_format=None): method_params = dict(locals()) conf = {'name': param_name} @@ -186,6 +193,8 @@ def create_script_param_config( 'pass_as': 'pass_as', 'stdin_expected_text': 'stdin_expected_text', 'values_ui_mapping': 'values_ui_mapping', + 'date_format': 'date_format', + 'time_format': 'time_format', } if values_script is not None: @@ -222,7 +231,7 @@ def create_config_model(name, *, script_command='ls', output_files=None, requires_terminal=None, - schedulable=True, + schedulable=None, logging_config: LoggingConfig = None, output_format=None): result_config = {} @@ -236,7 +245,9 @@ def create_config_model(name, *, result_config['parameters'] = parameters if path is None: - path = name + path = create_file(name + '.json', text='{}', overwrite=True) + elif not os.path.exists(path): + path = create_file(path, text='{}', overwrite=True) if output_files is not None: result_config['output_files'] = output_files @@ -245,7 +256,10 @@ def create_config_model(name, *, result_config['requires_terminal'] = requires_terminal if schedulable is not None: - result_config['scheduling'] = {'enabled': schedulable} + if 'scheduling' in result_config: + result_config['scheduling']['enabled'] = schedulable + else: + result_config['scheduling'] = {'enabled': schedulable} if output_format: result_config['output_format'] = output_format @@ -255,9 +269,17 @@ def create_config_model(name, *, 'execution_file': logging_config.filename_pattern, 'execution_date_format': logging_config.date_format} - result_config['script_path'] = script_command + if script_command: + result_config['script_path'] = script_command - model = ConfigModel(result_config, path, username, audit_name, process_invoker) + model = ConfigModel( + result_config, + path, + username, + audit_name, + True, + temp_folder, + process_invoker) if parameter_values is not None: model.set_all_param_values(parameter_values) @@ -293,7 +315,9 @@ def create_parameter_model(name=None, stdin_expected_text=None, ui_separator_type=None, ui_separator_title=None, - values_ui_mapping=None): + values_ui_mapping=None, + date_format=None, + time_format=None): config = create_script_param_config( name, type=type, @@ -319,7 +343,9 @@ def create_parameter_model(name=None, stdin_expected_text=stdin_expected_text, ui_separator_type=ui_separator_type, ui_separator_title=ui_separator_title, - values_ui_mapping=values_ui_mapping) + values_ui_mapping=values_ui_mapping, + date_format=date_format, + time_format=time_format) if all_parameters is None: all_parameters = [] @@ -472,7 +498,7 @@ def get_argument(arg_name, default=None): return default return arguments.get(arg_name) - def set_secure_cookie(cookie_name, value): + def set_secure_cookie(cookie_name, value, **kwargs): cookies[cookie_name] = f'!SECURE!{value}!!!' def clear_cookie(cookie_name): @@ -488,10 +514,17 @@ def get_secure_cookie(cookie_name): return value[8:-3].encode('utf8') + server_config = mock_object() + server_config.cookie_secure = False + + application = mock_object() + application.server_config = server_config + request_handler.get_argument = get_argument request_handler.set_secure_cookie = set_secure_cookie request_handler.get_secure_cookie = get_secure_cookie request_handler.clear_cookie = clear_cookie + request_handler.application = application request_handler.request = mock_object() request_handler.request.method = method diff --git a/src/tests/web/script_config_socket_test.py b/src/tests/web/script_config_socket_test.py index 74a4be46..176429d8 100644 --- a/src/tests/web/script_config_socket_test.py +++ b/src/tests/web/script_config_socket_test.py @@ -234,7 +234,7 @@ def setUp(self): application.authorizer = Authorizer(ANY_USER, [], [], [], EmptyGroupProvider()) application.identification = IpBasedIdentification(TrustedIpValidator(['127.0.0.1']), None) application.config_service = ConfigService( - application.authorizer, test_utils.temp_folder, test_utils.process_invoker) + application.authorizer, test_utils.temp_folder, True, test_utils.process_invoker) server = httpserver.HTTPServer(application) socket, self.port = testing.bind_unused_port() diff --git a/src/tests/web/server_test.py b/src/tests/web/server_test.py index 7d19a3e8..5fd20c08 100644 --- a/src/tests/web/server_test.py +++ b/src/tests/web/server_test.py @@ -1,8 +1,8 @@ +import asyncio import json import os import threading import traceback -from asyncio import set_event_loop_policy from unittest import TestCase from unittest.mock import patch, MagicMock @@ -122,6 +122,23 @@ def test_xsrf_protection_when_token(self): self.assertEqual(start_response.status_code, 200) self.assertEqual(start_response.content, b'3') + def test_xsrf_cookie_not_httponly(self): + # In token mode (the default) the browser JS must read the _xsrf cookie + # and echo it back in the X-XSRFToken header, so the cookie must NOT be + # HttpOnly. (The other XSRF tests use `requests`, which ignores HttpOnly, + # so they can't catch a regression here — this asserts the raw attribute.) + self.start_server(12345, '127.0.0.1') + + response = self._user_session.get('http://127.0.0.1:12345/scripts') + + xsrf_cookie = next((c for c in response.cookies if c.name == '_xsrf'), None) + self.assertIsNotNone(xsrf_cookie, 'server should set an _xsrf cookie in token mode') + + rest_attrs = {k.lower() for k in xsrf_cookie._rest.keys()} + self.assertNotIn('httponly', rest_attrs, + 'the _xsrf cookie must not be HttpOnly, otherwise token-mode ' + 'XSRF breaks (JS cannot read the token to send X-XSRFToken)') + def test_xsrf_protection_when_token_failed(self): self.start_server(12345, '127.0.0.1') @@ -136,6 +153,8 @@ def test_xsrf_protection_when_token_failed(self): cookies=response.cookies ) self.assertEqual(start_response.status_code, 403) + # The response should carry an actionable reason, not a bare "Forbidden". + self.assertIn('XSRF', start_response.text) def test_xsrf_protection_when_header(self): self.start_server(12345, '127.0.0.1', xsrf_protection=XSRF_PROTECTION_HEADER) @@ -231,6 +250,64 @@ def test_update_script_config(self): script_content = file_utils.read_file(script_path) self.assertEqual('abcdef', script_content) + # --- Security header tests --- + + def test_security_headers_on_api_response(self): + self.start_server(12345, '127.0.0.1') + response = self._user_session.get('http://127.0.0.1:12345/scripts') + self.assertEqual(200, response.status_code) + self._assert_security_headers(response) + + def test_security_headers_on_static_response(self): + self.start_server(12345, '127.0.0.1') + # Theme files are served by BaseStaticHandler and are accessible without + # authentication (they appear in the allowed_during_login list). + theme_dir = os.path.join(self.conf_folder, 'theme') + os.makedirs(theme_dir, exist_ok=True) + with open(os.path.join(theme_dir, 'style.css'), 'w') as f: + f.write('body { color: red; }') + + response = requests.get('http://127.0.0.1:12345/theme/style.css') + self.assertEqual(200, response.status_code) + self._assert_security_headers(response) + + def test_security_headers_on_websocket_response(self): + self.start_server(12345, '127.0.0.1') + # Plain HTTP GET to the WebSocket endpoint (no Upgrade headers) → Tornado returns 400. + # set_default_headers() is called before the handshake check, so security + # headers must be present in the error response. + response = requests.get('http://127.0.0.1:12345/executions/io/1') + self.assertEqual(400, response.status_code) + self._assert_security_headers(response) + + def test_hsts_present_when_cookie_secure(self): + self.start_server(12345, '127.0.0.1', cookie_secure=True) + response = self._user_session.get('http://127.0.0.1:12345/scripts') + hsts = response.headers.get('Strict-Transport-Security', '') + self.assertIn('max-age=31536000', hsts, 'HSTS header missing when cookie_secure=True') + + def test_hsts_absent_when_not_cookie_secure(self): + self.start_server(12345, '127.0.0.1', cookie_secure=False) + response = self._user_session.get('http://127.0.0.1:12345/scripts') + self.assertNotIn('Strict-Transport-Security', response.headers, + 'HSTS header must not be sent over plain HTTP') + + def _assert_security_headers(self, response): + for header, expected in [ + ('X-Frame-Options', 'DENY'), + ('X-Content-Type-Options', 'nosniff'), + ('Referrer-Policy', 'strict-origin-when-cross-origin'), + ('Permissions-Policy', 'camera=(), microphone=(), geolocation=()'), + ]: + self.assertEqual(expected, response.headers.get(header), + 'Wrong or missing header: ' + header) + + csp = response.headers.get('Content-Security-Policy', '') + self.assertIn("default-src 'self'", csp, 'CSP missing default-src') + self.assertIn("frame-ancestors 'none'", csp, 'CSP missing frame-ancestors') + + # --- end security header tests --- + def test_on_fly_auth(self): self.start_server(12345, '127.0.0.1') @@ -253,7 +330,7 @@ def test_get_scripts_when_basic_auth_failure(self): test_utils.write_script_config({'name': 's1'}, 's1', self.runners_folder) response = requests.get('http://127.0.0.1:12345/scripts', auth=HTTPBasicAuth('normal_user', 'wrong_pass')) - self.assertEquals(401, response.status_code) + self.assertEqual(401, response.status_code) @staticmethod def get_xsrf_token(session): @@ -272,12 +349,13 @@ def check_server_running(self): response = self._user_session.get('http://127.0.0.1:12345/conf') self.assertEqual(response.status_code, 200) - def start_server(self, port, address, *, xsrf_protection=XSRF_PROTECTION_TOKEN): + def start_server(self, port, address, *, xsrf_protection=XSRF_PROTECTION_TOKEN, cookie_secure=False): file_download_feature = FileDownloadFeature(UserFileStorage(b'some_secret'), test_utils.temp_folder) config = ServerConfig() config.port = port config.address = address config.xsrf_protection = xsrf_protection + config.cookie_secure = cookie_secure config.max_request_size_mb = 1 authorizer = Authorizer(ANY_USER, ['admin_user'], [], ['admin_user'], EmptyGroupProvider()) @@ -295,7 +373,7 @@ def start_server(self, port, address, *, xsrf_protection=XSRF_PROTECTION_TOKEN): execution_service, MagicMock(), MagicMock(), - ConfigService(authorizer, self.conf_folder, test_utils.process_invoker), + ConfigService(authorizer, self.conf_folder, True, test_utils.process_invoker), MagicMock(), FileUploadFeature(UserFileStorage(cookie_secret), test_utils.temp_folder), file_download_feature, @@ -352,4 +430,4 @@ def kill_ioloop(self, io_loop): self.ioloop_thread.join(timeout=50) io_loop.close() - set_event_loop_policy(None) + asyncio.set_event_loop(None) diff --git a/src/utils/file_utils.py b/src/utils/file_utils.py index c2b87ab4..683d19d1 100644 --- a/src/utils/file_utils.py +++ b/src/utils/file_utils.py @@ -33,7 +33,7 @@ def is_root(path): return os.path.dirname(path) == path -def normalize_path(path_string, current_folder=None): +def normalize_path(path_string, current_folder=None, follow_symlinks=True): path_string = os.path.expanduser(path_string) path_string = os.path.normpath(path_string) @@ -41,13 +41,16 @@ def normalize_path(path_string, current_folder=None): return path_string if current_folder: - normalized_folder = normalize_path(current_folder) + normalized_folder = normalize_path(current_folder, follow_symlinks=follow_symlinks) return os.path.join(normalized_folder, path_string) if not os.path.exists(path_string): return path_string - return str(pathlib.Path(path_string).resolve()) + if follow_symlinks: + return str(pathlib.Path(path_string).resolve()) + else: + return str(pathlib.Path(path_string).absolute()) def read_file(filename, byte_content=False, keep_newlines=False): @@ -143,17 +146,22 @@ def last_modification(folder_paths): def relative_path(path, parent_path): - path = normalize_path(path) - parent_path = normalize_path(parent_path) + def normalize(path, follow_symlinks=True): + path = normalize_path(path, follow_symlinks=follow_symlinks) + if os_utils.is_win(): + path = path.capitalize() + return path - if os_utils.is_win(): - path = path.capitalize() - parent_path = parent_path.capitalize() + normalized_path = normalize(path) + normalized_parent_path = normalize(parent_path) - if not path.startswith(parent_path): - raise ValueError(path + ' is not subpath of ' + parent_path) + if not normalized_path.startswith(normalized_parent_path): + normalized_path = normalize(path, follow_symlinks=False) + normalized_parent_path = normalize(parent_path, follow_symlinks=False) + if not normalized_path.startswith(normalized_parent_path): + raise ValueError(path + ' is not subpath of ' + parent_path) - relative_path = path[len(parent_path):] + relative_path = normalized_path[len(normalized_parent_path):] if relative_path.startswith(os.path.sep): return relative_path[1:] @@ -227,7 +235,7 @@ def _pre_3_5_recursive_glob(path_pattern, parent_path=None): if path_pattern.startswith('~'): path_pattern = os.path.expanduser(path_pattern) - file_name_regex = '([\w.-]|(\\\ ))*' + file_name_regex = r'([\w.-]|(\\\ ))*' pattern_chunks = path_pattern.split(os_utils.path_sep()) diff --git a/src/utils/tool_utils.py b/src/utils/tool_utils.py index 27f59077..d8abfcdf 100644 --- a/src/utils/tool_utils.py +++ b/src/utils/tool_utils.py @@ -14,7 +14,9 @@ def validate_web_build_exists(project_path): if not os.path.exists(web_folder): raise InvalidWebBuildException(web_folder + ' does not exist. \n' + how_to_fix_build_message) - required_files = ['index.html', 'admin.html', 'login.html', 'js', 'css', 'img'] + # Vite emits all JS/CSS/fonts/images into a single hashed `assets/` folder + # (the old Vue CLI build split them into js/, css/, img/). + required_files = ['index.html', 'admin.html', 'login.html', 'assets'] for file in required_files: file_path = os.path.join(web_folder, file) diff --git a/src/web/server.py b/src/web/server.py index 08c4389d..e338cfa1 100755 --- a/src/web/server.py +++ b/src/web/server.py @@ -85,9 +85,28 @@ def exception_to_code_and_message(exception): return None, None +def _set_security_headers(handler): + handler.set_header('X-Frame-Options', 'DENY') + handler.set_header('X-Content-Type-Options', 'nosniff') + handler.set_header('Referrer-Policy', 'strict-origin-when-cross-origin') + handler.set_header( + 'Content-Security-Policy', + "default-src 'self'; " + "style-src 'self' 'unsafe-inline'; " + "img-src 'self' data:; " + "font-src 'self' data:; " + "connect-src 'self' ws: wss:; " + "frame-ancestors 'none'; " + "object-src 'none'" + ) + handler.set_header('Permissions-Policy', 'camera=(), microphone=(), geolocation=()') + if handler.application.server_config.cookie_secure: + handler.set_header('Strict-Transport-Security', 'max-age=31536000; includeSubDomains') + + class BaseRequestHandler(tornado.web.RequestHandler): def set_default_headers(self): - self.set_header('X-Frame-Options', 'DENY') + _set_security_headers(self) if self.application.server_config.xsrf_protection == XSRF_PROTECTION_TOKEN: # This is needed to initialize cookie (by default tornado does it only on html template rendering) @@ -97,12 +116,28 @@ def set_default_headers(self): def check_xsrf_cookie(self): xsrf_protection = self.application.server_config.xsrf_protection if xsrf_protection == XSRF_PROTECTION_TOKEN: - return super().check_xsrf_cookie() + try: + return super().check_xsrf_cookie() + except tornado.web.HTTPError: + # Tornado's default reason is just "Forbidden", which is opaque in + # the UI. Surface an actionable reason instead (this becomes the + # response body via write_error). The usual cause is the browser not + # sending the token: the _xsrf cookie must be readable by JS (not + # HttpOnly) and, over plain HTTP, security.cookie_secure must be + # false so the cookie is stored at all. + raise tornado.web.HTTPError( + 403, + "XSRF token missing/invalid. The browser must read the _xsrf cookie " + "and send it as X-XSRFToken; over HTTP set security.cookie_secure=false.", + reason='XSRF token missing or invalid') elif xsrf_protection == XSRF_PROTECTION_HEADER: requested_with = self.request.headers.get('X-Requested-With') if not requested_with: - raise tornado.web.HTTPError(403, 'X-Requested-With header is missing for XSRF protection') + raise tornado.web.HTTPError( + 403, + 'X-Requested-With header is missing for XSRF protection', + reason='X-Requested-With header required') return def write_error(self, status_code, **kwargs): @@ -119,7 +154,7 @@ def write_error(self, status_code, **kwargs): class BaseStaticHandler(tornado.web.StaticFileHandler): def set_default_headers(self): - self.set_header('X-Frame-Options', 'DENY') + _set_security_headers(self) class GetServerConf(BaseRequestHandler): @@ -263,6 +298,9 @@ def __init__(self, application, request, **kwargs): self.executor = None + def set_default_headers(self): + _set_security_headers(self) + @check_authorization @inject_user def open(self, user, execution_id): @@ -864,6 +902,16 @@ def init(server_config: ServerConfig, 'websocket_ping_timeout': 300, 'compress_response': True, 'xsrf_cookies': server_config.xsrf_protection != XSRF_PROTECTION_DISABLED, + 'xsrf_cookie_kwargs': { + # The XSRF cookie is a double-submit CSRF token, not a secret: in + # token mode (the default) the browser JS must read it and echo it + # back in the X-XSRFToken header. It therefore must NOT be httponly, + # otherwise every POST (e.g. starting an execution) is rejected with + # 403 "_xsrf argument missing". + 'httponly': False, + 'secure': server_config.cookie_secure, + 'samesite': 'Lax' + }, } application = tornado.web.Application(handlers, **settings) @@ -886,7 +934,7 @@ def init(server_config: ServerConfig, application.max_request_size_mb = server_config.max_request_size_mb if os_utils.is_win() and env_utils.is_min_version('3.8'): - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + asyncio.set_event_loop(asyncio.SelectorEventLoop()) io_loop = tornado.ioloop.IOLoop.current() global _http_server diff --git a/tools/Dockerfile b/tools/Dockerfile index d1c8fd8a..7e5cecdc 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -1,8 +1,30 @@ -FROM python:3.9-slim +FROM python:3.13-slim + +# Create a non-root user +RUN useradd --create-home --shell /bin/bash scriptserver -COPY build/script-server /app WORKDIR /app -RUN pip install -r requirements.txt + +# Install Python dependencies first for better layer caching +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code (web/ contains pre-built frontend assets) +COPY src/ ./src/ +COPY web/ ./web/ +COPY conf/ ./conf/ +COPY launcher.py ./ + +# Ensure writable directories exist and belong to the app user +RUN mkdir -p conf/runners logs \ + && chown -R scriptserver:scriptserver /app + +USER scriptserver + +# conf/runners — mount your script configs here +# logs/ — execution logs written here +VOLUME ["/app/conf/runners", "/app/logs"] EXPOSE 5000 -CMD [ "python3", "launcher.py" ] \ No newline at end of file + +CMD ["python3", "launcher.py"] diff --git a/tools/deploy_docker.sh b/tools/deploy_docker.sh index 5ac6586e..c9bba19d 100755 --- a/tools/deploy_docker.sh +++ b/tools/deploy_docker.sh @@ -19,13 +19,17 @@ else DOCKER_TAG="$TRAVIS_BRANCH" fi -docker login -u "$DOCKER_USER" -p "$DOCKER_PASSWORD" +docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -docker build -f tools/Dockerfile -t "$IMAGE_NAME":"$DOCKER_TAG" . +docker login -u "$DOCKER_USER" -p "$DOCKER_PASSWORD" -echo "NEW_GIT_TAG=$NEW_GIT_TAG" +ADDITIONAL_TAG_ARG="" if [ ! -z "$NEW_GIT_TAG" ]; then - docker tag "$IMAGE_NAME":"$DOCKER_TAG" "$IMAGE_NAME":"$NEW_GIT_TAG" + ADDITIONAL_TAG_ARG="-t $IMAGE_NAME:$NEW_GIT_TAG" fi -docker push --all-tags "$IMAGE_NAME" +docker buildx create --use +docker buildx build --platform linux/amd64,linux/arm64 --push -f tools/Dockerfile \ + -t "$IMAGE_NAME":"$DOCKER_TAG" \ + $ADDITIONAL_TAG_ARG \ + . diff --git a/web-src/public/admin.html b/web-src/admin.html similarity index 66% rename from web-src/public/admin.html rename to web-src/admin.html index e002b582..b2a73cc0 100644 --- a/web-src/public/admin.html +++ b/web-src/admin.html @@ -5,13 +5,11 @@ - + - -