Revert "feat!: separate CIM data models into standalone splunk-cim-models package [BREAKING]" (#948)

Reverts #946

Author: mkolasinski-splunk

.github/workflows/build-test-release.yml

@@ -74,8 +74,6 @@ jobs:
           submodules: true
       - name: Install dependencies
         run: |
-          git config --global --add url."https://${{ secrets.GH_TOKEN_ADMIN }}@github.com".insteadOf https://github.com
-          git config --global --add url."https://${{ secrets.GH_TOKEN_ADMIN }}@github.com".insteadOf ssh://git@github.com
           curl -sSL https://install.python-poetry.org | python3 - --version 1.5.1
           poetry install
           poetry run pytest -v tests/unit
@@ -101,8 +99,6 @@ jobs:
         run: |
           mkdir test-results-${{ matrix.splunk.version }}
       - name: Test
-        env:
-          GH_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }}
         run: |
           export SPLUNK_APP_PACKAGE=./tests/e2e/addons/TA_fiction_indextime
           export SPLUNK_ADDON=TA_fiction_indextime
@@ -163,105 +159,14 @@ jobs:
         with:
           python-version: 3.7
       - run: |
-          git config --global --add url."https://${{ secrets.GH_TOKEN_ADMIN }}@github.com".insteadOf https://github.com
-          git config --global --add url."https://${{ secrets.GH_TOKEN_ADMIN }}@github.com".insteadOf ssh://git@github.com
           curl -sSL https://install.python-poetry.org | python3 - --version 1.5.1
           poetry install
           poetry run pytest -v --splunk-version=${{ matrix.splunk.version }} -m docker -m ${{ matrix.test-marker }} tests/e2e
-        env:
-          GH_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }}
-
-  # ---------------------------------------------------------------------------
-  # Validate PSA against multiple splunk-cim-models package versions.
-  #
-  # Matrix axes:
-  #   splunk          – every supported Splunk version (from meta job)
-  #   cim-models      – every package git ref listed below
-  #
-  # How to add a new CIM models release:
-  #   1. Append a new entry to the cim-models list with the git ref (branch
-  #      or tag) and a short human-readable label.
-  #   2. The matrix will automatically run the splunk_cim_model e2e test
-  #      for each (Splunk version, CIM models version) combination.
-  #
-  # Only the splunk_cim_model marker is used here because the other CIM
-  # e2e markers (splunk_app_cim_fiction, splunk_app_cim_broken) always pass
-  # --splunk-dm-path with custom fiction models and are therefore unaffected
-  # by the installed splunk-cim-models version.
-  # ---------------------------------------------------------------------------
-  test-cim-models-versions:
-    needs:
-      - meta
-      - pre-commit
-      - fossa-scan
-      - compliance-copyrights
-      - test-splunk-unit
-    runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: false
-      matrix:
-        splunk: ${{ fromJson(needs.meta.outputs.matrix_supportedSplunk) }}
-        cim-models:
-          - ref: "v1"
-            label: "1.x"
-          - ref: "v2"
-            label: "2.x"
-          # Add future releases here, e.g.:
-          # - ref: "v3"
-          #   label: "3.x"
-    name: "CIM models ${{ matrix.cim-models.label }} / Splunk ${{ matrix.splunk.version }}"
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.7"
-      - name: Install PSA with CIM models ${{ matrix.cim-models.label }}
-        env:
-          GH_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }}
-        run: |
-          git config --global --add url."https://${{ secrets.GH_TOKEN_ADMIN }}@github.com".insteadOf https://github.com
-          git config --global --add url."https://${{ secrets.GH_TOKEN_ADMIN }}@github.com".insteadOf ssh://git@github.com
-          curl -sSL https://install.python-poetry.org | python3 - --version 1.5.1
-          poetry install
-          # Override the dev-dependency CIM models with the matrix version.
-          # Using pip inside the Poetry venv so the pytester subprocess sees
-          # the same environment.
-          poetry run pip install \
-            "git+https://github.com/splunk/psa-cim-models.git@${{ matrix.cim-models.ref }}"
-      - name: Verify splunk-cim-models ${{ matrix.cim-models.label }}
-        run: |
-          poetry run python - <<'EOF'
-          import splunk_cim_models as scm
-          print("DATA_MODELS_PATH:", scm.DATA_MODELS_PATH)
-          print("CIM versions available:", list(scm.datamodels.keys()))
-          EOF
-      - name: Run CIM compatibility e2e tests
-        env:
-          GH_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }}
-        run: |
-          # splunk_app_cim_fiction uses --splunk-dm-path with local fiction
-          # data models, so expected test outcomes are identical across all
-          # splunk-cim-models versions.  It still exercises all PSA import
-          # paths (COMMON_FIELDS_PATH, DATAMODEL_SCHEMA_PATH, datamodels) so
-          # an incompatible package version will surface as a collection or
-          # runtime error rather than a wrong-field assertion.
-          #
-          # splunk_cim_model (real Change/Network_Traffic models) is left to
-          # test-splunk-matrix, which always uses the default v2 dev
-          # dependency and asserts version-specific field outcomes.
-          poetry run pytest -v \
-            --splunk-version=${{ matrix.splunk.version }} \
-            -m docker \
-            -m splunk_app_cim_fiction \
-            tests/e2e
 
   publish:
     needs:
       - test-splunk-external
       - test-splunk-matrix
-      - test-cim-models-versions
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4

.licenserc.yaml

@@ -38,4 +38,3 @@ header:
     - "pytest_splunk_addon/.ignore_splunk_internal_errors"
     - "pytest_splunk_addon/docker_class.py"
     - "deps"
-    - "splunk-cim-models/**"

docker-compose-ci.yml

@@ -32,8 +32,6 @@ services:
       - sc4s
     volumes:
       - results:/work/test-results
-    environment:
-      - GH_TOKEN
 
   sc4s:
     image: ghcr.io/splunk/splunk-connect-for-syslog/container2:latest

docs/cim_tests.md

@@ -2,42 +2,12 @@
 
 ## Overview
 
-The CIM tests are written with a purpose of testing the compatibility of the add-on with CIM Data Models.
+The CIM tests are written with a purpose of testing the compatibility of the add-on with CIM Data Models (Based on Splunk_SA_CIM 4.15.0).
 An add-on is said to be CIM compatible if it fulfils the following two criteria:
 
-1. The add-on extracts all the fields with valid values, which are marked as required by the [Data Model Definitions](https://github.com/splunk/psa-cim-models).
+1. The add-on extracts all the fields with valid values, which are marked as required by the [Data Model Definitions](https://github.com/splunk/pytest-splunk-addon/tree/main/pytest_splunk_addon/standard_lib/data_models).
 2. Any event for the add-on is not mapped with more than one data model.
 
-## CIM Data Models Package (`splunk-cim-models`)
-
-CIM data model definitions are provided by the separate
-[`splunk-cim-models`](https://github.com/splunk/psa-cim-models) package.
-This lets you update or pin CIM definitions independently of `pytest-splunk-addon` itself.
-
-**Install for CIM testing:**
-
-```console
-pip install splunk-cim-models
-```
-
-Or, during development, install the latest from the repository:
-
-```console
-pip install git+https://github.com/splunk/psa-cim-models.git@v2
-```
-
-The package exposes:
-
-| Symbol | Description |
-|---|---|
-| `DATA_MODELS_PATH` | Path to the directory containing the built-in CIM JSON data model files |
-| `COMMON_FIELDS_PATH` | Path to `CommonFields.json` (fields forbidden in props/search) |
-| `DATAMODEL_SCHEMA_PATH` | Path to `DatamodelSchema.json` (JSON schema for validating custom data model files) |
-| `datamodels` | Dict mapping CIM version strings to recommended fields per model |
-
-If `splunk-cim-models` is not installed, any test run that exercises CIM functionality
-will raise an `ImportError` with a message pointing to the package.
-
 ______________________________________________________________________
 
 To generate test cases only for CIM compatibility, append the following marker to pytest command:
@@ -60,7 +30,7 @@ To generate test cases only for CIM compatibility, append the following marker t
  **Workflow:**
 
  - Plugin parses tags.conf to get a list of tags for each eventtype.
- - Plugin parses all the [supported datamodels](https://github.com/splunk/psa-cim-models).
+ - Plugin parses all the [supported datamodels](https://github.com/splunk/pytest-splunk-addon/tree/main/pytest_splunk_addon/standard_lib/data_models).
  - Then it gets a list of the datasets mapped with an eventtype.
  - Generates test case for each eventtype.
 
@@ -109,11 +79,11 @@ To generate test cases only for CIM compatibility, append the following marker t
 
  **Workflow:**
 
- - Plugin collects the list of not_allowed_in_search fields from mapped datasets and [CommonFields.json](https://github.com/splunk/psa-cim-models/blob/v2/splunk_cim_models/CommonFields.json).
+ - Plugin collects the list of not_allowed_in_search fields from mapped datasets and [CommonFields.json](https://github.com/splunk/pytest-splunk-addon/blob/main/pytest_splunk_addon/standard_lib/cim_tests/CommonFields.json).
  - Using search query the test case verifies if not_allowed_in_search fields are populated in search or not.
 
 > **_NOTE:_** 
- [CommonFields.json](https://github.com/splunk/psa-cim-models/blob/v2/splunk_cim_models/CommonFields.json) contains fields which are automatically provided by asset and identity correlation features of applications like Splunk Enterprise Security.
+ [CommonFields.json](https://github.com/splunk/pytest-splunk-addon/blob/main/pytest_splunk_addon/standard_lib/cim_tests/CommonFields.json) contains fields which are automatically provided by asset and identity correlation features of applications like Splunk Enterprise Security.
 
 
 **4. Testcase for all not_allowed_in_props fields**

docs/contributing.md

@@ -18,42 +18,6 @@ To install currently checked out version of pytest-splunk-addon use:
 $ poetry install
 ```
 
-This installs `splunk-cim-models` automatically as a dev dependency (from the `v2` branch of
-[psa-cim-models](https://github.com/splunk/psa-cim-models)). In CI or when installing from
-PyPI, install `splunk-cim-models` separately before running CIM tests:
-
-```
-$ pip install splunk-cim-models
-```
-
-### Working with `splunk-cim-models`
-
-CIM data model definitions live in the separate
-[`splunk-cim-models`](https://github.com/splunk/psa-cim-models) package.
-PSA imports four symbols from it:
-
-| Symbol | Used by |
-|---|---|
-| `DATA_MODELS_PATH` | `app_test_generator.py` — default path for `--splunk_dm_path` |
-| `COMMON_FIELDS_PATH` | `cim_tests/test_generator.py` — fields forbidden in props/search |
-| `DATAMODEL_SCHEMA_PATH` | `cim_tests/json_schema.py` — JSON schema for custom data models |
-| `datamodels` | `splunk.py` — CIM version → recommended fields map |
-
-To update CIM definitions locally without waiting for a `splunk-cim-models` release:
-
-```bash
-# Edit files under splunk-cim-models/ then reinstall editable
-pip install -e path/to/psa-cim-models/
-
-# Verify
-python -c "from splunk_cim_models import datamodels; print(list(datamodels.keys()))"
-```
-
-After editing, run the unit tests to confirm nothing is broken:
-
-```bash
-poetry run pytest -v tests/unit
-```
 
 ### Unit tests
 

docs/how_to_use.md

@@ -460,12 +460,6 @@ def splunk_setup(splunk):
 
  How can this be achieved :
 
-  - Make json representation of the data models, which satisfies the [DatamodelSchema](https://github.com/splunk/psa-cim-models/blob/v2/splunk_cim_models/DatamodelSchema.json) provided by the `splunk-cim-models` package.
-  - Provide the path to the directory having all the data models by adding `--splunk_dm_path path_to_dir` to the pytest command.
-  - The test cases will now be generated for the data models provided to the plugin and not for the [default data models](https://github.com/splunk/psa-cim-models/tree/v2/splunk_cim_models/data_models) bundled in `splunk-cim-models`.
-
-> **_NOTE:_** CIM data model definitions are provided by the [`splunk-cim-models`](https://github.com/splunk/psa-cim-models) package. Install it separately before running CIM tests:
->
-> ```console
-> pip install splunk-cim-models
-> ```
+  - Make json representation of the data models, which satisfies this [DataModelSchema](https://github.com/splunk/pytest-splunk-addon/blob/main/pytest_splunk_addon/cim_tests/DatamodelSchema.json).
+  - Provide the path to the directory having all the data models by adding `--splunk_dm_path path_to_dir` to the pytest command
+  - The test cases will now be generated for the data models provided to the plugin and not for the [default data models](https://github.com/splunk/pytest-splunk-addon/tree/main/pytest_splunk_addon/data_models).

docs/index.md

@@ -8,7 +8,6 @@ which allows the user to test [knowledge objects](https://docs.splunk.com/Splexi
 - Splunk App or Add-on package
 - Splunk instance with App or Add-on installed (Not required if using Docker)
 - Docker (Not required if using external Splunk instance)
-- [`splunk-cim-models`](https://github.com/splunk/psa-cim-models) — required for [CIM Compatibility Tests](./cim_tests.md) (not installed automatically)
 
 ## Support
 
@@ -23,17 +22,6 @@ pytest-splunk-addon can be installed via pip from PyPI:
 pip3 install pytest-splunk-addon
 ```
 
-To run **CIM compatibility tests** you also need the CIM data models package:
-
-```console
-pip3 install splunk-cim-models
-```
-
-`splunk-cim-models` is versioned independently of `pytest-splunk-addon` so that CIM
-definitions can be updated without a full PSA release. See the
-[CIM Compatibility Tests](./cim_tests.md) page for details on how the two packages
-interact and how to pin or override data model definitions.
-
 ## Features
 
 The pytest-splunk-addon works by dynamically generating different types of tests for Splunk apps and add-ons by parsing their configuration files. Specifically, it looks at the .conf files in the provided Splunk app/add-on to create:

entrypoint.sh

@@ -9,10 +9,6 @@ pyenv local 3.7.8
 curl -sSL https://install.python-poetry.org | python - --version 1.5.1
 export PATH="/root/.local/bin:$PATH"
 source ~/.poetry/env
-if [ -n "$GH_TOKEN" ]; then
-  git config --global --add url."https://$GH_TOKEN@github.com".insteadOf https://github.com
-  git config --global --add url."https://$GH_TOKEN@github.com".insteadOf ssh://git@github.com
-fi
 sleep 15
 poetry install
 exec poetry run pytest -vv $@