From 2d4b22cac9ab0decc70b21041120ccc7ddf1c148 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Wed, 13 Mar 2024 18:00:30 -0700 Subject: [PATCH 1/6] Update skill.json parsing per https://openvoiceos.github.io/ovos-technical-manual/skill_json/ --- .github/workflows/skill_update_json_spec.yml | 9 ++-- .github/workflows/skill_update_meta_repo.yml | 5 +- scripts/requirements.txt | 2 +- scripts/update_skill_json.py | 51 +++++++++++++++++--- 4 files changed, 54 insertions(+), 13 deletions(-) diff --git a/.github/workflows/skill_update_json_spec.yml b/.github/workflows/skill_update_json_spec.yml index 082ac40..29b5483 100644 --- a/.github/workflows/skill_update_json_spec.yml +++ b/.github/workflows/skill_update_json_spec.yml @@ -11,12 +11,12 @@ jobs: runs-on: ${{inputs.runner}} timeout-minutes: 15 steps: - - name: Checkout Repository - uses: actions/checkout@v2 + - name: Checkout Skill Repository + uses: actions/checkout@v4 with: path: action/skill/ - name: Checkout Scripts Repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: path: action/github/ repository: NeonGeckoCom/.github @@ -29,8 +29,7 @@ jobs: sudo apt update sudo apt install -y gcc git libpulse-dev pip install --upgrade pip - pip install wheel "cython<3.0.0" # TODO: cython patching https://github.com/yaml/pyyaml/issues/724 - pip install --no-build-isolation pyyaml~=5.4 # TODO: patching https://github.com/yaml/pyyaml/issues/724 + pip install wheel numpy # numpy patching https://github.com/benfred/parsesetup/issues/3 pip install -r action/github/scripts/requirements.txt - name: Get Updated skill.json run: | diff --git a/.github/workflows/skill_update_meta_repo.yml b/.github/workflows/skill_update_meta_repo.yml index c94f47b..efab229 100644 --- a/.github/workflows/skill_update_meta_repo.yml +++ b/.github/workflows/skill_update_meta_repo.yml @@ -12,10 +12,13 @@ jobs: runs-on: ${{inputs.runner}} timeout-minutes: 5 steps: - - uses: actions/checkout@v2 + - name: Checkout Skill Repository + uses: actions/checkout@v4 - name: Update skill.json in neon_skills run: | + version=$(python3 setup.py --version) git clone https://github.com/neongeckocom/neon_skills -b ${{github.ref_name}} + cp skill.json neon_skills/skill_metadata/${{github.event.repository.name}}-${version}.json cp skill.json neon_skills/skill_metadata/${{github.event.repository.name}}.json - name: Push neon_skills changes uses: cpina/github-action-push-to-another-repository@main diff --git a/scripts/requirements.txt b/scripts/requirements.txt index a5110a0..bf0d201 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,4 +1,4 @@ neon-utils~=1.2 -ovos-skills-manager~=0.0 +parsesetup~=0.0.1 cattrs != 23.1.0 # TODO: cattrs patching https://github.com/python-attrs/cattrs/issues/369 \ No newline at end of file diff --git a/scripts/update_skill_json.py b/scripts/update_skill_json.py index baf4d1f..f1678a7 100644 --- a/scripts/update_skill_json.py +++ b/scripts/update_skill_json.py @@ -27,16 +27,53 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json +import shutil from sys import argv -from os.path import join +from os.path import join, isfile from pprint import pprint -from neon_utils.packaging_utils import build_skill_spec +from parsesetup import parse_setup +from neon_utils.file_utils import parse_skill_readme_file -def get_skill_json(skill_dir: str): - print(f"skill_dir={skill_dir}") - skill_json = join(skill_dir, "skill.json") +def _get_setup_py_data(setup_py: str) -> dict: + setup_meta = parse_setup(setup_py, trusted=True) + skill_id = list(setup_meta['entrypoints'].items())[0].split('=')[0] + return {"skill_id": skill_id, + "source": setup_meta.get("url"), + "package_name": setup_meta['name'], + "license": setup_meta.get("license"), + "author": setup_meta.get("author"), + "description": setup_meta.get("description")} + + +def build_skill_spec(skill_dir: str) -> dict: + readme_file = join(skill_dir, "README.md") + setup_py = join(skill_dir, "setup.py") + if isfile(setup_py): + skill_data = _get_setup_py_data(setup_py) + readme_data = parse_skill_readme_file(readme_file) + skill_data["description"] = readme_data.get("summary") or skill_data["description"] + skill_data["name"] = readme_data.get("title") + skill_data["examples"] = readme_data.get("examples") or [] + skill_data["tags"] = readme_data.get("categories", + []) + readme_data.get("tags", []) + skill_data["icon"] = readme_data.get("icon") + return skill_data + + +def write_skill_json(skill_dir: str, default_lang="en-us"): + + old_skill_json = join(skill_dir, "skill.json") + setup_py = join(skill_dir, "setup.py") + pkg_dir = list(parse_setup(setup_py, + trusted=True)['package_dir'].values())[0] + skill_json = join(skill_dir, pkg_dir, "locale", default_lang, "skill.json") + print(f"skill_dir={skill_dir}|skill.json={skill_json}") + + if isfile(old_skill_json): + shutil.move(old_skill_json, skill_json) + skill_spec = build_skill_spec(skill_dir) pprint(skill_spec) try: @@ -45,6 +82,8 @@ def get_skill_json(skill_dir: str): except Exception as e: print(e) current = None + if current: + skill_spec["extra_plugins"] = current.get("extra_plugins") or dict() if current != skill_spec: print("Skill Updated. Writing skill.json") with open(skill_json, 'w+') as f: @@ -54,4 +93,4 @@ def get_skill_json(skill_dir: str): if __name__ == "__main__": - get_skill_json(argv[1]) + write_skill_json(argv[1]) From 2ea47933a8eb616b7906a300a6628ab8ad305443 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Mon, 18 Mar 2024 16:27:31 -0700 Subject: [PATCH 2/6] Update to test changes --- .github/workflows/skill_update_json_spec.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/skill_update_json_spec.yml b/.github/workflows/skill_update_json_spec.yml index 29b5483..a69cb46 100644 --- a/.github/workflows/skill_update_json_spec.yml +++ b/.github/workflows/skill_update_json_spec.yml @@ -20,6 +20,8 @@ jobs: with: path: action/github/ repository: NeonGeckoCom/.github + ref: FEAT_UpdateSkillMetaParsing + # TODO: Remove ref before merge - name: Setup Python uses: actions/setup-python@v2 with: From e147f5784a83d734939c2930f504f9d49cb719e5 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Mon, 18 Mar 2024 16:34:49 -0700 Subject: [PATCH 3/6] Update shared actions to latest versions --- .github/workflows/skill_update_json_spec.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/skill_update_json_spec.yml b/.github/workflows/skill_update_json_spec.yml index a69cb46..ec0d874 100644 --- a/.github/workflows/skill_update_json_spec.yml +++ b/.github/workflows/skill_update_json_spec.yml @@ -23,7 +23,7 @@ jobs: ref: FEAT_UpdateSkillMetaParsing # TODO: Remove ref before merge - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install Dependencies @@ -37,7 +37,7 @@ jobs: run: | python action/github/scripts/update_skill_json.py action/skill - name: Push skill.json Change - uses: stefanzweifel/git-auto-commit-action@v4 + uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: Update skill.json repository: action/skill/ From 9f9e3567a18a00b63fe18cdf340d76c73365e8fa Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Mon, 18 Mar 2024 16:42:58 -0700 Subject: [PATCH 4/6] Troubleshoot actions failure https://github.com/NeonGeckoCom/skill-about/actions/runs/8335110313/job/22809964498?pr=82 --- .github/workflows/skill_update_json_spec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/skill_update_json_spec.yml b/.github/workflows/skill_update_json_spec.yml index ec0d874..55cdc60 100644 --- a/.github/workflows/skill_update_json_spec.yml +++ b/.github/workflows/skill_update_json_spec.yml @@ -35,7 +35,7 @@ jobs: pip install -r action/github/scripts/requirements.txt - name: Get Updated skill.json run: | - python action/github/scripts/update_skill_json.py action/skill + python3 action/github/scripts/update_skill_json.py action/skill - name: Push skill.json Change uses: stefanzweifel/git-auto-commit-action@v5 with: From 559e3dfefa8ac77b66b1dcce9cc47e11f8b8283f Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Mon, 18 Mar 2024 16:46:13 -0700 Subject: [PATCH 5/6] Troubleshoot relative path errors --- .github/workflows/skill_update_json_spec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/skill_update_json_spec.yml b/.github/workflows/skill_update_json_spec.yml index 55cdc60..8870c17 100644 --- a/.github/workflows/skill_update_json_spec.yml +++ b/.github/workflows/skill_update_json_spec.yml @@ -35,7 +35,7 @@ jobs: pip install -r action/github/scripts/requirements.txt - name: Get Updated skill.json run: | - python3 action/github/scripts/update_skill_json.py action/skill + python3 "${{ github.workspace }}/action/github/scripts/update_skill_json.py" "${{ github.workspace }}/action/skill" - name: Push skill.json Change uses: stefanzweifel/git-auto-commit-action@v5 with: From bcc536a12145bac5867f6e3930828c19950f11a0 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Mon, 18 Mar 2024 16:54:26 -0700 Subject: [PATCH 6/6] Fix bug in parsing --- scripts/update_skill_json.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/update_skill_json.py b/scripts/update_skill_json.py index f1678a7..ee2f07b 100644 --- a/scripts/update_skill_json.py +++ b/scripts/update_skill_json.py @@ -38,7 +38,8 @@ def _get_setup_py_data(setup_py: str) -> dict: setup_meta = parse_setup(setup_py, trusted=True) - skill_id = list(setup_meta['entrypoints'].items())[0].split('=')[0] + pprint(setup_meta) + skill_id = list(setup_meta['entry_points'].values())[0].split('=')[0] return {"skill_id": skill_id, "source": setup_meta.get("url"), "package_name": setup_meta['name'],