Update skill.json parsing and automation

.github/workflows/skill_update_json_spec.yml

@@ -11,32 +11,33 @@ jobs:
     runs-on: ${{inputs.runner}}
     timeout-minutes: 15
     steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v2
+      - name: Checkout Skill Repository
+        uses: actions/checkout@v4
         with:
           path: action/skill/
       - name: Checkout Scripts Repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           path: action/github/
           repository: NeonGeckoCom/.github
+          ref: FEAT_UpdateSkillMetaParsing
+          # TODO: Remove ref before merge
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: Install Dependencies
         run: |
           sudo apt update
           sudo apt install -y gcc git libpulse-dev
           pip install --upgrade pip
-          pip install wheel "cython<3.0.0"  # TODO: cython patching https://github.com/yaml/pyyaml/issues/724
-          pip install --no-build-isolation pyyaml~=5.4  # TODO: patching https://github.com/yaml/pyyaml/issues/724
+          pip install wheel numpy  # numpy patching https://github.com/benfred/parsesetup/issues/3
           pip install -r action/github/scripts/requirements.txt
       - name: Get Updated skill.json
         run: |
-          python action/github/scripts/update_skill_json.py action/skill
+          python3 "${{ github.workspace }}/action/github/scripts/update_skill_json.py" "${{ github.workspace }}/action/skill"
       - name: Push skill.json Change
-        uses: stefanzweifel/git-auto-commit-action@v4
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: Update skill.json
           repository: action/skill/

.github/workflows/skill_update_meta_repo.yml

@@ -12,10 +12,13 @@ jobs:
     runs-on: ${{inputs.runner}}
     timeout-minutes: 5
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout Skill Repository
+        uses: actions/checkout@v4
       - name: Update skill.json in neon_skills
         run: |
+          version=$(python3 setup.py --version)
           git clone https://github.com/neongeckocom/neon_skills -b ${{github.ref_name}}
+          cp skill.json neon_skills/skill_metadata/${{github.event.repository.name}}-${version}.json
           cp skill.json neon_skills/skill_metadata/${{github.event.repository.name}}.json
       - name: Push neon_skills changes
         uses: cpina/github-action-push-to-another-repository@main

scripts/requirements.txt

@@ -1,4 +1,4 @@
 neon-utils~=1.2
-ovos-skills-manager~=0.0
+parsesetup~=0.0.1
 cattrs != 23.1.0
 # TODO: cattrs patching https://github.com/python-attrs/cattrs/issues/369

scripts/update_skill_json.py

@@ -27,16 +27,54 @@
 # SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import json
+import shutil
 
 from sys import argv
-from os.path import join
+from os.path import join, isfile
 from pprint import pprint
-from neon_utils.packaging_utils import build_skill_spec
+from parsesetup import parse_setup
+from neon_utils.file_utils import parse_skill_readme_file
 
 
-def get_skill_json(skill_dir: str):
-    print(f"skill_dir={skill_dir}")
-    skill_json = join(skill_dir, "skill.json")
+def _get_setup_py_data(setup_py: str) -> dict:
+    setup_meta = parse_setup(setup_py, trusted=True)
+    pprint(setup_meta)
+    skill_id = list(setup_meta['entry_points'].values())[0].split('=')[0]
+    return {"skill_id": skill_id,
+            "source": setup_meta.get("url"),
+            "package_name": setup_meta['name'],
+            "license": setup_meta.get("license"),
+            "author": setup_meta.get("author"),
+            "description": setup_meta.get("description")}
+
+
+def build_skill_spec(skill_dir: str) -> dict:
+    readme_file = join(skill_dir, "README.md")
+    setup_py = join(skill_dir, "setup.py")
+    if isfile(setup_py):
+        skill_data = _get_setup_py_data(setup_py)
+    readme_data = parse_skill_readme_file(readme_file)
+    skill_data["description"] = readme_data.get("summary") or skill_data["description"]
+    skill_data["name"] = readme_data.get("title")
+    skill_data["examples"] = readme_data.get("examples") or []
+    skill_data["tags"] = readme_data.get("categories",
+                                         []) + readme_data.get("tags", [])
+    skill_data["icon"] = readme_data.get("icon")
+    return skill_data
+
+
+def write_skill_json(skill_dir: str, default_lang="en-us"):
+
+    old_skill_json = join(skill_dir, "skill.json")
+    setup_py = join(skill_dir, "setup.py")
+    pkg_dir = list(parse_setup(setup_py,
+                               trusted=True)['package_dir'].values())[0]
+    skill_json = join(skill_dir, pkg_dir, "locale", default_lang, "skill.json")
+    print(f"skill_dir={skill_dir}|skill.json={skill_json}")
+
+    if isfile(old_skill_json):
+        shutil.move(old_skill_json, skill_json)
+
     skill_spec = build_skill_spec(skill_dir)
     pprint(skill_spec)
     try:
@@ -45,6 +83,8 @@ def get_skill_json(skill_dir: str):
     except Exception as e:
         print(e)
         current = None
+    if current:
+        skill_spec["extra_plugins"] = current.get("extra_plugins") or dict()
     if current != skill_spec:
         print("Skill Updated. Writing skill.json")
         with open(skill_json, 'w+') as f:
@@ -54,4 +94,4 @@ def get_skill_json(skill_dir: str):
 
 
 if __name__ == "__main__":
-    get_skill_json(argv[1])
+    write_skill_json(argv[1])