remove short descriptions. Fixes #487

src/somef/export/json_export.py

@@ -975,7 +975,15 @@ def unify_results(repo_data: dict) -> dict:
             result[constants.PROP_TYPE] = normalized_type
             value = result.get(constants.PROP_VALUE)
             value_type = result.get(constants.PROP_TYPE)
-
+
+            # Descriptions of <5 words should probably be removed
+            if category == constants.CAT_DESCRIPTION:
+                value = result.get(constants.PROP_VALUE, "")
+                source = item.get(constants.PROP_SOURCE, "")
+                if isinstance(value, str) and len(value.split()) < 5:
+                    if isinstance(source, str) and "readme" in source.lower():
+                        continue
+
             # --- SPECIAL LOGIC FOR LICENSES and citations ---
             if category == constants.CAT_LICENSE and result.get(constants.PROP_SPDX_ID):
                 # If we have SPDX, that is our unification key

src/somef/test/test_JSON_export.py

@@ -1003,4 +1003,41 @@ def test_issue_980_reconciliation_requirements(self):
 
         os.remove(output_path)
 
+    def test_issue_487_short_descriptions(self):
+        """Checks that descriptions with less than 5 words are filtered out from the output."""
+        somef_cli.run_cli(threshold=0.8,
+                            ignore_classifiers=False,
+                            repo_url=None,
+                            local_repo=test_data_repositories + "sunpy_short_desc",
+                            doc_src=None,
+                            in_file=None,
+                            output=test_data_path + "test_issue_487_short_descriptions.json",
+                            graph_out=None,
+                            graph_format="turtle",
+                            codemeta_out=None,
+                            pretty=True,
+                            missing=False,
+                            readme_only=False)
+
+        text_file = open(test_data_path + "test_issue_487_short_descriptions.json", "r")
+        data = text_file.read()
+        text_file.close()
+        json_content = json.loads(data)
+
+        descriptions = json_content[constants.CAT_DESCRIPTION]
+
+        # assert all(len(d[constants.PROP_RESULT][constants.PROP_VALUE].split()) >= 5
+        #        for d in descriptions if isinstance(d[constants.PROP_RESULT][constants.PROP_VALUE], str)), \
+        # f"Found descriptions with less than 5 words: {descriptions}"
+        pyproject_descriptions = [d for d in descriptions if "pyproject.toml" in d.get("source", "")]
+        assert len(pyproject_descriptions) >= 1, f"Short description from pyproject.toml was incorrectly filtered: {descriptions}"
+
+        # descriptions from README should have >= 5 words. But rest of files can have short descriptions.
+        readme_descriptions = [d for d in descriptions if "readme" in d.get("source", "").lower()]
+        assert all(len(d[constants.PROP_RESULT][constants.PROP_VALUE].split()) >= 5
+                for d in readme_descriptions if isinstance(d[constants.PROP_RESULT][constants.PROP_VALUE], str)), \
+            f"Found short descriptions from README that should have been filtered: {readme_descriptions}"
+
+        os.remove(test_data_path + "test_issue_487_short_descriptions.json")
+
 

src/somef/test/test_data/repositories/sunpy_short_desc/README.rst

@@ -0,0 +1,82 @@
+``sunpy``
+=========
+
+SunPy core package: Python for Solar Physics
+
+|SunPy Logo|
+
++-----------------------------------+-----------------------------------+-----------------------------------+
+|           Release                 |           Development             |           Community               |
++===================================+===================================+===================================+
+|       |Latest PyPi Version|       |        |Python Versions|          |         |Matrix Chat Room|        |
++-----------------------------------+-----------------------------------+-----------------------------------+
+|     |Latest Conda Version|        |     |Project Status: Active|      |     |OpenAstronomy Discourse|     |
++-----------------------------------+-----------------------------------+-----------------------------------+
+|      |Zenodo - Latest DOI|        |  |Continuous Integration Status|  |    |Google Groups Mailing List|   |
++-----------------------------------+-----------------------------------+-----------------------------------+
+|    |sunpy stable documentation|   |     |CodeCov Code Coverage|       |       |Powered by NumFOCUS|       |
++-----------------------------------+-----------------------------------+-----------------------------------+
+|         |sunpy citation|          |                                   |            |pyOpenSci|            |
++-----------------------------------+-----------------------------------+-----------------------------------+
+
+
+``sunpy`` is a software.
+
+Installation
+------------ 
+
+We recommend following the `installation guide <https://docs.sunpy.org/en/stable/guide/installation.html>`__ in the ``sunpy`` documentation.
+This will walk you through installing ``sunpy`` and all of its dependencies.
+
+Usage
+-----
+
+If you are new to ``sunpy``, the best place to start is the `tutorial <https://docs.sunpy.org/en/stable/tutorial/index.html>`__.
+The `example gallery <https://docs.sunpy.org/en/stable/generated/gallery/index.html>`__ also includes a collection of shorter and more specific examples of using ``sunpy``.
+
+Changes
+-------
+
+See our `changelog <https://docs.sunpy.org/en/stable/whatsnew/changelog.html>`__ for the latest changes in ``sunpy``.
+
+Getting Help
+------------
+
+For more information or to ask questions about ``sunpy`` or any other SunPy Project package, please check out:
+
+-  `sunpy documentation <https://docs.sunpy.org/en/stable/>`__
+-  `SunPy Affiliated Packages <https://sunpy.org/affiliated>`__
+-  `SunPy Chat <https://app.element.io/#/room/#sunpy:openastronomy.org>`__
+-  `SunPy mailing list <https://groups.google.com/forum/#!forum/sunpy>`__
+-  `SunPy Community forum <https://community.openastronomy.org/c/sunpy/5>`__
+
+Acknowledging or Citing ``sunpy``
+---------------------------------
+
+If you use ``sunpy`` in your scientific work, we would appreciate your `citing it in your publications <https://docs.sunpy.org/en/stable/citation.html>`__.
+The continued growth and development of ``sunpy`` is dependent on the community being aware of ``sunpy``.
+
+Usage of Generative AI
+----------------------
+
+If you use generative AI tools as an aid in developing code or documentation changes, ensure that you fully understand the proposed changes and can explain why they are the correct approach and an improvement to the current state.
+For more information see our documentation on fair and appropriate `AI usage <https://docs.sunpy.org/en/latest/dev_guide/contents/ai_usage.html>`__.
+
+Contributing
+------------
+
+We love contributions! sunpy is open source,
+built on open source, and we'd love to have you hang out in our community.
+
+If you would like to get involved, check out the `Developers Guide`_ section of the SunPy docs.
+Stop by our chat room `#sunpy:openastronomy.org`_ if you have any questions.
+Help is always welcome so let us know what you like to work on, or check out the `issues page`_ for the list of known outstanding items.
+
+For more information on contributing to SunPy, please read our `Newcomers' guide`_.
+
+.. _Developers Guide: https://docs.sunpy.org/en/latest/dev_guide/index.html
+.. _`#sunpy:openastronomy.org`: https://app.element.io/#/room/#sunpy:openastronomy.org
+.. _issues page: https://github.com/sunpy/sunpy/issues
+.. _Newcomers' guide: https://docs.sunpy.org/en/latest/dev_guide/contents/newcomers.html
+
+When you are interacting with the SunPy community you are asked at to follow our `code of conduct <https://sunpy.org/coc>`__.