Added sort function and tested! fixes #51 (#223)

* added sort function and tested! fixes #51

* added CLI with test New command: sort #51

* applied sort

* sort_df_column effects fixed

* sort rows byt column 1 and bug fix for negation

* sort rows by column 1 and bug fix for negation #51

* renamed sort_columns to have rows and added bools

* renamed CLI to `sort`

* changes as per #225

* changes as per #225

Author: hrshdhgd

sssom/cli.py

@@ -40,6 +40,7 @@
     merge_msdf,
     reconcile_prefix_and_data,
     remove_unmatched,
+    sort_df_rows_columns,
     to_mapping_set_dataframe,
 )
 from .writers import write_table
@@ -77,9 +78,9 @@
     type=click.Path(),
     help="The path to a file containing the sssom metadata (including prefix_map) to be used.",
 )
-transpose_option = click.option("-t", "--transpose/--no-transpose", default=False)
+transpose_option = click.option("-t", "--transpose", default=False)
 fields_option = click.option(
-    "-F",
+    "-f",
     "--fields",
     nargs=2,
     default=("subject_category", "object_category"),
@@ -212,7 +213,7 @@ def dedupe(input: str, output: TextIO):
 
 
 @main.command()
-@click.option("-q", "--query", help='SQL query. Use "df" as table name.')
+@click.option("-Q", "--query", help='SQL query. Use "df" as table name.')
 @click.argument("inputs", nargs=-1)
 @output_option
 def dosql(query: str, inputs: List[str], output: TextIO):
@@ -425,7 +426,7 @@ def correlations(input: str, output: TextIO, transpose: bool, fields: Tuple):
 @main.command()
 @click.argument("inputs", nargs=-1)
 @click.option(
-    "-r",
+    "-R",
     "--reconcile",
     default=True,
     help="Boolean indicating the need for reconciliation of the SSSOM tsv file.",
@@ -501,5 +502,34 @@ def reconcile_prefixes(input: str, reconcile_prefix_file: Path, output: TextIO):
     write_table(recon_msdf, output)
 
 
+@main.command()
+@input_argument
+@output_option
+@click.option(
+    "-k",
+    "--by-columns",
+    default=True,
+    help="Sort columns of DataFrame canonically.",
+)
+@click.option(
+    "-r",
+    "--by-rows",
+    default=True,
+    help="Sort rows by DataFrame column #1 (ascending).",
+)
+def sort(input: str, output: TextIO, by_columns: bool, by_rows: bool):
+    """
+    Sort DataFrame columns canonically.
+
+    :param input: SSSOM TSV file.
+    :param by_columns: Boolean flag to sort columns canonically.
+    :param by_rows: Boolean flag to sort rows by column #1 (ascending order).
+    :param output: SSSOM TSV file with columns sorted.
+    """
+    msdf = read_sssom_table(input)
+    msdf.df = sort_df_rows_columns(msdf.df, by_columns, by_rows)
+    write_table(msdf, output)
+
+
 if __name__ == "__main__":
     main()

sssom/util.py

@@ -635,7 +635,8 @@ def deal_with_negation(df: pd.DataFrame) -> pd.DataFrame:
         reconciled_df_subset = reconciled_df_subset.append(
             combined_normalized_subset.loc[
                 match_condition_1[match_condition_1].index, :
-            ]
+            ],
+            ignore_index=True,
         )
 
     # Add negations (PREDICATE_MODIFIER) back to DataFrame
@@ -657,9 +658,14 @@ def deal_with_negation(df: pd.DataFrame) -> pd.DataFrame:
             PREDICATE_MODIFIER
         ].fillna("")
 
+    # .fillna(df) towards the end fills an empty value
+    # with a corresponding value from df.
+    # This needs to happen because the columns in df
+    # not in reconciled_df_subset will be NaN otherwise
+    # which is incorrect.
     reconciled_df = df.merge(
         reconciled_df_subset, how="right", on=list(reconciled_df_subset.columns)
-    ).fillna("")
+    ).fillna(df)
 
     if nan_df.empty:
         return_df = reconciled_df
@@ -769,7 +775,8 @@ def read_pandas(
         else:
             sep = "\t"
             logging.warning("Cannot automatically determine table format, trying tsv.")
-    return read_csv(file, comment="#", sep=sep).fillna("")
+        df = read_csv(file, comment="#", sep=sep).fillna("")
+    return sort_df_rows_columns(df)
 
 
 def extract_global_metadata(msdoc: MappingSetDocument) -> Dict[str, PrefixMap]:
@@ -820,6 +827,7 @@ def to_mapping_set_dataframe(doc: MappingSetDocument) -> MappingSetDataFrame:
         np.nan, "", inplace=True
     )
     msdf = MappingSetDataFrame(df=df, prefix_map=doc.prefix_map, metadata=meta)
+    msdf.df = sort_df_rows_columns(msdf.df)
     return msdf
 
 
@@ -1117,3 +1125,24 @@ def reconcile_prefix_and_data(
 
     # TODO: When expansion of 2 prefixes in the prefix_map are the same.
     return msdf
+
+
+def sort_df_rows_columns(
+    df: pd.DataFrame, by_columns: bool = True, by_rows: bool = True
+) -> pd.DataFrame:
+    """
+    Canonical sorting of DataFrame columns.
+
+    :param df: Pandas DataFrame with random column sequence.
+    :param by_columns: Boolean flag to sort columns canonically.
+    :param by_rows: Boolean flag to sort rows by column #1 (ascending order).
+    :return: Pandas DataFrame columns sorted canonically.
+    """
+    if by_columns:
+        column_sequence = [
+            col for col in SCHEMA_DICT["slots"].keys() if col in df.columns
+        ]
+        df = df.reindex(column_sequence, axis=1)
+    if by_rows:
+        df = df.sort_values(by=df.columns[0], ignore_index=True)
+    return df

tests/data/basic6.tsv

@@ -0,0 +1,24 @@
+#license: "https://creativecommons.org/publicdomain/zero/1.0/"
+#mapping_set_id: http://w3id.org/sssom/mapping/tests/data/basic3.tsv
+#mapping_tool: "https://github.com/cmungall/rdf_matcher"
+#creator_id: "cjm"
+#mapping_date: "2020-05-30"
+#curie_map:
+#  a: "http://example.org/a/"
+#  b: "http://example.org/b/"
+#  c: "http://example.org/c/"
+#  d: "http://example.org/d/"
+#  rdfs: "http://example.org/rdfs/"
+#  owl: "http://example.org/owl/"
+comment	mapping_tool	subject_label	confidence	object_id	match_type	subject_source	object_source	subject_id	subject_match_field	object_match_field	object_label	subject_category	predicate_modifier	object_category	match_string	predicate_id
+mock data	rdf_matcher	YYYYY	0.81	b:something	Lexical	c	d	c:something	rdfs:label	rdfs:label	yyyyyy	biolink:AnatomicalEntity		biolink:AnatomicalEntity	xxxxx	owl:equivalentClass
+mock data	rdf_matcher	YYYYY	0.82	a:something	Lexical	d	a	d:something	rdfs:label	rdfs:label	yyyyyy	biolink:AnatomicalEntity	Not	biolink:AnatomicalEntity	xxxxx	owl:equivalentClass
+mock data	rdf_matcher	XYXYX	0.83	c:something	Lexical	a	c	a:something	rdfs:label	rdfs:label	xyxyxy	biolink:AnatomicalEntity		biolink:AnatomicalEntity	xxxxx	owl:equivalentClass
+mock data	rdf_matcher	YXYXY	0.845	b:something	HumanCurated	c	b	c:something	rdfs:label	rdfs:label	yxyxyx	biolink:AnatomicalEntity		biolink:AnatomicalEntity	xxxxx	owl:equivalentClass
+mock data	rdf_matcher	XXXXX	0.8	a:something	Lexical	b	a	b:something	rdfs:label	rdfs:label	xxxxxx	biolink:AnatomicalEntity		biolink:AnatomicalEntity	xxxxx	owl:equivalentClass
+mock data	rdf_matcher	YYYYY	0.81	d:something	Lexical	c	d	c:something	rdfs:label	rdfs:label	yyyyyy	biolink:AnatomicalEntity		biolink:AnatomicalEntity	xxxxx	owl:equivalentClass
+mock data	rdf_matcher	YXYXY	0.845	b:something	HumanCurated	c	b	c:something	rdfs:label	rdfs:label	yxyxyx	biolink:AnatomicalEntity	Not	biolink:AnatomicalEntity	xxxxx	owl:equivalentClass
+mock data	rdf_matcher	XYXYX	0.83	b:something	Lexical	d	b	d:something	rdfs:label	rdfs:label	xyxyxy	biolink:AnatomicalEntity		biolink:AnatomicalEntity	xxxxx	owl:equivalentClass
+mock data	rdf_matcher	XXXXX	0.8	b:something	Lexical	a	b	a:something	rdfs:label	rdfs:label	xxxxxx	biolink:AnatomicalEntity		biolink:AnatomicalEntity	xxxxx	owl:subClassOf
+mock data	rdf_matcher	YYYYY	0.82	a:something	HumanCurated	d	a	d:something	rdfs:label	rdfs:label	yyyyyy	biolink:AnatomicalEntity		biolink:AnatomicalEntity	xxxxx	owl:equivalentClass
+mock data	rdf_matcher	XXXXX	0.8	b:something	Lexical	a	b	a:something	rdfs:label	rdfs:label	xxxxxx	biolink:AnatomicalEntity	Not	biolink:AnatomicalEntity	xxxxx	owl:subClassOf

tests/test_cli.py

@@ -19,6 +19,7 @@
     partition,
     ptable,
     reconcile_prefixes,
+    sort,
     split,
     validate,
 )
@@ -58,6 +59,7 @@ def test_cli_single_input(self):
                     self.run_correlations(runner, test)
                     self.run_reconcile_prefix(runner, test)
                     self.run_dosql(runner, test)
+                    self.run_sort_rows_columns(runner, test)
 
         self.assertTrue(len(test_cases) > 2)
 
@@ -258,7 +260,7 @@ def run_dosql(self, runner: CliRunner, test_case: SSSOMTestCase) -> Result:
         result = runner.invoke(
             dosql,
             [
-                "-q",
+                "-Q",
                 "SELECT * FROM df WHERE subject_label = 'heart'",
                 test_case.filepath,
                 "-o",
@@ -267,3 +269,24 @@ def run_dosql(self, runner: CliRunner, test_case: SSSOMTestCase) -> Result:
         )
         self.run_successful(result, test_case)
         return result
+
+    def run_sort_rows_columns(
+        self, runner: CliRunner, test_case: SSSOMTestCase
+    ) -> Result:
+        """Test sorting of DataFrame columns."""
+        out_file = os.path.join(test_out_dir, "sort_column_test.tsv")
+        in_file = test_case.filepath.replace("basic", "basic6")
+        result = runner.invoke(
+            sort,
+            [
+                in_file,
+                "-o",
+                os.path.join(test_out_dir, out_file),
+                "-k",
+                True,
+                "-r",
+                True,
+            ],
+        )
+        self.run_successful(result, test_case)
+        return result

tests/test_parsers.py

@@ -21,7 +21,7 @@
     read_sssom_table,
     to_mapping_set_document,
 )
-from sssom.util import PREFIX_MAP_KEY, to_mapping_set_dataframe
+from sssom.util import PREFIX_MAP_KEY, sort_df_rows_columns, to_mapping_set_dataframe
 from sssom.writers import write_table
 from tests.test_data import data_dir as test_data_dir
 from tests.test_data import test_out_dir
@@ -190,6 +190,7 @@ def test_read_sssom_table(self):
         input_path = os.path.join(test_data_dir, "basic3.tsv")
         msdf = read_sssom_table(input_path)
         imported_df = pd.read_csv(input_path, comment="#", sep="\t")
+        imported_df = sort_df_rows_columns(imported_df)
         self.assertEqual(set(imported_df.columns), set(msdf.df.columns))
         list_cols = [
             "subject_match_field",

tests/test_reconcile.py

@@ -23,7 +23,7 @@ def test_filter(self):
     def test_deal_with_negation(self):
         """Test handling negating returns the right number of rows."""
         df = deal_with_negation(self.msdf.df)
-        self.assertEqual(7, len(df.index))
+        self.assertEqual(8, len(df.index))
 
     def test_merge(self):
         """Test merging two tables."""

tests/test_sort.py

@@ -0,0 +1,24 @@
+"""Test for sorting MappingSetDataFrame columns."""
+
+import unittest
+
+from sssom.constants import SCHEMA_DICT
+from sssom.parsers import read_sssom_table
+from sssom.util import sort_df_rows_columns
+from tests.constants import data_dir
+
+
+class TestSort(unittest.TestCase):
+    """A test case for sorting msdf columns."""
+
+    def setUp(self) -> None:
+        """Test up the test cases with the third basic example."""
+        self.msdf = read_sssom_table(f"{data_dir}/basic6.tsv")
+
+    def test_sort(self):
+        """Test sorting of columns."""
+        new_df = sort_df_rows_columns(self.msdf.df)
+        column_sequence = [
+            col for col in SCHEMA_DICT["slots"].keys() if col in new_df.columns
+        ]
+        self.assertListEqual(column_sequence, list(new_df.columns))