integrate feedback from KelseyCreekSoftware

* add conversion checks
* add UX improvements
* windows path issues
* require input and output files for converters

Author: Teque5

docs/source/converters.rst

@@ -97,10 +97,10 @@ BLUE Converter
 
 The BLUE converter handles CDIF (.cdif) recordings while placing BLUE header information into the following global fields:
 
-* ``blue:fixed`` - fixed header information (at start of file)
-* ``blue:adjunct`` - adjunct header information (after fixed header)
-* ``blue:extended`` - extended header information (at end of file)
-* ``blue:keywords`` - user-defined key-value pairs
+* ``blue:fixed`` - Fixed header information (at start of file).
+* ``blue:adjunct`` - Adjunct header information (after fixed header).
+* ``blue:extended`` - Extended header information (at end of file). Note any duplicate fields will have a suffix like ``_1``, ``_2``, etc appended.
+* ``blue:keywords`` - User-defined key-value pairs.
 
 .. autofunction:: sigmf.convert.blue.blue_to_sigmf
 

sigmf/__init__.py

@@ -5,11 +5,7 @@
 # SPDX-License-Identifier: LGPL-3.0-or-later
 
 # version of this python module
-<<<<<<< HEAD
-__version__ = "1.5.1"
-=======
 __version__ = "1.6.0"
->>>>>>> 9220522... increment minor version
 # matching version of the SigMF specification
 __specification__ = "1.2.6"
 

sigmf/convert/__main__.py

@@ -27,16 +27,37 @@ def main() -> None:
     The converter detects the file type based on magic bytes and invokes the appropriate conversion function.
 
     By default it will output a SigMF pair (.sigmf-meta and .sigmf-data).
+
+    Converter Processing Pattern
+    ----------------------------
+    if out_path is None:
+        create_ncd = True
+    <create global_info and capture_info>
+    if create_ncd:
+        <create Non-Conforming Dataset (NCD) with .sigmf-meta only>
+        if out_path:
+            <write out_path.sigmf-meta>
+        return SigMFFile
+    if create_archive:
+        with TemporaryDirectory() as temp_dir:
+            <write .sigmf-data>
+        <write out_path.sigmf>
+    else:
+        <write out_path.sigmf-data>
+        <write out_path.sigmf-meta>
+    return SigMFFile
     """
     parser = argparse.ArgumentParser(
         description=textwrap.dedent(main.__doc__),
         formatter_class=argparse.RawDescriptionHelpFormatter,
+        prog="sigmf_convert",
     )
     parser.add_argument("input", type=str, help="Input recording path")
     parser.add_argument("output", type=str, help="Output SigMF path (no extension)")
     parser.add_argument("-v", "--verbose", action="count", default=0, help="Increase verbosity level")
-    parser.add_argument("-a", "--archive", action="store_true", help="Output .sigmf archive only")
-    parser.add_argument(
+    exclusive_group = parser.add_mutually_exclusive_group()
+    exclusive_group.add_argument("-a", "--archive", action="store_true", help="Output .sigmf archive only")
+    exclusive_group.add_argument(
         "--ncd", action="store_true", help="Output .sigmf-meta only and process as a Non-Conforming Dataset (NCD)"
     )
     parser.add_argument("--version", action="version", version=f"%(prog)s v{toolversion}")
@@ -52,13 +73,17 @@ def main() -> None:
     input_path = Path(args.input)
     output_path = Path(args.output)
 
-    # validate that ncd files are in same directory as input
+    # for ncd check that input & output files are in same directory
     if args.ncd and input_path.parent.resolve() != output_path.parent.resolve():
         raise SigMFConversionError(
             f"NCD files must be in the same directory as input file. "
             f"Input: {input_path.parent.resolve()}, Output: {output_path.parent.resolve()}"
         )
 
+    # check that the output path is a file and not a directory
+    if output_path.is_dir():
+        raise SigMFConversionError(f"Output path must be a filename, not a directory: {output_path}")
+
     # detect file type using magic bytes (same logic as fromfile())
     magic_bytes = get_magic_bytes(input_path, count=4, offset=0)
 

sigmf/convert/blue.py

@@ -224,7 +224,7 @@ def read_hcb(file_path):
 
     try:
         spec_str = "Unknown"
-        version = Version(h_keywords.get("VER", "1.0"))
+        version = Version(h_keywords.get("VER", "0.0"))
         if version.major == 1:
             spec_str = f"BLUE {version}"
         elif version.major == 2:
@@ -381,7 +381,7 @@ def data_loopback(blue_path: Path, data_path: Path, h_fixed: dict) -> None:
 
     # save out as SigMF IQ data file
     samples.tofile(data_path)
-    log.info("wrote %s", data_path)
+    log.info("wrote SigMF dataset to %s", data_path)
 
 
 def _build_common_metadata(
@@ -432,7 +432,7 @@ def get_tag(tag):
 
     # get sigmf datatype from blue format and endianness
     datatype = blue_to_sigmf_type_str(h_fixed)
-    log.info(f"Using SigMF datatype: {datatype} for BLUE format {h_fixed['format']}")
+    log.info(f"using SigMF datatype {datatype} for BLUE format {h_fixed['format']} {h_fixed['data_rep']}")
 
     # sample rate: prefer adjunct.xdelta, else extended header SAMPLE_RATE
     if "xdelta" in h_adjunct:
@@ -490,8 +490,9 @@ def get_tag(tag):
     blue_start_time += float(h_keywords.get("TC_PREC", 0))
 
     capture_info = {}
+
     if blue_start_time == 0:
-        log.warning("BLUE timecode is zero or missing; datetime metadata will be absent.")
+        log.warning("BLUE timecode is zero or missing; capture datetime metadata will be absent.")
     else:
         # timecode uses 1950-01-01 as epoch, datetime uses 1970-01-01
         blue_epoch = blue_start_time - 631152000  # seconds between 1950 and 1970
@@ -651,17 +652,17 @@ def construct_sigmf(
             global_info=global_info,
         )
     meta.add_capture(0, metadata=capture_info)
-    log.debug("created %r", meta)
 
     if create_archive:
         meta.tofile(filenames["archive_fn"], toarchive=True)
-        log.info("wrote %s", filenames["archive_fn"])
+        log.info("wrote SigMF archive to %s", filenames["archive_fn"])
         # metadata returned should be for this archive
         meta = fromfile(filenames["archive_fn"])
     else:
         meta.tofile(filenames["meta_fn"], toarchive=False)
-        log.info("wrote %s", filenames["meta_fn"])
+        log.info("wrote SigMF metadata to %s", filenames["meta_fn"])
 
+    log.debug("created %r", meta)
     return meta
 
 
@@ -790,10 +791,10 @@ def blue_to_sigmf(
     data_size_bytes = int(h_fixed.get("data_size", 0))
     metadata_only = data_size_bytes == 0
 
-    # handle NCD case where no output files are created
-    if create_ncd and out_path is None:
+    # handle NCD case
+    if create_ncd:
         # create metadata-only SigMF for NCD pointing to original file
-        return construct_sigmf_ncd(
+        ncd_meta = construct_sigmf_ncd(
             blue_path=blue_path,
             h_fixed=h_fixed,
             h_keywords=h_keywords,
@@ -803,6 +804,13 @@ def blue_to_sigmf(
             trailing_bytes=trailing_bytes,
         )
 
+        # write NCD metadata to specified output path if provided
+        if out_path is not None:
+            ncd_meta.tofile(filenames["meta_fn"])
+            log.info("wrote SigMF non-conforming metadata to %s", filenames["meta_fn"])
+
+        return ncd_meta
+
     with tempfile.TemporaryDirectory() as temp_dir:
         if not metadata_only:
             if create_archive:

sigmf/convert/wav.py

@@ -161,7 +161,6 @@ def wav_to_sigmf(
         global_info[SigMFFile.DATASET_KEY] = wav_path.name
         capture_info[SigMFFile.HEADER_BYTES_KEY] = header_bytes
 
-    if create_ncd:
         # create metadata-only SigMF for NCD pointing to original file
         meta = SigMFFile(global_info=global_info)
         meta.set_data_file(data_file=wav_path, offset=header_bytes)
@@ -174,9 +173,9 @@ def wav_to_sigmf(
             output_dir = filenames["meta_fn"].parent
             output_dir.mkdir(parents=True, exist_ok=True)
             meta.tofile(filenames["meta_fn"], toarchive=False)
-            log.info("wrote %s", filenames["meta_fn"])
+            log.info("wrote SigMF non-conforming metadata to %s", filenames["meta_fn"])
 
-        log.debug("created NCD SigMF: %r", meta)
+        log.debug("created %r", meta)
         return meta
 
     if out_path is None:
@@ -197,23 +196,22 @@ def wav_to_sigmf(
 
             meta = SigMFFile(data_file=data_path, global_info=global_info)
             meta.add_capture(0, metadata=capture_info)
-            log.debug("created %r", meta)
 
             meta.tofile(filenames["archive_fn"], toarchive=True)
-            log.info("wrote %s", filenames["archive_fn"])
-
+            log.info("wrote SigMF archive to %s", filenames["archive_fn"])
             # metadata returned should be for this archive
             meta = fromfile(filenames["archive_fn"])
     else:
         # write separate meta and data files
         data_path = filenames["data_fn"]
         wav_data.tofile(data_path)
+        log.info("wrote SigMF dataset to %s", data_path)
 
         meta = SigMFFile(data_file=data_path, global_info=global_info)
         meta.add_capture(0, metadata=capture_info)
-        log.debug("created %r", meta)
 
         meta.tofile(filenames["meta_fn"], toarchive=False)
-        log.info("wrote %s and %s", filenames["meta_fn"], filenames["data_fn"])
+        log.info("wrote SigMF metadata to %s", filenames["meta_fn"])
 
+    log.debug("created %r", meta)
     return meta

sigmf/sigmffile.py

@@ -883,7 +883,6 @@ def _read_datafile(self, first_byte, nitems):
         is_fixedpoint_data = dtype["is_fixedpoint"]
         is_unsigned_data = dtype["is_unsigned"]
         data_type_in = dtype["sample_dtype"]
-        component_type_in = dtype["component_dtype"]
         component_size = dtype["component_size"]
 
         data_type_out = np.dtype("f4") if not self.is_complex_data else np.dtype("f4, f4")
@@ -1315,6 +1314,9 @@ def fromfile(filename, skip_checksum=False, autoscale=True):
 
     # try auto-detection for non-SigMF files only
     if Path.is_file(file_path) and not ext.endswith(sigmf_extensions):
+        if not autoscale:
+            # TODO: allow autoscale=False for converters
+            warnings.warn("non-SigMF auto-detection conversion only supports autoscale=True; ignoring autoscale=False")
         magic_bytes = get_magic_bytes(file_path, count=4, offset=0)
 
         if magic_bytes == b"RIFF":

tests/test_convert_blue.py

@@ -46,26 +46,23 @@ def tearDown(self) -> None:
     def test_sigmf_pair(self):
         """test standard blue to sigmf conversion with file pairs"""
         for blue_path in self.blue_paths:
+            print(blue_path)
             sigmf_path = self.tmp_path / blue_path.stem
             meta = blue_to_sigmf(blue_path=blue_path, out_path=sigmf_path)
             self.assertIsInstance(meta, sigmf.SigMFFile)
-            # FIXME: REPLACE BELOW WITH BELOW COMMENTED AFTER PR #121 MERGED
             if not meta.get_global_field("core:metadata_only"):
-                _ = meta.read_samples(count=10)
                 # check sample read consistency
-                # np.testing.assert_array_equal(meta.read_samples(count=10), meta[0:10])
+                np.testing.assert_array_equal(meta.read_samples(count=10), meta[0:10])
 
     def test_sigmf_archive(self):
         """test blue to sigmf conversion with archive output"""
         for blue_path in self.blue_paths:
             sigmf_path = self.tmp_path / f"{blue_path.stem}_archive"
             meta = blue_to_sigmf(blue_path=blue_path, out_path=sigmf_path, create_archive=True)
             self.assertIsInstance(meta, sigmf.SigMFFile)
-            # FIXME: REPLACE BELOW WITH BELOW COMMENTED AFTER PR #121 MERGED
             if not meta.get_global_field("core:metadata_only"):
-                _ = meta.read_samples(count=10)
                 # check sample read consistency
-                # np.testing.assert_array_equal(meta.read_samples(count=10), meta[0:10])
+                np.testing.assert_array_equal(meta.read_samples(count=10), meta[0:10])
 
     def test_create_ncd(self):
         """test direct NCD conversion"""

tests/test_convert_wav.py

@@ -129,27 +129,19 @@ def test_sigmf_pair(self):
             sigmf_path = self.tmp_path / wav_path.stem
             meta = wav_to_sigmf(wav_path=wav_path, out_path=sigmf_path)
             self.assertIsInstance(meta, sigmf.SigMFFile)
-            # FIXME: REPLACE BELOW WITH BELOW COMMENTED AFTER PR #121 MERGED
             if not meta.get_global_field("core:metadata_only"):
-                _ = meta.read_samples(count=10)
                 # check sample read consistency
-                # np.testing.assert_array_equal(meta.read_samples(count=10), meta[0:10])
+                np.testing.assert_array_equal(meta.read_samples(count=10), meta[0:10])
 
     def test_sigmf_archive(self):
         """test wav to sigmf conversion with archive output"""
         for wav_path in self.wav_paths:
             sigmf_path = self.tmp_path / f"{wav_path.stem}_archive"
             meta = wav_to_sigmf(wav_path=wav_path, out_path=sigmf_path, create_archive=True)
-            # FIXME: I believe this error related to sample_count being 0 is fixed by PR 121
-            print("dbug", meta)
-            print("dbug len", len(meta))
-            print("dbug sample_count", meta.sample_count)
             self.assertIsInstance(meta, sigmf.SigMFFile)
-            # FIXME: REPLACE BELOW WITH BELOW COMMENTED AFTER PR #121 MERGED
             if not meta.get_global_field("core:metadata_only"):
-                _ = meta.read_samples(count=10)
                 # check sample read consistency
-                # np.testing.assert_array_equal(meta.read_samples(count=10), meta[0:10])
+                np.testing.assert_array_equal(meta.read_samples(count=10), meta[0:10])
 
     def test_create_ncd(self):
         """test direct NCD conversion"""