Drop pkg_resources and nose; migrate to modern Python 3.12 stack

dcase_util/__init__.py

@@ -24,11 +24,49 @@ def check_installation():
     """Utility function to check package installation.
     """
 
-    import pkg_resources
     import os
-    import sys
     import platform
     import subprocess
+    import sys
+    from importlib import metadata as importlib_metadata
+
+    from packaging.requirements import Requirement
+    from packaging.utils import canonicalize_name
+
+    def _parse_requirements(requirement_lines):
+        parsed_requirements = []
+        for requirement_line in requirement_lines:
+            line = requirement_line.strip()
+            if not line or line.startswith('#'):
+                continue
+
+            requirement = Requirement(line)
+            if requirement.marker is None or requirement.marker.evaluate():
+                parsed_requirements.append(requirement)
+
+        return parsed_requirements
+
+    def _requirements_from_file(filename):
+        if filename and os.path.isfile(filename):
+            with open(filename) as file_handle:
+                return _parse_requirements(file_handle.readlines())
+        return []
+
+    def _requirement_spec(requirement):
+        if requirement.specifier:
+            return str(requirement.specifier)
+        return '-'
+
+    def _installed_status(requirement):
+        try:
+            installed_version = importlib_metadata.version(requirement.name)
+        except importlib_metadata.PackageNotFoundError:
+            return 'N/A', 'MISSING'
+
+        if requirement.specifier and not requirement.specifier.contains(installed_version, prereleases=True):
+            return installed_version, 'CHECK'
+
+        return installed_version, 'OK'
 
     log = ui.FancyPrinter()
 
@@ -53,55 +91,72 @@ def check_installation():
     log.data(field='Version', value=__version__)
     log.line()
 
-    package = pkg_resources.require('dcase_util')[0]
+    package_distribution = None
+    core_requirements = []
+    requirements_filename = os.path.abspath(
+        os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'requirements.txt')
+    )
+
+    try:
+        package_distribution = importlib_metadata.distribution('dcase_util')
+    except importlib_metadata.PackageNotFoundError:
+        package_distribution = None
 
-    # Get core requirements
-    core_requirements = package.requires()
+    if package_distribution is not None:
+        core_requirements = _parse_requirements(package_distribution.requires or [])
+        package_requirements_filename = os.path.abspath(str(package_distribution.locate_file('requirements.txt')))
+        if os.path.isfile(package_requirements_filename):
+            requirements_filename = package_requirements_filename
 
-    # Load requirements.txt
-    requirements_filename = os.path.join(package.location, 'requirements.txt')
-    with open(requirements_filename) as fp:
-        requirements_file = fp.read()
+    if not core_requirements:
+        core_requirements = _requirements_from_file(requirements_filename)
 
-    # Get all requirements
-    all_requirements = []
-    for r in pkg_resources.parse_requirements(requirements_file):
-        if r.marker:
-            raise ValueError("environment markers are not supported, in '%s'" % r)
-        all_requirements.append(r)
+    all_requirements = _requirements_from_file(requirements_filename)
+    if not all_requirements:
+        all_requirements = list(core_requirements)
 
-    processed = []
+    processed = set()
 
     log.line('Core requirements')
     log.row('Package', 'Required', 'Installed', 'Status', widths=[25, 15, 15, 15])
     log.row_sep()
     for requirement in core_requirements:
-        if requirement.key not in processed:
+        requirement_name = canonicalize_name(requirement.name)
+        if requirement_name not in processed:
+            installed, status = _installed_status(requirement)
             log.row(
-                requirement.key,
-                ''.join(requirement.specs[0]),
-                pkg_resources.get_distribution(requirement.key).version,
-                'OK' if requirement.__contains__(pkg_resources.get_distribution(requirement.key)) else 'CHECK'
+                requirement.name,
+                _requirement_spec(requirement),
+                installed,
+                status
             )
-            processed.append(requirement.key)
+            processed.add(requirement_name)
     log.line()
 
     log.line('Extra requirements')
     log.row('Package', 'Required', 'Installed', 'Status', widths=[25, 15, 15, 15])
     log.row_sep()
     for requirement in all_requirements:
-        if requirement.key not in processed:
+        requirement_name = canonicalize_name(requirement.name)
+        if requirement_name not in processed:
+            installed, status = _installed_status(requirement)
             log.row(
-                requirement.key,
-                ''.join(requirement.specs[0]),
-                pkg_resources.get_distribution(requirement.key).version,
-                'OK' if requirement.__contains__(pkg_resources.get_distribution(requirement.key)) else 'CHECK'
+                requirement.name,
+                _requirement_spec(requirement),
+                installed,
+                status
             )
-            processed.append(requirement.key)
+            processed.add(requirement_name)
     log.line()
 
     # Get system level requirements
     log.line('System')
-    ffmpeg_info = subprocess.check_output(['ffmpeg', '-version']).decode('utf-8')
+    try:
+        ffmpeg_info = subprocess.check_output(
+            ['ffmpeg', '-version'],
+            stderr=subprocess.STDOUT
+        ).decode('utf-8')
+    except (subprocess.CalledProcessError, OSError) as error:
+        ffmpeg_info = str(error)
 
     log.data(field='FFMPEG', value=ffmpeg_info)

dcase_util/containers/mapping.py

@@ -51,7 +51,7 @@ def load(self, filename=None):
 
             if self.format == FileFormat.TXT or self.format == FileFormat.CSV:
                 map_data = {}
-                with open(self.filename, 'rtU') as f:
+                with open(self.filename, 'r', newline='') as f:
                     for row in csv.reader(f, delimiter=self.delimiter()):
                         if len(row) == 2:
                             map_data[row[0]] = row[1]

dcase_util/data/manipulators.py

@@ -849,10 +849,12 @@ def aggregate(self, data=None, **kwargs):
                         aggregated_frame.append(numpy.cov(current_frame).flatten())
 
                     if 'kurtosis' in self.recipe:
-                        aggregated_frame.append(scipy.stats.kurtosis(current_frame, axis=data.time_axis))
+                        kurtosis_values = scipy.stats.kurtosis(current_frame, axis=data.time_axis)
+                        aggregated_frame.append(numpy.nan_to_num(kurtosis_values, nan=-3.0))
 
                     if 'skew' in self.recipe:
-                        aggregated_frame.append(scipy.stats.skew(current_frame, axis=data.time_axis))
+                        skew_values = scipy.stats.skew(current_frame, axis=data.time_axis)
+                        aggregated_frame.append(numpy.nan_to_num(skew_values, nan=0.0))
 
                     if 'flatten' in self.recipe:
                         if data.time_axis == 0:

dcase_util/datasets/datasets.py

@@ -1977,7 +1977,7 @@ def validation_files_random(self,
             training_meta = self.train(fold=fold)
 
         training_files = training_meta.unique_files
-        random.shuffle(training_files, random.random)
+        random.shuffle(training_files)
 
         validation_split_index = int(numpy.ceil(validation_amount * len(training_files)))
         validation_files = training_files[0:validation_split_index]
@@ -2421,7 +2421,7 @@ def validation_files_balanced(self,
             for scene_id, scene_label in enumerate(training_meta.unique_scene_labels):
                 scene_files = training_meta.filter(scene_label=scene_label).unique_files
 
-                random.shuffle(scene_files, random.random)
+                random.shuffle(scene_files)
                 validation_split_index = int(numpy.ceil(validation_amount * len(scene_files)))
                 current_validation_files = scene_files[0:validation_split_index]
                 current_training_files = scene_files[validation_split_index:]
@@ -2466,7 +2466,7 @@ def validation_files_balanced(self,
 
                 for i in iteration_progress:
                     current_locations = list(data.keys())
-                    random.shuffle(current_locations, random.random)
+                    random.shuffle(current_locations)
                     validation_split_index = int(numpy.ceil(validation_amount * len(data)))
                     current_validation_identifiers = current_locations[0:validation_split_index]
                     current_training_identifiers = current_locations[validation_split_index:]
@@ -2561,7 +2561,7 @@ def validation_files_balanced(self,
                     current_validation_identifiers2 = 0
                     for identifier1 in identifier_first_level:
                         current_ids = list(data[identifier1].keys())
-                        random.shuffle(current_ids, random.random)
+                        random.shuffle(current_ids)
                         validation_split_index = int(numpy.ceil(validation_amount * len(current_ids)))
                         current_validation = current_ids[0:validation_split_index]
                         current_training = current_ids[validation_split_index:]
@@ -3300,7 +3300,7 @@ def validation_files_random(self,
             scene_labels = self.scene_labels()
 
         training_files = training_meta.unique_files
-        random.shuffle(training_files, random.random)
+        random.shuffle(training_files)
 
         validation_split_index = int(numpy.ceil(validation_amount * len(training_files)))
         validation_files = training_files[0:validation_split_index]
@@ -3503,7 +3503,7 @@ def validation_files_balanced(self,
 
                 for i in iteration_progress:
                     item_ids = list(range(0, len(data[scene_label])))
-                    random.shuffle(item_ids, random.random)
+                    random.shuffle(item_ids)
 
                     valid_percentage_index = int(numpy.ceil(validation_amount * len(item_ids)))
 
@@ -3604,7 +3604,7 @@ def validation_files_balanced(self,
 
                 for i in iteration_progress:
                     identifiers = list(data[scene_label].keys())
-                    random.shuffle(identifiers, random.random)
+                    random.shuffle(identifiers)
 
                     valid_percentage_index = int(numpy.ceil(validation_amount * len(identifiers)))
 
@@ -3811,7 +3811,7 @@ def validation_files_random(self,
         scene_labels = self.scene_labels()
 
         training_files = training_meta.unique_files
-        random.shuffle(training_files, random.random)
+        random.shuffle(training_files)
 
         validation_split_index = int(numpy.ceil(validation_amount * len(training_files)))
         validation_files = training_files[0:validation_split_index]
@@ -3965,7 +3965,7 @@ def validation_files_balanced(self,
 
                 for i in iteration_progress:
                     identifiers = list(data[scene_label].keys())
-                    random.shuffle(identifiers, random.random)
+                    random.shuffle(identifiers)
 
                     valid_percentage_index = int(numpy.ceil(validation_amount * len(identifiers)))
 
@@ -4061,7 +4061,7 @@ def validation_files_balanced(self,
 
                 for i in iteration_progress:
                     items_id = list(range(0, len(data[scene_label])))
-                    random.shuffle(items_id, random.random)
+                    random.shuffle(items_id)
 
                     valid_percentage_index = int(numpy.ceil(validation_amount * len(items_id)))
 

dcase_util/features/features.py

@@ -379,18 +379,28 @@ def get_window_function(self, n, window_type='hamming_asymmetric'):
 
         """
 
+        def _window(window_name, symmetric):
+            if hasattr(scipy.signal, 'windows') and hasattr(scipy.signal.windows, window_name):
+                return getattr(scipy.signal.windows, window_name)(n, sym=symmetric)
+
+            if hasattr(scipy.signal, window_name):
+                return getattr(scipy.signal, window_name)(n, sym=symmetric)
+
+            # Fallback for compatibility across SciPy versions.
+            return scipy.signal.get_window(window_name, n, fftbins=not symmetric)
+
         # Windowing function
         if window_type == 'hamming_asymmetric':
-            return scipy.signal.hamming(n, sym=False)
+            return _window('hamming', symmetric=False)
 
         elif window_type == 'hamming_symmetric' or window_type == 'hamming':
-            return scipy.signal.hamming(n, sym=True)
+            return _window('hamming', symmetric=True)
 
         elif window_type == 'hann_asymmetric':
-            return scipy.signal.hann(n, sym=False)
+            return _window('hann', symmetric=False)
 
         elif window_type == 'hann_symmetric' or window_type == 'hann':
-            return scipy.signal.hann(n, sym=True)
+            return _window('hann', symmetric=True)
 
         else:
             message = '{name}: Unknown window type [{window_type}]'.format(

dcase_util/files/serialization.py

@@ -176,14 +176,28 @@ def load_msgpack(cls, filename):
             import msgpack
 
         except ImportError:
-            message = '{name}: Unable to import msgpack module. You can install it with `pip install msgpack-python`.'.format(
+            message = '{name}: Unable to import msgpack module. You can install it with `pip install msgpack`.'.format(
                 name=cls.__class__.__name__
             )
 
             cls.logger().exception(message)
             raise ImportError(message)
 
-        return msgpack.load(open(filename, "rb"), encoding='utf-8')
+        with open(filename, 'rb') as file_handle:
+            # Keep compatibility with both modern msgpack and older environments.
+            try:
+                return msgpack.load(file_handle, raw=False, strict_map_key=False)
+            except TypeError:
+                file_handle.seek(0)
+                try:
+                    return msgpack.load(file_handle, raw=False)
+                except TypeError:
+                    file_handle.seek(0)
+                    try:
+                        return msgpack.load(file_handle, encoding='utf-8')
+                    except TypeError:
+                        file_handle.seek(0)
+                        return msgpack.load(file_handle)
 
     @classmethod
     def load_marshal(cls, filename):
@@ -339,14 +353,15 @@ def save_msgpack(cls, filename, data):
             import msgpack
 
         except ImportError:
-            message = '{name}: Unable to import msgpack module. You can install it with `pip install msgpack-python`.'.format(
+            message = '{name}: Unable to import msgpack module. You can install it with `pip install msgpack`.'.format(
                 name=cls.__class__.__name__
             )
 
             cls.logger().exception(message)
             raise ImportError(message)
 
-        msgpack.dump(data, open(filename, 'wb'), use_bin_type=True)
+        with open(filename, 'wb') as file_handle:
+            msgpack.dump(data, file_handle, use_bin_type=True)
 
     @classmethod
     def save_marshal(cls, filename, data):

dcase_util/utils/examples.py

@@ -3,8 +3,7 @@
 
 from __future__ import print_function, absolute_import
 
-import os
-import pkg_resources
+from importlib import resources
 import numpy
 
 
@@ -15,21 +14,25 @@ class Example(object):
     def __init__(self):
         pass
 
+    @classmethod
+    def _resource_filename(cls, filename):
+        return str(resources.files(__package__).joinpath(cls.example_folder, filename))
+
     @classmethod
     def audio_filename(cls):
-        return pkg_resources.resource_filename(__name__, os.path.join(cls.example_folder, 'acoustic_scene.wav'))
+        return cls._resource_filename('acoustic_scene.wav')
 
     @classmethod
     def acoustic_scene_audio_filename(cls):
-        return pkg_resources.resource_filename(__name__, os.path.join(cls.example_folder, 'acoustic_scene.wav'))
+        return cls._resource_filename('acoustic_scene.wav')
 
     @classmethod
     def audio_filename_mp3(cls):
-        return pkg_resources.resource_filename(__name__, os.path.join(cls.example_folder, 'acoustic_scene.mp3'))
+        return cls._resource_filename('acoustic_scene.mp3')
 
     @classmethod
     def acoustic_scene_audio_filename_mp3(cls):
-        return pkg_resources.resource_filename(__name__, os.path.join(cls.example_folder, 'acoustic_scene.mp3'))
+        return cls._resource_filename('acoustic_scene.mp3')
 
     @classmethod
     def audio_container(cls):
@@ -207,4 +210,4 @@ def feature_repository(cls, filename=None):
                 }
             }
         ])
-        return chain.process(filename=filename)
+        return chain.process(filename=filename)