diff --git a/.gitignore b/.gitignore index 3d22ef6c3..6029e326a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ __pycache__/ .mypy_cache/** **/.mypy_cache/** +# we are a library, don't lock the requirements +uv.lock + # PyCharm # Keep some to help new users... .idea/codeStyles diff --git a/documentation/testDocumentation.py b/documentation/testDocumentation.py index 2789d2284..0ab5b15df 100644 --- a/documentation/testDocumentation.py +++ b/documentation/testDocumentation.py @@ -200,10 +200,6 @@ def main(runOne: str|bool = False): totalFailures = 0 timeStart = time.time() - unused_dtr = doctest.DocTestRunner(doctest.OutputChecker(), - verbose=False, - optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE) - for mt in getDocumentationFiles(runOne): # if 'examples' in mt.module: # continue diff --git a/music21/analysis/windowed.py b/music21/analysis/windowed.py index 370912c94..90ff56864 100644 --- a/music21/analysis/windowed.py +++ b/music21/analysis/windowed.py @@ -345,7 +345,7 @@ def process(self, # ----------------------------------------------------------------------------- -class TestMockProcessor: +class MockObjectProcessor: def process(self, subStream): ''' @@ -380,7 +380,7 @@ def testWindowing(self): ''' Test that windows are doing what they are supposed to do ''' - p = TestMockProcessor() + p = MockObjectProcessor() from music21 import note s1 = stream.Stream() diff --git a/music21/common/enums.py b/music21/common/enums.py index 634ba646c..fce3232e1 100644 --- a/music21/common/enums.py +++ b/music21/common/enums.py @@ -247,7 +247,7 @@ class AppendSpanners(StrEnum): AppendSpanners.NONE means do not append the related spanners at all (i.e. only append the object). - * new in v9. + * New in v9. ''' NORMAL = 'normal' RELATED_ONLY = 'related_only' @@ -262,7 +262,7 @@ class OrnamentDelay(StrEnum): OrnamentDelay.NO_DELAY means there is no delay (this is equivalent to setting delay to 0.0) OrnamentDelay.DEFAULT_DELAY means the delay is half the duration of the ornamented note. - * new in v9. + * New in v9. ''' NO_DELAY = 'noDelay' DEFAULT_DELAY = 'defaultDelay' diff --git a/music21/common/formats.py b/music21/common/formats.py index 230b67dfb..438476d56 100644 --- a/music21/common/formats.py +++ b/music21/common/formats.py @@ -141,21 +141,20 @@ def findFormat(fmt): >>> common.findFormat('wpd') (None, None) - - - These don't work but should eventually: - - # >>> common.findFormat('png') - # ('musicxml.png', '.png') - - # >>> common.findFormat('ipython') - # ('ipython', '.png') - # >>> common.findFormat('ipython.png') - # ('ipython', '.png') - - # >>> common.findFormat('musicxml.png') - # ('musicxml.png', '.png') ''' + # These don't work but should eventually: + # + # # >>> common.findFormat('png') + # # ('musicxml.png', '.png') + # + # # >>> common.findFormat('ipython') + # # ('ipython', '.png') + # # >>> common.findFormat('ipython.png') + # # ('ipython', '.png') + # + # # >>> common.findFormat('musicxml.png') + # # ('musicxml.png', '.png') + from music21 import converter c = converter.Converter() fileFormat = c.regularizeFormat(fmt) diff --git a/music21/common/parallel.py b/music21/common/parallel.py index 6703db6a1..f05ceeb46 100644 --- a/music21/common/parallel.py +++ b/music21/common/parallel.py @@ -10,18 +10,24 @@ from __future__ import annotations __all__ = [ + 'cpus', 'runParallel', 'runNonParallel', - 'cpus', + 'safeToParallize', ] import multiprocessing +import os import unittest -def runParallel(iterable, parallelFunction, *, - updateFunction=None, updateMultiply=3, - unpackIterable=False, updateSendsIterable=False): +def runParallel(iterable, + parallelFunction, + *, + updateFunction=None, + updateMultiply=3, + unpackIterable=False, + updateSendsIterable=False): ''' runs parallelFunction over iterable in parallel, optionally calling updateFunction after each common.cpus * updateMultiply calls. @@ -103,9 +109,7 @@ def runParallel(iterable, parallelFunction, *, >>> outputs [99, 11, 123] ''' - numCpus = cpus() - - if numCpus == 1 or multiprocessing.current_process().daemon: + if not safeToParallize(): return runNonParallel(iterable, parallelFunction, updateFunction=updateFunction, updateMultiply=updateMultiply, @@ -135,7 +139,7 @@ def callUpdate(ii): else: thisResult = resultsList[thisPosition] - if updateSendsIterable is False: + if not updateSendsIterable: updateFunction(thisPosition, iterLength, thisResult) else: updateFunction(thisPosition, iterLength, thisResult, iterable[thisPosition]) @@ -143,6 +147,7 @@ def callUpdate(ii): callUpdate(0) from joblib import Parallel, delayed # type: ignore + numCpus = cpus() with Parallel(n_jobs=numCpus) as para: delayFunction = delayed(parallelFunction) while totalRun < iterLength: @@ -161,9 +166,13 @@ def callUpdate(ii): return resultsList -def runNonParallel(iterable, parallelFunction, *, - updateFunction=None, updateMultiply=3, - unpackIterable=False, updateSendsIterable=False): +def runNonParallel(iterable, + parallelFunction, + *, + updateFunction=None, + updateMultiply=3, + unpackIterable=False, + updateSendsIterable=False): ''' This is intended to be a perfect drop in replacement for runParallel, except that it runs on one core only, and not in parallel. @@ -190,7 +199,7 @@ def callUpdate(ii): else: thisResult = resultsList[thisPosition] - if updateSendsIterable is False: + if not updateSendsIterable: updateFunction(thisPosition, iterLength, thisResult) else: updateFunction(thisPosition, iterLength, thisResult, iterable[thisPosition]) @@ -219,6 +228,22 @@ def cpus(): else: return cpuCount +def safeToParallize() -> bool: + ''' + Check to see if it is safe or even useful to start a parallel process. + + Will return False if we are in a multiprocessing child process or if + there is only one CPU or if pytest's x-dist worker flag is in the environment. + + * New in v10 + ''' + return ( + cpus() > 1 + and not multiprocessing.parent_process() + and 'PYTEST_XDIST_WORKER' not in os.environ + ) + + # Not shown to work. # def pickleCopy(obj): # ''' @@ -252,12 +277,15 @@ def x_figure_out_segfault_testMultiprocess(self): from music21.common.parallel import _countN, _countUnpacked output = runParallel(files, _countN) self.assertEqual(output, [165, 50, 131]) - runParallel(files, _countN, + runParallel(files, + _countN, updateFunction=self._customUpdate1) - runParallel(files, _countN, + runParallel(files, + _countN, updateFunction=self._customUpdate2, updateSendsIterable=True) - passed = runParallel(list(enumerate(files)), _countUnpacked, + passed = runParallel(list(enumerate(files)), + _countUnpacked, unpackIterable=True) self.assertEqual(len(passed), 3) self.assertNotIn(False, passed) diff --git a/music21/converter/__init__.py b/music21/converter/__init__.py index 71e4b9108..07aa68a70 100644 --- a/music21/converter/__init__.py +++ b/music21/converter/__init__.py @@ -27,7 +27,7 @@ faster since we store a parsed version of each file as a "pickle" object in the temp folder on the disk. ->>> #_DOCS_SHOW s = converter.parse('d:/myDocs/schubert.krn') +>>> #_DOCS_SHOW s = converter.parse('D:/myDocs/schubert.krn') >>> s = converter.parse(humdrum.testFiles.schubert) #_DOCS_HIDE >>> s @@ -426,7 +426,7 @@ def registerSubConverter(newSubConverter: type[subConverters.SubConverter]) -> N ... x, scf[x] ('abc', ) ... - ('sonix', ) + ('sonix', ) ... See `converter.qmConverter` for an example of an extended subConverter. @@ -444,14 +444,19 @@ def unregisterSubConverter( ''' Remove a SubConverter from the list of registered subConverters. + (Note that the list is a shared list across all Converters currently -- + that has long been considered a feature, but with multiprocessing, this + could change in the future.) + >>> converter.resetSubConverters() #_DOCS_HIDE >>> mxlConverter = converter.subConverters.ConverterMusicXML >>> c = converter.Converter() >>> mxlConverter in c.subConvertersList() True - >>> converter.unregisterSubConverter(mxlConverter) - >>> mxlConverter in c.subConvertersList() + >>> #_DOCS_SHOW converter.unregisterSubConverter(mxlConverter) + >>> #_DOCS_SHOW mxlConverter in c.subConvertersList() + >>> False #_DOCS_HIDE -- this breaks on parallel runs False If there is no such subConverter registered, and it is not a default subConverter, @@ -460,18 +465,25 @@ def unregisterSubConverter( >>> class ConverterSonix(converter.subConverters.SubConverter): ... registerFormats = ('sonix',) ... registerInputExtensions = ('mus',) + >>> converter.unregisterSubConverter(ConverterSonix) Traceback (most recent call last): - music21.converter.ConverterException: Could not remove from + music21.converter.ConverterException: Could not remove from registered subConverters The special command "all" removes everything including the default converters: - >>> converter.unregisterSubConverter('all') - >>> c.subConvertersList() + >>> #_DOCS_SHOW converter.unregisterSubConverter('all') + >>> #_DOCS_SHOW c.subConvertersList() + >>> [] #_DOCS_HIDE [] - >>> converter.resetSubConverters() #_DOCS_HIDE + Housekeeping: let's reset our subconverters and check things are okay again. + + >>> converter.resetSubConverters() + >>> c = converter.Converter() + >>> mxlConverter in c.subConvertersList() + True ''' if removeSubConverter == 'all': _registeredSubConverters.clear() @@ -594,7 +606,7 @@ def getFormatFromFileExtension(self, fp): else: useFormat = common.findFormatFile(fp) if useFormat is None: - raise ConverterFileException(f'cannot find a format extensions for: {fp}') + raise ConverterFileException(f'cannot find format from file extensions for: {fp}') return useFormat # noinspection PyShadowingBuiltins @@ -861,7 +873,7 @@ def subConvertersList( >>> ConverterSonix in c.subConvertersList() True - Newly registered subConveters appear first, so they will be used instead + Newly registered subConverters appear first, so they will be used instead of any default subConverters that work on the same format or extension. >>> class BadMusicXMLConverter(converter.subConverters.SubConverter): @@ -872,7 +884,7 @@ def subConvertersList( >>> converter.registerSubConverter(BadMusicXMLConverter) >>> c.subConvertersList() - [, + [, ... , ...] @@ -896,6 +908,8 @@ def subConvertersList( >>> #_DOCS_SHOW s = corpus.parse('beach/prayer_of_a_tired_child') >>> s.id 'empty' + >>> len(s.parts) + 0 >>> s = corpus.parse('beach/prayer_of_a_tired_child', forceSource=True) >>> len(s.parts) 6 @@ -1512,7 +1526,6 @@ def freezeStr(streamObj, fmt=None): the `fmt` argument; 'pickle' (the default), is the only one presently supported. - >>> c = converter.parse('tinyNotation: 4/4 c4 d e f', makeNotation=False) >>> c.show('text') {0.0} @@ -1530,7 +1543,6 @@ def freezeStr(streamObj, fmt=None): {1.0} {2.0} {3.0} - ''' from music21 import freezeThaw v = freezeThaw.StreamFreezer(streamObj) @@ -1570,7 +1582,8 @@ def testMusicXMLConversion(self): from music21.musicxml import testFiles for mxString in testFiles.ALL: a = subConverters.ConverterMusicXML() - a.parseData(mxString) + a.parseData(mxString.strip()) + break class TestExternal(unittest.TestCase): diff --git a/music21/features/base.py b/music21/features/base.py index 41a606540..c212ddf02 100644 --- a/music21/features/base.py +++ b/music21/features/base.py @@ -18,6 +18,7 @@ import unittest from music21 import common +from music21.common.parallel import safeToParallize from music21.common.types import StreamType from music21 import converter from music21 import corpus @@ -931,7 +932,7 @@ def process(self): Process all Data with all FeatureExtractors. Processed data is stored internally as numerous Feature objects. ''' - if self.runParallel: + if self.runParallel and safeToParallize(): return self._processParallel() else: return self._processNonParallel() @@ -947,10 +948,10 @@ def _processParallel(self): # print('about to run parallel') outputData = common.runParallel([(di, self.failFast) for di in self.dataInstances], - _dataSetParallelSubprocess, - updateFunction=shouldUpdate, - updateMultiply=1, - unpackIterable=True + _dataSetParallelSubprocess, + updateFunction=shouldUpdate, + updateMultiply=1, + unpackIterable=True ) featureData, errors, classValues, ids = zip(*outputData) errors = common.flattenList(errors) diff --git a/music21/figuredBass/harmony.py b/music21/figuredBass/harmony.py index c2f190ad6..96e5a97e9 100644 --- a/music21/figuredBass/harmony.py +++ b/music21/figuredBass/harmony.py @@ -60,7 +60,7 @@ class FiguredBass(Harmony): >>> fb.pitches () - * new in v9.3 + * New in v9.3 ''' def __init__(self, figureString: str = '', diff --git a/music21/metadata/__init__.py b/music21/metadata/__init__.py index c325113ee..63c348c63 100755 --- a/music21/metadata/__init__.py +++ b/music21/metadata/__init__.py @@ -2399,6 +2399,8 @@ def convertValue(uniqueName: str, value: t.Any) -> ValueType: >>> metadata.Metadata.convertValue('dateCreated', ... metadata.DateBetween(['1938', '1939'])) + + * Added in v10 -- newly exposed as a public function (was private) ''' valueType: type[ValueType]|None = properties.UNIQUE_NAME_TO_VALUE_TYPE.get( uniqueName, None diff --git a/music21/musicxml/testFiles.py b/music21/musicxml/testFiles.py index 842c0f2a9..3ee6e8b27 100644 --- a/music21/musicxml/testFiles.py +++ b/music21/musicxml/testFiles.py @@ -16066,8 +16066,7 @@ ''' -pianoRepeatEndings = r''' - +pianoRepeatEndings = r''' diff --git a/music21/prebase.py b/music21/prebase.py index 18f8a2982..65c4ae3be 100644 --- a/music21/prebase.py +++ b/music21/prebase.py @@ -43,7 +43,7 @@ class ProtoM21Object: >>> pc.classSet.isdisjoint(classList) True >>> repr(pc) - '' + '<...PitchCounter no pitches>' ProtoM21Objects, like other Python primitives, cannot be put into streams -- diff --git a/music21/spanner.py b/music21/spanner.py index ddc158865..43da40e5f 100644 --- a/music21/spanner.py +++ b/music21/spanner.py @@ -97,7 +97,7 @@ class Spanner(base.Music21Object): >>> for e in s: ... print(e) - > + <...CarterAccelerandoSign > @@ -107,13 +107,13 @@ class Spanner(base.Music21Object): >>> spannerCollection = s.spanners # a stream object >>> for thisSpanner in spannerCollection: ... print(thisSpanner) - > + <...CarterAccelerandoSign > (3) we can get the spanner by looking at the list getSpannerSites() on any object that has a spanner: >>> n2.getSpannerSites() - [>] In this example we will slur a few notes and then iterate over the stream to diff --git a/music21/test/pytest_plugin.py b/music21/test/pytest_plugin.py new file mode 100644 index 000000000..0a2640f00 --- /dev/null +++ b/music21/test/pytest_plugin.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import platform +import sys +from typing import Any +from unittest import TestCase + +import pytest # noqa # part of dev not running +try: + from _pytest.doctest import DoctestItem # noqa +except ImportError: + class DoctestItem: # type: ignore[no-redef] + pass + +import music21 +from music21.test.testRunner import fix312OrderedDict, stripAddresses + +@pytest.fixture(scope='session') +def doctest_namespace() -> dict[str, Any]: + ns: dict[str, Any] = {} + + all_names: list[str] = list(getattr(music21, '__all__', [])) + for name in all_names: + ns[name] = getattr(music21, name) + + # let doctests reference "music21" itself too. + ns['music21'] = music21 + return ns + +def pytest_collection_modifyitems(config, items) -> None: + ''' + Apply music21-style doctest normalizations to pytest-collected doctests. + + This mutates each doctest Example.want in-place, like fixDoctests() does + for doctest.DocTestSuite. + ''' + windows: bool = platform.system() == 'Windows' + is_python312: bool = sys.version_info >= (3, 12) + + kept = [] + for item in items: + parent = getattr(item, 'parent', None) + cls = getattr(parent, 'cls', None) + if cls is not None and issubclass(cls, TestCase) and cls.__name__ != 'Test': + # filter out TestSlow, TestExternal etc. + continue + if (getattr(item, 'dtest', None) + and item.dtest.name == 'music21.common.decorators.deprecated'): + # problem in pytest but not in doctest -- must run it once and therefore + # does not get the "first time run" vs "second time run" behavior. + continue + + kept.append(item) + + items[:] = kept + + for item in items: + if not isinstance(item, DoctestItem): + continue + + dt = item.dtest + # doctest.DocTest (pytest stores it here) + # (https://docs.pytest.org/en/stable/_modules/_pytest/doctest.html) + + for example in dt.examples: + example.want = stripAddresses(example.want, '0x...') + + if is_python312: + example.want = fix312OrderedDict(example.want, '...') + + if windows: + example.want = example.want.replace('PosixPath', 'WindowsPath') diff --git a/music21/test/testRunner.py b/music21/test/testRunner.py index 38e1954cb..40b3490c4 100644 --- a/music21/test/testRunner.py +++ b/music21/test/testRunner.py @@ -58,7 +58,7 @@ def addDocAttrTestsToSuite(suite, client () ''' dtp = doctest.DocTestParser() - if globs is False: + if not globs: globs = __import__(defaultImports[0]).__dict__.copy() elif globs is None: @@ -285,7 +285,7 @@ def testHello(self): testClasses = [] # remove cases for t in testClasses: if not isinstance(t, str): - if displayNames is True: + if displayNames: for tName in unittest.defaultTestLoader.getTestCaseNames(t): print(f'Unit Test Method: {tName}') if runThisTest is not None: @@ -322,7 +322,7 @@ def testHello(self): localVariables = list(outerFrame.f_locals.values()) addDocAttrTestsToSuite(s1, localVariables, outerFilename, globs, optionflags) - if runAllTests is True: + if runAllTests: fixDoctests(s1) runner = unittest.TextTestRunner() diff --git a/music21/test/test_base.py b/music21/test/test_base.py index 34d8d83a2..0674231cd 100644 --- a/music21/test/test_base.py +++ b/music21/test/test_base.py @@ -33,7 +33,7 @@ from music21 import tempo # ----------------------------------------------------------------------------- -class TestMock(Music21Object): +class MockObject(Music21Object): pass @@ -45,7 +45,7 @@ def testM21ObjRepr(self): self.assertEqual(repr(a), f'') def testObjectCreation(self): - a = TestMock() + a = MockObject() a.groups.append('hello') a.id = 'hi' a.offset = 2.0 diff --git a/pyproject.toml b/pyproject.toml index 0ff7bbd42..f55cf885f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -163,3 +163,7 @@ ignore = [ inline-quotes = 'single' multiline-quotes = 'single' docstring-quotes = 'single' + +[tool.pytest.ini_options] +addopts = ['--doctest-modules', '-p', 'music21.test.pytest_plugin'] +doctest_optionflags = ['NORMALIZE_WHITESPACE', 'ELLIPSIS']