From 61806275958b75c1ae185b7f52c51042839f22b1 Mon Sep 17 00:00:00 2001 From: Erwan Pannier Date: Tue, 31 Dec 2024 22:51:31 +0100 Subject: [PATCH 01/10] fix ruamel api --- ctwrap/parser.py | 13 +++++-------- environment.yml | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/ctwrap/parser.py b/ctwrap/parser.py index 993fda5..0399d0d 100644 --- a/ctwrap/parser.py +++ b/ctwrap/parser.py @@ -65,14 +65,10 @@ from typing import Optional, Dict, Any, Tuple, KeysView, Generator, Union from copy import deepcopy from pint import UnitRegistry +from ruamel.yaml import YAML import warnings import re -try: - import ruamel_yaml as yaml -except ImportError: - from ruamel import yaml - __all__ = ['Parser'] @@ -253,12 +249,13 @@ def from_yaml( elif path is not None: fname = Path(path) / fname + yaml = YAML(typ="rt") try: - _ = fname.is_file() # will raise error + _ = fname.is_file() # will raise error with open(fname) as stream: - out = yaml.load(stream, Loader=yaml.SafeLoader) + out = yaml.load(stream) except OSError: - out = yaml.load(yml, Loader=yaml.SafeLoader) + out = yaml.load(yml) if keys is None: return cls(out) diff --git a/environment.yml b/environment.yml index c67fe07..213a8a6 100644 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ dependencies: - numpy - h5py - pint - - ruamel.yaml + - ruamel.yaml>=0.17.0 - pandas - cantera - setuptools From 5e1ce74147b2c3804ccee2d32857f5c5448a30b7 Mon Sep 17 00:00:00 2001 From: Erwan Pannier Date: Tue, 31 Dec 2024 23:01:42 +0100 Subject: [PATCH 02/10] safe loader to mirror yaml.SafeLoader --- ctwrap/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctwrap/parser.py b/ctwrap/parser.py index 0399d0d..ae4f4dd 100644 --- a/ctwrap/parser.py +++ b/ctwrap/parser.py @@ -249,7 +249,7 @@ def from_yaml( elif path is not None: fname = Path(path) / fname - yaml = YAML(typ="rt") + yaml = YAML(typ="safe") try: _ = fname.is_file() # will raise error with open(fname) as stream: From e4c372cc6269a8824a2e42e3020c1345dece143e Mon Sep 17 00:00:00 2001 From: Erwan Pannier Date: Tue, 31 Dec 2024 23:11:15 +0100 Subject: [PATCH 03/10] fixing handler --- ctwrap/handler.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/ctwrap/handler.py b/ctwrap/handler.py index b4e188e..4ebb266 100644 --- a/ctwrap/handler.py +++ b/ctwrap/handler.py @@ -60,11 +60,7 @@ from multiprocessing import queues as mpq from multiprocessing import synchronize as mps import queue # imported for using queue.Empty exception - -try: - import ruamel_yaml as yaml -except ImportError: - from ruamel import yaml +from ruamel.yaml import YAML # ctwrap specific import from .parser import _parse, _write, Parser @@ -169,7 +165,8 @@ def from_yaml(cls, yaml_file: str, "".format(yaml_file)) with open(full_name) as stream: - content = yaml.load(stream, Loader=yaml.SafeLoader) + yaml = YAML(typ="safe") + content = yaml.load(stream) output = content.get('output', {}) From b0585b26e20ec6d876d04b397110569b66a26bcc Mon Sep 17 00:00:00 2001 From: Erwan Pannier Date: Tue, 31 Dec 2024 23:14:18 +0100 Subject: [PATCH 04/10] fixing dump --- ctwrap/parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ctwrap/parser.py b/ctwrap/parser.py index ae4f4dd..259fe75 100644 --- a/ctwrap/parser.py +++ b/ctwrap/parser.py @@ -264,6 +264,7 @@ def from_yaml( def to_yaml(self): """Convert Parser content to YAML string""" + yaml = YAML(typ="safe") return yaml.dump(self.raw, Dumper=yaml.SafeDumper) def get(self, key, default=None): From 935753a2efbb414c8e2ab4344f43a5ff6d54316a Mon Sep 17 00:00:00 2001 From: Erwan Pannier Date: Tue, 31 Dec 2024 23:23:19 +0100 Subject: [PATCH 05/10] fixed deprecation of pandas.Dataframe.append --- ctwrap/output.py | 2 +- environment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ctwrap/output.py b/ctwrap/output.py index 459ccdf..e310d3d 100644 --- a/ctwrap/output.py +++ b/ctwrap/output.py @@ -236,7 +236,7 @@ def save(self, data, entry, variation=None, mode=None, errored=False): df = pd.read_csv(fname) else: df = pd.DataFrame(columns=row.keys()) - df = df.append(row, ignore_index=True) + df = pd.concat([df, row], ignore_index=True) df.to_csv(fname, index=False) def dir(self): diff --git a/environment.yml b/environment.yml index 213a8a6..c659baa 100644 --- a/environment.yml +++ b/environment.yml @@ -10,7 +10,7 @@ dependencies: - h5py - pint - ruamel.yaml>=0.17.0 - - pandas + - pandas>=1.4 - cantera - setuptools - pytest From 35359a915b0d95601037cbd48d53004cf872d542 Mon Sep 17 00:00:00 2001 From: Erwan Pannier Date: Wed, 1 Jan 2025 00:00:00 +0100 Subject: [PATCH 06/10] fix pkg_resources dependency, pint, and all tests --- ctwrap/output.py | 9 ++++----- tests/test_output.py | 10 +++++----- tests/test_parser.py | 10 ++++------ tests/test_strategy.py | 6 ------ tests/test_wrapper.py | 13 +++++-------- 5 files changed, 18 insertions(+), 30 deletions(-) diff --git a/ctwrap/output.py b/ctwrap/output.py index e310d3d..b588f29 100644 --- a/ctwrap/output.py +++ b/ctwrap/output.py @@ -14,15 +14,14 @@ from typing import Dict, List, Any, Optional, Union -import pkg_resources +import importlib # avoid explicit dependence on cantera -try: - pkg_resources.get_distribution('cantera') -except pkg_resources.DistributionNotFound: +ct_spec = importlib.util.find_spec("cantera") +if ct_spec is None: ct = ImportError('Method requires a working cantera installation.') else: - import cantera as ct + ct = importlib.import_module("cantera") class Output: diff --git a/tests/test_output.py b/tests/test_output.py index a788c7a..feea395 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -21,15 +21,14 @@ # pylint: disable=no-member import ctwrap.output as cwo -import pkg_resources +import importlib # avoid explicit dependence on cantera -try: - pkg_resources.get_distribution('cantera') -except pkg_resources.DistributionNotFound: +ct_spec = importlib.util.find_spec("cantera") +if ct_spec is None: ct = ImportError('Method requires a working cantera installation.') else: - import cantera as ct + ct = importlib.import_module("cantera") PWD = Path(__file__).parents[0] @@ -46,6 +45,7 @@ class TestOutput(unittest.TestCase): @classmethod def setUpClass(cls): with open(EXAMPLES / cls._yaml) as stream: + yaml = YAML(typ='safe') cls._config = yaml.load(stream, Loader=yaml.SafeLoader) out = cls._config.get('output') diff --git a/tests/test_parser.py b/tests/test_parser.py index 09c8d4b..0dcb582 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -5,12 +5,8 @@ import unittest from pathlib import Path -import pint.quantity as pq - -try: - import ruamel_yaml as yaml -except ImportError: - from ruamel import yaml +import pint as pq +from ruamel.yaml import YAML import warnings # add exception as pywintypes imports a deprecated module @@ -202,6 +198,7 @@ class TestYAML(unittest.TestCase): def test_minimal(self): with open(EXAMPLES / 'minimal.yaml') as stream: + yaml = YAML(typ='safe') defaults = yaml.load(stream, Loader=yaml.SafeLoader) p = cw.Parser.from_yaml('minimal.yaml', path=EXAMPLES) self.assertEqual(len(p), len(defaults)) @@ -216,6 +213,7 @@ def test_minimal(self): def test_ignition(self): with open(EXAMPLES / 'ignition.yaml') as stream: + yaml = YAML(typ='safe') yml = yaml.load(stream, Loader=yaml.SafeLoader) initial = cw.Parser(yml['defaults']['initial']) self.assertIsInstance(initial.T, pq.Quantity) diff --git a/tests/test_strategy.py b/tests/test_strategy.py index 763c8b7..6634a24 100644 --- a/tests/test_strategy.py +++ b/tests/test_strategy.py @@ -5,12 +5,6 @@ import unittest from pathlib import Path -import pint.quantity as pq - -try: - import ruamel_yaml as yaml -except ImportError: - from ruamel import yaml import warnings # add exception as pywintypes imports a deprecated module diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index bfe25a7..22c6839 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -6,14 +6,9 @@ import unittest from pathlib import Path import subprocess -import pint.quantity as pq import importlib import h5py - -try: - import ruamel_yaml as yaml -except ImportError: - from ruamel import yaml +from ruamel.yaml import YAML import warnings # add exception as pywintypes imports a deprecated module @@ -54,7 +49,8 @@ class TestWrap(unittest.TestCase): def setUp(self): with open(EXAMPLES / self._yaml) as stream: - self.config = yaml.load(stream, Loader=yaml.SafeLoader) + yaml = YAML(typ='safe') + self.config = yaml.load(stream) self.sim = cw.Simulation.from_module(self._module) self.sh = cw.SimulationHandler.from_yaml(self._yaml, strategy=self._strategy, database=EXAMPLES) @@ -247,7 +243,8 @@ class TestInvalid(TestWrap): def setUp(self): with open(EXAMPLES / self._yaml) as stream: - self.config = yaml.load(stream, Loader=yaml.SafeLoader) + yaml = YAML(typ='safe') + self.config = yaml.load(stream) self.sim = cw.Simulation.from_module(self._module) self.sh = cw.SimulationHandler.from_dict(self._dict) From 3df62e7d453c4189e5a34ff45cbe04dffa687161 Mon Sep 17 00:00:00 2001 From: Erwan Pannier Date: Wed, 1 Jan 2025 01:01:38 +0100 Subject: [PATCH 07/10] fix pandas & output --- ctwrap/output.py | 3 ++- tests/test_output.py | 8 ++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/ctwrap/output.py b/ctwrap/output.py index b588f29..75f9efc 100644 --- a/ctwrap/output.py +++ b/ctwrap/output.py @@ -235,7 +235,7 @@ def save(self, data, entry, variation=None, mode=None, errored=False): df = pd.read_csv(fname) else: df = pd.DataFrame(columns=row.keys()) - df = pd.concat([df, row], ignore_index=True) + df = pd.concat([df, row.to_frame().T], ignore_index=True) df.to_csv(fname, index=False) def dir(self): @@ -245,6 +245,7 @@ def dir(self): return [] df = pd.read_csv(fname) + print("DEBUG dir df", df) return list(df.output) def load_like(self, entry, other): diff --git a/tests/test_output.py b/tests/test_output.py index feea395..716a5ed 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -7,11 +7,7 @@ import unittest from pathlib import Path import h5py - -try: - import ruamel_yaml as yaml -except ImportError: - from ruamel import yaml +from ruamel.yaml import YAML import warnings # add exception as pywintypes imports a deprecated module @@ -46,7 +42,7 @@ class TestOutput(unittest.TestCase): def setUpClass(cls): with open(EXAMPLES / cls._yaml) as stream: yaml = YAML(typ='safe') - cls._config = yaml.load(stream, Loader=yaml.SafeLoader) + cls._config = yaml.load(stream) out = cls._config.get('output') if out: From 68fcc2d09947503708d30b14f1d77869e0065952 Mon Sep 17 00:00:00 2001 From: Erwan Pannier Date: Wed, 1 Jan 2025 01:13:49 +0100 Subject: [PATCH 08/10] typo --- ctwrap/output.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ctwrap/output.py b/ctwrap/output.py index 75f9efc..4b88ab1 100644 --- a/ctwrap/output.py +++ b/ctwrap/output.py @@ -245,7 +245,6 @@ def dir(self): return [] df = pd.read_csv(fname) - print("DEBUG dir df", df) return list(df.output) def load_like(self, entry, other): From 9b2457f4ded8adb254897251dd94b81594e96fc7 Mon Sep 17 00:00:00 2001 From: Erwan Pannier Date: Wed, 1 Jan 2025 01:20:41 +0100 Subject: [PATCH 09/10] fix SafeLoader failures --- tests/test_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_parser.py b/tests/test_parser.py index 0dcb582..a476a5f 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -199,7 +199,7 @@ def test_minimal(self): with open(EXAMPLES / 'minimal.yaml') as stream: yaml = YAML(typ='safe') - defaults = yaml.load(stream, Loader=yaml.SafeLoader) + defaults = yaml.load(stream) p = cw.Parser.from_yaml('minimal.yaml', path=EXAMPLES) self.assertEqual(len(p), len(defaults)) self.assertEqual(p.keys(), defaults.keys()) @@ -214,7 +214,7 @@ def test_ignition(self): with open(EXAMPLES / 'ignition.yaml') as stream: yaml = YAML(typ='safe') - yml = yaml.load(stream, Loader=yaml.SafeLoader) + yml = yaml.load(stream) initial = cw.Parser(yml['defaults']['initial']) self.assertIsInstance(initial.T, pq.Quantity) self.assertIsInstance(initial['fuel'], str) From 210de751a5550c5f3f3f49e6297364863b6d3f07 Mon Sep 17 00:00:00 2001 From: Erwan Pannier Date: Wed, 19 Feb 2025 10:31:39 +0100 Subject: [PATCH 10/10] fix problems when reading pd.Series "(if X and pd.Series) was creating ambiguity errors in the evaluation of True/False of the Series" --- ctwrap/handler.py | 3 ++- ctwrap/output.py | 5 ++++- ctwrap/parser.py | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ctwrap/handler.py b/ctwrap/handler.py index 4ebb266..f4c695c 100644 --- a/ctwrap/handler.py +++ b/ctwrap/handler.py @@ -464,6 +464,7 @@ def _worker( else: out = None + data = None other = None reschedule = 0 while reschedule < len(tasks): @@ -522,7 +523,7 @@ def _worker( errored = True # save output - if out and obj.data: + if out is not None and data is not None: if parallel: with lock: out.save(data, entry=task, variation=variations[task], errored=errored) diff --git a/ctwrap/output.py b/ctwrap/output.py index 4b88ab1..1dfb720 100644 --- a/ctwrap/output.py +++ b/ctwrap/output.py @@ -194,7 +194,7 @@ class WriteCSV(Output): def save(self, data, entry, variation=None, mode=None, errored=False): "" - if not data: + if data is None: return returns = self.kwargs.get('returns') @@ -238,6 +238,9 @@ def save(self, data, entry, variation=None, mode=None, errored=False): df = pd.concat([df, row.to_frame().T], ignore_index=True) df.to_csv(fname, index=False) + else: + raise NotImplementedError("Saving of '{}' not implemented".format(type(data).__name__)) + def dir(self): "" fname = Path(self.output_name) diff --git a/ctwrap/parser.py b/ctwrap/parser.py index 259fe75..954294b 100644 --- a/ctwrap/parser.py +++ b/ctwrap/parser.py @@ -85,6 +85,7 @@ def _parse(val: str): Returns: `Tuple` containing value and unit """ + # TODO :seems parser does not handle "e" scientific notation e.g; "1e-6 m**3" is not parsed as dimensioned unit. if not isinstance(val, str): raise TypeError("Method requires string input") @@ -254,6 +255,8 @@ def from_yaml( _ = fname.is_file() # will raise error with open(fname) as stream: out = yaml.load(stream) + except FileNotFoundError: + raise FileNotFoundError("File '{}' not found".format(fname)) except OSError: out = yaml.load(yml)