From 4107bbd6e1a8c5b150ccf8b7a3708369050f82bd Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 9 Jul 2025 15:21:18 -0700 Subject: [PATCH 01/17] tmp: temporarily points the dandischema dependency to the `devendorize` branch This commit should be dropped before the containing PR is merged. This is a temporary measure to test the behavior of `dandischema` with the changes in https://github.com/dandi/dandi-schema/pull/294 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 5efe08400..9c3240a80 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ install_requires = # >=8.2.0: https://github.com/pallets/click/issues/2911 click >= 7.1, <8.2.0 click-didyoumean - dandischema >= 0.11.0, < 0.12.0 + dandischema @ git+https://github.com/dandi/dandi-schema.git@devendorize etelemetry >= 0.2.2 # For pydantic to be able to use type annotations like `X | None` eval_type_backport; python_version < "3.10" From af36aa028cfb60200b4c169fe7f78941dd8d13a1 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 10 Jul 2025 21:43:46 -0700 Subject: [PATCH 02/17] feat: expand `ServerInfo` to conclude `instance_config` --- dandi/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dandi/utils.py b/dandi/utils.py index 06e461fbc..caeac7ac2 100644 --- a/dandi/utils.py +++ b/dandi/utils.py @@ -26,6 +26,7 @@ import types from typing import IO, Any, List, Optional, Protocol, TypeVar, Union +from dandischema.conf import Config as InstanceConfig import dateutil.parser from multidict import MultiDict # dependency of yarl from pydantic import BaseModel, Field @@ -546,6 +547,7 @@ class ServerServices(BaseModel): class ServerInfo(BaseModel): + instance_config: InstanceConfig # schema_version: str # schema_url: str version: str From 7b15519b56b759e6bdabeea045a262f5884c40fe Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 10 Jul 2025 23:07:04 -0700 Subject: [PATCH 03/17] feat: provide func to fetch server info --- dandi/cli/base.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/dandi/cli/base.py b/dandi/cli/base.py index 30fe73247..e88a6367c 100644 --- a/dandi/cli/base.py +++ b/dandi/cli/base.py @@ -2,11 +2,47 @@ import os import click +import requests +from yarl import URL + +from dandi.consts import known_instances +from dandi.utils import ServerInfo from .. import get_logger lgr = get_logger() + +def get_server_info(dandi_id: str) -> ServerInfo: + """ + Get server info from a particular DANDI instance + + Parameters + ---------- + dandi_id : str + The ID specifying the particular known DANDI instance to query for server info. + This is a key in the `dandi.consts.known_instances` dictionary. + + Returns + ------- + ServerInfo + An object representing the server information responded by the DANDI instance. + + Raises + ------ + valueError + If the provided `dandi_id` is not a valid key in the + `dandi.consts.known_instances` dictionary. + """ + if dandi_id not in known_instances: + raise ValueError(f"Unknown DANDI instance: {dandi_id}") + + info_url = str(URL(known_instances[dandi_id].api) / "info/") + resp = requests.get(info_url) + resp.raise_for_status() + return ServerInfo.model_validate(resp.json()) + + # Aux common functionality From e28b6e761b9c10b3b58f56c271d654cad1ffbd3a Mon Sep 17 00:00:00 2001 From: Isaac To Date: Fri, 11 Jul 2025 11:34:34 -0700 Subject: [PATCH 04/17] feat: provide func to bind client to server This function binds the DANDI client to a specific DANDI server instance so that subsequence commands executed by the client is executed in the context of the DANDI sever instance --- dandi/cli/base.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dandi/cli/base.py b/dandi/cli/base.py index e88a6367c..81dd21f63 100644 --- a/dandi/cli/base.py +++ b/dandi/cli/base.py @@ -2,6 +2,7 @@ import os import click +from dandischema.conf import set_instance_config import requests from yarl import URL @@ -43,6 +44,20 @@ def get_server_info(dandi_id: str) -> ServerInfo: return ServerInfo.model_validate(resp.json()) +def bind_client(server_info: ServerInfo) -> None: + """ + Bind the DANDI client to a specific DANDI server instance. I.e., to set the DANDI + server instance as the context of subsequent command executions by the DANDI client + + Parameters + ---------- + server_info : ServerInfo + An object containing the information of the DANDI server instance to bind to. + This is typically obtained by calling `get_server_info()`. + """ + set_instance_config(server_info.instance_config) + + # Aux common functionality From 04f7c26207fe383ab1fcf6531cd56b0a5fdfc74a Mon Sep 17 00:00:00 2001 From: Isaac To Date: Fri, 11 Jul 2025 12:09:08 -0700 Subject: [PATCH 05/17] feat: provide func to initialize the DANDI client This function should be run before a command is executed so that it can set the context of the execution such as which DANDI sever instance to interact with. --- dandi/cli/base.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dandi/cli/base.py b/dandi/cli/base.py index 81dd21f63..0f54a1769 100644 --- a/dandi/cli/base.py +++ b/dandi/cli/base.py @@ -58,6 +58,26 @@ def bind_client(server_info: ServerInfo) -> None: set_instance_config(server_info.instance_config) +def init_client(dandi_id: str) -> None: + """ + Initialize the DANDI client, including binding the client to a specific DANDI server + instance + + Parameters + ---------- + dandi_id : str + The ID specifying the particular known DANDI instance to bind the client to. + This is a key in the `dandi.consts.known_instances` dictionary. + + Raises + ------ + ValueError + If the provided `dandi_id` is not a valid key in the + `dandi.consts.known_instances` dictionary. + """ + bind_client(get_server_info(dandi_id)) + + # Aux common functionality From 50a67ca9741102e86edb0c4f0d8fb24af04e0d68 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 17 Jul 2025 01:17:01 -0700 Subject: [PATCH 06/17] feat: delay import of `dandischema.models` in `dandi.dandiapi` --- dandi/dandiapi.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 553ec672c..cb411130b 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -17,7 +17,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional import click -from dandischema import models from pydantic import BaseModel, Field, PrivateAttr import requests import tenacity @@ -36,7 +35,6 @@ ) from .exceptions import HTTP404Error, NotFoundError, SchemaVersionError from .keyring import keyring_lookup, keyring_save -from .misctypes import Digest, RemoteReadableAsset from .utils import ( USER_AGENT, check_dandi_version, @@ -50,8 +48,11 @@ ) if TYPE_CHECKING: + from dandischema import models from typing_extensions import Self + from .misctypes import Digest, RemoteReadableAsset + lgr = get_logger() @@ -653,6 +654,8 @@ def check_schema_version(self, schema_version: str | None = None) -> None: uses; if not set, the schema version for the installed ``dandischema`` library is used """ + from dandischema import models + if schema_version is None: schema_version = models.get_schema_version() server_info = self.get("/info/") @@ -1065,6 +1068,8 @@ def get_metadata(self) -> models.Dandiset: metadata. Consider using `get_raw_metadata()` instead in order to fetch unstructured, possibly-invalid metadata. """ + from dandischema import models + return models.Dandiset.model_validate(self.get_raw_metadata()) def get_raw_metadata(self) -> dict[str, Any]: @@ -1469,6 +1474,8 @@ def get_metadata(self) -> models.Asset: valid metadata. Consider using `get_raw_metadata()` instead in order to fetch unstructured, possibly-invalid metadata. """ + from dandischema import models + return models.Asset.model_validate(self.get_raw_metadata()) def get_raw_metadata(self) -> dict[str, Any]: @@ -1495,6 +1502,8 @@ def get_raw_digest(self, digest_type: str | models.DigestType | None = None) -> .. versionchanged:: 0.36.0 Renamed from ``get_digest()`` to ``get_raw_digest()`` """ + from dandischema import models + if digest_type is None: digest_type = self.digest_type.value elif isinstance(digest_type, models.DigestType): @@ -1517,6 +1526,8 @@ def get_digest(self) -> Digest: a dandi-etag digest for blob resources or a dandi-zarr-checksum for Zarr resources """ + from .misctypes import Digest + algorithm = self.digest_type return Digest(algorithm=algorithm, value=self.get_raw_digest(algorithm)) @@ -1651,6 +1662,8 @@ def digest_type(self) -> models.DigestType: determined based on its underlying data: dandi-etag for blob resources, dandi-zarr-checksum for Zarr resources """ + from dandischema import models + if self.asset_type is AssetType.ZARR: return models.DigestType.dandi_zarr_checksum else: @@ -1683,6 +1696,8 @@ def as_readable(self) -> RemoteReadableAsset: Returns a `Readable` instance that can be used to obtain a file-like object for reading bytes directly from the asset on the server """ + from .misctypes import RemoteReadableAsset + md = self.get_raw_metadata() local_prefix = self.client.api_url.lower() for url in md.get("contentUrl", []): @@ -1965,6 +1980,10 @@ def from_server_data( cls, asset: BaseRemoteZarrAsset, data: ZarrEntryServerData ) -> RemoteZarrEntry: """:meta private:""" + from dandischema import models + + from .misctypes import Digest + return cls( client=asset.client, zarr_id=asset.zarr, From c6aec313a5f8169fe82eccca5acdd313b169d15a Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 17 Jul 2025 13:32:18 -0700 Subject: [PATCH 07/17] feat: delay import of `dandischema.models` in `dandi.files.bases` --- dandi/files/bases.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/dandi/files/bases.py b/dandi/files/bases.py index 15bc616c4..fa2754228 100644 --- a/dandi/files/bases.py +++ b/dandi/files/bases.py @@ -10,15 +10,12 @@ from pathlib import Path import re from threading import Lock -from typing import IO, Any, Generic +from typing import IO, TYPE_CHECKING, Any, Generic from xml.etree.ElementTree import fromstring import dandischema from dandischema.consts import DANDI_SCHEMA_VERSION from dandischema.digests.dandietag import DandiETag -from dandischema.models import BareAsset, CommonModel -from dandischema.models import Dandiset as DandisetMeta -from dandischema.models import get_schema_version from packaging.version import Version from pydantic import ValidationError from pydantic_core import ErrorDetails @@ -41,6 +38,10 @@ Validator, ) +if TYPE_CHECKING: + from dandischema.models import BareAsset, CommonModel + from dandischema.models import Dandiset as DandisetMeta + lgr = dandi.get_logger() # TODO -- should come from schema. This is just a simplistic example for now @@ -107,6 +108,8 @@ def get_metadata( ignore_errors: bool = True, ) -> DandisetMeta: """Return the Dandiset metadata inside the file""" + from dandischema.models import Dandiset as DandisetMeta + with open(self.filepath) as f: meta = yaml_load(f, typ="safe") return DandisetMeta.model_construct(**meta) @@ -126,6 +129,9 @@ def get_validation_errors( meta, _required_dandiset_metadata_fields, str(self.filepath) ) else: + from dandischema.models import Dandiset as DandisetMeta + from dandischema.models import get_schema_version + current_version = get_schema_version() if schema_version != current_version: raise ValueError( @@ -186,6 +192,8 @@ def get_validation_errors( schema_version: str | None = None, devel_debug: bool = False, ) -> list[ValidationResult]: + from dandischema.models import BareAsset, get_schema_version + current_version = get_schema_version() if schema_version is None: schema_version = current_version From a437bcb3e50abb9cf8c70a5b3313129657a8e772 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 17 Jul 2025 15:02:30 -0700 Subject: [PATCH 08/17] feat: delay import of `dandischema.models` in `dandi.metadata.core` --- dandi/metadata/core.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/dandi/metadata/core.py b/dandi/metadata/core.py index 12a53264e..b77afabe0 100644 --- a/dandi/metadata/core.py +++ b/dandi/metadata/core.py @@ -3,21 +3,26 @@ from datetime import datetime from pathlib import Path import re +from typing import TYPE_CHECKING -from dandischema import models from pydantic import ByteSize -from .util import extract_model, get_generator from .. import get_logger -from ..misctypes import Digest, LocalReadableFile, Readable from ..utils import get_mime_type, get_utcnow_datetime +if TYPE_CHECKING: + from dandischema import models + + from ..misctypes import Digest, Readable + lgr = get_logger() def get_default_metadata( path: str | Path | Readable, digest: Digest | None = None ) -> models.BareAsset: + from dandischema import models + metadata = models.BareAsset.model_construct() # type: ignore[call-arg] start_time = end_time = datetime.now().astimezone() add_common_metadata(metadata, path, start_time, end_time, digest) @@ -35,6 +40,11 @@ def add_common_metadata( Update a `dict` of raw "schemadata" with the fields that are common to both NWB assets and non-NWB assets """ + from dandischema import models + + from .util import get_generator + from ..misctypes import LocalReadableFile, Readable + if digest is not None: metadata.digest = digest.asdict() else: @@ -73,4 +83,8 @@ def prepare_metadata(metadata: dict) -> models.BareAsset: .. [2] metadata in the form used by the ``dandischema`` library """ + from dandischema import models + + from .util import extract_model + return extract_model(models.BareAsset, metadata) From 7503e4bd1e8953ceacf5b47f7556530fe736ec01 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Fri, 18 Jul 2025 15:42:38 -0700 Subject: [PATCH 09/17] feat: delay import of `dandischema.models` in `dandi.files.bases` --- dandi/files/bases.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/dandi/files/bases.py b/dandi/files/bases.py index fa2754228..7297d9624 100644 --- a/dandi/files/bases.py +++ b/dandi/files/bases.py @@ -10,7 +10,7 @@ from pathlib import Path import re from threading import Lock -from typing import IO, TYPE_CHECKING, Any, Generic +from typing import IO, TYPE_CHECKING, Any, Generic, TypeVar from xml.etree.ElementTree import fromstring import dandischema @@ -24,7 +24,6 @@ import dandi from dandi.dandiapi import RemoteAsset, RemoteDandiset, RESTFullAPIClient from dandi.metadata.core import get_default_metadata -from dandi.misctypes import DUMMY_DANDI_ETAG, Digest, LocalReadableFile, P from dandi.utils import post_upload_size_check, pre_upload_size_check, yaml_load from dandi.validate_types import ( ORIGIN_INTERNAL_DANDI, @@ -42,6 +41,11 @@ from dandischema.models import BareAsset, CommonModel from dandischema.models import Dandiset as DandisetMeta + # noinspection PyUnresolvedReferences + from dandi.misctypes import BasePath, Digest, LocalReadableFile + +P = TypeVar("P", bound="BasePath") + lgr = dandi.get_logger() # TODO -- should come from schema. This is just a simplistic example for now @@ -153,6 +157,8 @@ def as_readable(self) -> LocalReadableFile: Returns a `Readable` instance wrapping the local file """ + from dandi.misctypes import LocalReadableFile + return LocalReadableFile(self.filepath) @@ -167,7 +173,11 @@ class LocalAsset(DandiFile): #: (i.e., relative to the Dandiset's root) path: str - _DUMMY_DIGEST = DUMMY_DANDI_ETAG + @staticmethod + def _get_dummy_digest() -> Digest: + from dandi.misctypes import DUMMY_DANDI_ETAG + + return DUMMY_DANDI_ETAG @abstractmethod def get_digest(self) -> Digest: @@ -202,7 +212,7 @@ def get_validation_errors( f"Unsupported schema version: {schema_version}; expected {current_version}" ) try: - asset = self.get_metadata(digest=self._DUMMY_DIGEST) + asset = self.get_metadata(digest=self._get_dummy_digest()) BareAsset(**asset.model_dump()) except ValidationError as e: if devel_debug: @@ -317,6 +327,7 @@ def get_metadata( def get_digest(self) -> Digest: """Calculate a dandi-etag digest for the asset""" # Avoid heavy import by importing within function: + from dandi.misctypes import Digest from dandi.support.digests import get_digest value = get_digest(self.filepath, digest="dandi-etag") @@ -484,6 +495,8 @@ def as_readable(self) -> LocalReadableFile: Returns a `Readable` instance wrapping the local file """ + from dandi.misctypes import LocalReadableFile + return LocalReadableFile(self.filepath) From f21d8483f2a3d99f67c14088b9b1ccc6dc9b34eb Mon Sep 17 00:00:00 2001 From: Isaac To Date: Sun, 20 Jul 2025 11:34:25 -0700 Subject: [PATCH 10/17] feat: delay import of `dandi.misctypes.DUMMY_DANDI_ZARR_CHECKSUM` in `dandi.files.zarr.py` This is a step in delaying import of `schema.models` in `dandi.files.zarr.py` --- dandi/files/zarr.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dandi/files/zarr.py b/dandi/files/zarr.py index 6220e2294..226de031a 100644 --- a/dandi/files/zarr.py +++ b/dandi/files/zarr.py @@ -34,7 +34,7 @@ RESTFullAPIClient, ) from dandi.metadata.core import get_default_metadata -from dandi.misctypes import DUMMY_DANDI_ZARR_CHECKSUM, BasePath, Digest +from dandi.misctypes import BasePath, Digest from dandi.utils import ( chunked, exclude_from_zarr, @@ -363,7 +363,11 @@ class ZarrStat: class ZarrAsset(LocalDirectoryAsset[LocalZarrEntry]): """Representation of a local Zarr directory""" - _DUMMY_DIGEST = DUMMY_DANDI_ZARR_CHECKSUM + @staticmethod + def _get_dummy_digest() -> Digest: + from dandi.misctypes import DUMMY_DANDI_ZARR_CHECKSUM + + return DUMMY_DANDI_ZARR_CHECKSUM @property def filetree(self) -> LocalZarrEntry: From eb963904b33e96badbce7cbfa7394c1c35d019a9 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Sun, 20 Jul 2025 12:55:34 -0700 Subject: [PATCH 11/17] feat: replace `Digest` constants in `misctypes.py` with cache funcs Replace`DUMMY_DANDI_ETAG` and `DUMMY_DANDI_ZARR_CHECKSUM` in `dandi.misctypes` with caching functions. This change is needed to delay the import of `dandischema.models` in `dandi.misctypes` and in this package in general --- dandi/files/bases.py | 4 ++-- dandi/files/zarr.py | 4 ++-- dandi/metadata/nwb.py | 4 ++-- dandi/misctypes.py | 15 ++++++++++----- dandi/tests/test_metadata.py | 6 ++++-- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/dandi/files/bases.py b/dandi/files/bases.py index 7297d9624..d1c547959 100644 --- a/dandi/files/bases.py +++ b/dandi/files/bases.py @@ -175,9 +175,9 @@ class LocalAsset(DandiFile): @staticmethod def _get_dummy_digest() -> Digest: - from dandi.misctypes import DUMMY_DANDI_ETAG + from dandi.misctypes import get_dummy_dandi_etag - return DUMMY_DANDI_ETAG + return get_dummy_dandi_etag() @abstractmethod def get_digest(self) -> Digest: diff --git a/dandi/files/zarr.py b/dandi/files/zarr.py index 226de031a..822c08c45 100644 --- a/dandi/files/zarr.py +++ b/dandi/files/zarr.py @@ -365,9 +365,9 @@ class ZarrAsset(LocalDirectoryAsset[LocalZarrEntry]): @staticmethod def _get_dummy_digest() -> Digest: - from dandi.misctypes import DUMMY_DANDI_ZARR_CHECKSUM + from dandi.misctypes import get_dummy_dandi_zarr_checksum - return DUMMY_DANDI_ZARR_CHECKSUM + return get_dummy_dandi_zarr_checksum() @property def filetree(self) -> LocalZarrEntry: diff --git a/dandi/metadata/nwb.py b/dandi/metadata/nwb.py index 38c2ac4e7..1413e2917 100644 --- a/dandi/metadata/nwb.py +++ b/dandi/metadata/nwb.py @@ -13,7 +13,7 @@ from .. import get_logger from ..consts import metadata_all_fields from ..files import bids, dandi_file, find_bids_dataset_description -from ..misctypes import DUMMY_DANDI_ETAG, Digest, LocalReadableFile, Readable +from ..misctypes import Digest, LocalReadableFile, Readable, get_dummy_dandi_etag from ..pynwb_utils import ( _get_pynwb_metadata, get_neurodata_types, @@ -68,7 +68,7 @@ def get_metadata( ) assert isinstance(df, bids.BIDSAsset) if not digest: - digest = DUMMY_DANDI_ETAG + digest = get_dummy_dandi_etag() path_metadata = df.get_metadata(digest=digest) meta["bids_version"] = df.get_validation_bids_version() # there might be a more elegant way to do this: diff --git a/dandi/misctypes.py b/dandi/misctypes.py index 9ed2dcf07..22c1cca3f 100644 --- a/dandi/misctypes.py +++ b/dandi/misctypes.py @@ -11,6 +11,7 @@ from dataclasses import dataclass from datetime import datetime from fnmatch import fnmatchcase +from functools import cache import os.path from pathlib import Path from typing import IO, TypeVar, cast @@ -54,11 +55,15 @@ def asdict(self) -> dict[DigestType, str]: #: Placeholder digest used in some situations where a digest is required but #: not actually relevant and would be too expensive to calculate -DUMMY_DANDI_ETAG = Digest(algorithm=DigestType.dandi_etag, value=32 * "d" + "-1") -DUMMY_DANDI_ZARR_CHECKSUM = Digest( - algorithm=DigestType.dandi_zarr_checksum, - value=32 * "d" + "-1--1", -) +@cache +def get_dummy_dandi_etag() -> Digest: + return Digest(algorithm=DigestType.dandi_etag, value=32 * "d" + "-1") + + +@cache +def get_dummy_dandi_zarr_checksum() -> Digest: + return Digest(algorithm=DigestType.dandi_zarr_checksum, value=32 * "d" + "-1--1") + P = TypeVar("P", bound="BasePath") diff --git a/dandi/tests/test_metadata.py b/dandi/tests/test_metadata.py index 5c20acf0d..2bd2ff0fb 100644 --- a/dandi/tests/test_metadata.py +++ b/dandi/tests/test_metadata.py @@ -49,7 +49,7 @@ species_map, timedelta2duration, ) -from ..misctypes import DUMMY_DANDI_ETAG +from ..misctypes import get_dummy_dandi_etag from ..utils import ensure_datetime METADATA_DIR = Path(__file__).with_name("data") / "metadata" @@ -836,7 +836,9 @@ def test_ndtypes(ndtypes, asset_dict): def test_nwb2asset(simple2_nwb: Path) -> None: # Classes with ANY_AWARE_DATETIME fields need to be constructed with # model_construct() - assert nwb2asset(simple2_nwb, digest=DUMMY_DANDI_ETAG) == BareAsset.model_construct( + assert nwb2asset( + simple2_nwb, digest=get_dummy_dandi_etag() + ) == BareAsset.model_construct( schemaKey="Asset", schemaVersion=DANDI_SCHEMA_VERSION, keywords=["keyword1", "keyword 2"], From 2c06e3d3726a32378796b622632fc454223cf2f3 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Sun, 20 Jul 2025 13:21:45 -0700 Subject: [PATCH 12/17] feat: delay `dandischema.models` import in `dandi/misctypes.py` --- dandi/misctypes.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/dandi/misctypes.py b/dandi/misctypes.py index 22c1cca3f..8df0aa283 100644 --- a/dandi/misctypes.py +++ b/dandi/misctypes.py @@ -14,9 +14,10 @@ from functools import cache import os.path from pathlib import Path -from typing import IO, TypeVar, cast +from typing import IO, TYPE_CHECKING, TypeVar, cast -from dandischema.models import DigestType +if TYPE_CHECKING: + from dandischema.models import DigestType @dataclass @@ -35,6 +36,8 @@ def dandi_etag(cls, value: str) -> Digest: Construct a `Digest` with the given value and a ``algorithm`` of ``DigestType.dandi_etag`` """ + from dandischema.models import DigestType + return cls(algorithm=DigestType.dandi_etag, value=value) @classmethod @@ -43,6 +46,8 @@ def dandi_zarr(cls, value: str) -> Digest: Construct a `Digest` with the given value and a ``algorithm`` of ``DigestType.dandi_zarr_checksum`` """ + from dandischema.models import DigestType + return cls(algorithm=DigestType.dandi_zarr_checksum, value=value) def asdict(self) -> dict[DigestType, str]: @@ -57,11 +62,15 @@ def asdict(self) -> dict[DigestType, str]: #: not actually relevant and would be too expensive to calculate @cache def get_dummy_dandi_etag() -> Digest: + from dandischema.models import DigestType + return Digest(algorithm=DigestType.dandi_etag, value=32 * "d" + "-1") @cache def get_dummy_dandi_zarr_checksum() -> Digest: + from dandischema.models import DigestType + return Digest(algorithm=DigestType.dandi_zarr_checksum, value=32 * "d" + "-1--1") From 19171a5718045b58dbeef172c8a7227d173a06cc Mon Sep 17 00:00:00 2001 From: Isaac To Date: Sun, 20 Jul 2025 13:40:55 -0700 Subject: [PATCH 13/17] feat: delay `dandischema.models` import in `dandi/files/zarr.py` --- dandi/files/zarr.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dandi/files/zarr.py b/dandi/files/zarr.py index 822c08c45..42077d06e 100644 --- a/dandi/files/zarr.py +++ b/dandi/files/zarr.py @@ -11,9 +11,8 @@ import os.path from pathlib import Path from time import sleep -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, Optional -from dandischema.models import BareAsset, DigestType from pydantic import BaseModel, ConfigDict, ValidationError import requests from zarr_checksum.tree import ZarrChecksumTree @@ -55,6 +54,9 @@ Validator, ) +if TYPE_CHECKING: + from dandischema.models import BareAsset + lgr = get_logger() @@ -320,6 +322,8 @@ def get_digest(self) -> Digest: directory, the algorithm will be the DANDI Zarr checksum algorithm; if it is a file, it will be MD5. """ + from dandischema.models import DigestType + # Avoid heavy import by importing within function: from dandi.support.digests import get_digest, get_zarr_checksum From 571de07515c27e8a77de76b771458bccb2690ce4 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Sun, 20 Jul 2025 14:16:40 -0700 Subject: [PATCH 14/17] feat: delay `dandischema.models` import in `dandi/files/bids.py` --- dandi/files/bids.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dandi/files/bids.py b/dandi/files/bids.py index a441e5999..39c8d713a 100644 --- a/dandi/files/bids.py +++ b/dandi/files/bids.py @@ -5,10 +5,9 @@ from datetime import datetime from pathlib import Path from threading import Lock +from typing import TYPE_CHECKING import weakref -from dandischema.models import BareAsset - from dandi.bids_validator_deno import bids_validate from .bases import GenericAsset, LocalFileAsset, NWBAsset @@ -18,6 +17,9 @@ from ..misctypes import Digest from ..validate_types import ValidationResult +if TYPE_CHECKING: + from dandischema.models import BareAsset + BIDS_ASSET_ERRORS = ("BIDS.NON_BIDS_PATH_PLACEHOLDER",) BIDS_DATASET_ERRORS = ("BIDS.MANDATORY_FILE_MISSING_PLACEHOLDER",) @@ -84,6 +86,8 @@ def _get_metadata(self) -> None: This populates `self._asset_metadata` """ + from dandischema.models import BareAsset + with self._lock: if self._asset_metadata is None: # Import here to avoid circular import @@ -236,6 +240,8 @@ def get_metadata( digest: Digest | None = None, ignore_errors: bool = True, ) -> BareAsset: + from dandischema.models import BareAsset + bids_metadata = BIDSAsset.get_metadata(self, digest, ignore_errors) nwb_metadata = NWBAsset.get_metadata(self, digest, ignore_errors) return BareAsset( From 007e89a134663b53c0afd8ca11ed3aa6f55a8ee1 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Sun, 20 Jul 2025 14:25:35 -0700 Subject: [PATCH 15/17] feat: delay `dandischema.models` import in `dandi/dandiset.py` --- dandi/dandiset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dandi/dandiset.py b/dandi/dandiset.py index 63ebbfb1b..bce4a818d 100644 --- a/dandi/dandiset.py +++ b/dandi/dandiset.py @@ -7,8 +7,6 @@ from pathlib import Path, PurePath, PurePosixPath from typing import TYPE_CHECKING -from dandischema.models import get_schema_version - from . import get_logger from .consts import dandiset_metadata_file from .files import DandisetMetadataFile, LocalAsset, dandi_file, find_dandi_files @@ -32,6 +30,8 @@ def __init__( schema_version: str | None = None, ) -> None: if schema_version is not None: + from dandischema.models import get_schema_version + current_version = get_schema_version() if schema_version != current_version: raise ValueError( From f7861df1841a7dfd2aca64b98cd32f6bfed28f4c Mon Sep 17 00:00:00 2001 From: Isaac To Date: Sun, 20 Jul 2025 14:53:14 -0700 Subject: [PATCH 16/17] feat: delay `dandischema.models` import in `dandi/cli/cmd_ls.py` --- dandi/cli/cmd_ls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dandi/cli/cmd_ls.py b/dandi/cli/cmd_ls.py index 9ac9ed42f..4c01c370a 100644 --- a/dandi/cli/cmd_ls.py +++ b/dandi/cli/cmd_ls.py @@ -3,7 +3,6 @@ import os.path as op import click -from dandischema import models from .base import devel_option, lgr, map_to_click_exceptions from .formatter import JSONFormatter, JSONLinesFormatter, PYOUTFormatter, YAMLFormatter @@ -92,6 +91,8 @@ def ls( ): """List .nwb files and dandisets metadata.""" + from dandischema import models + # TODO: more logical ordering in case of fields = None common_fields = ("path", "size") if schema is not None: From 066e16e5922fade8d11a4049a551356ecc97cd84 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Sun, 20 Jul 2025 15:04:05 -0700 Subject: [PATCH 17/17] feat: delay `dandischema.models` import in `dandi/download.py` --- dandi/download.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dandi/download.py b/dandi/download.py index 51cf31de3..874b33b28 100644 --- a/dandi/download.py +++ b/dandi/download.py @@ -21,7 +21,6 @@ from typing import IO, Any, Literal from dandischema.digests.dandietag import ETagHashlike -from dandischema.models import DigestType from fasteners import InterProcessLock import humanize from interleave import FINISH_CURRENT, lazy_interleave @@ -995,6 +994,8 @@ def digest_callback(path: str, algoname: str, d: str) -> None: digests[path] = d def downloads_gen(): + from dandischema.models import DigestType + for entry in asset.iterfiles(): entries.append(entry) etag = entry.digest