Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 25 additions & 23 deletions diracx-cli/src/diracx/cli/internal/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ def generate_cs(config_repo: str):
raise typer.Exit(1)

config = Config(
Registry={},
DIRAC=DIRACConfig(),
Operations={"Defaults": OperationsConfig()},
registry={},
dirac=DIRACConfig(),
operations={"Defaults": OperationsConfig()},
)

git.Repo.init(repo_path, initial_branch="master")
Expand All @@ -79,21 +79,21 @@ def add_vo(

# A VO should at least contain a default group
new_registry = RegistryConfig(
IdP=IdpConfig(URL=idp_url, ClientID=idp_client_id),
DefaultGroup=default_group,
Users={},
Groups={
idp=IdpConfig(url=idp_url, client_id=idp_client_id),
default_group=default_group,
users={},
groups={
default_group: GroupConfig(
Properties={"NormalUser"}, Quota=None, Users=set()
properties={"NormalUser"}, quota=None, users=set()
)
},
)

if vo in config.Registry:
if vo in config.registry:
typer.echo(f"ERROR: VO {vo} already exists", err=True)
raise typer.Exit(1)

config.Registry[vo] = new_registry
config.registry[vo] = new_registry

update_config_and_commit(
repo_path=repo_path,
Expand All @@ -116,17 +116,17 @@ def add_group(
repo_path = get_repo_path(config_repo)
config = get_config_from_repo_path(repo_path)

new_group = GroupConfig(Properties=set(properties), Quota=None, Users=set())
new_group = GroupConfig(properties=set(properties), quota=None, users=set())

if vo not in config.Registry:
if vo not in config.registry:
typer.echo(f"ERROR: Virtual Organization {vo} does not exist", err=True)
raise typer.Exit(1)

if group in config.Registry[vo].Groups.keys():
if group in config.registry[vo].groups.keys():
typer.echo(f"ERROR: Group {group} already exists in {vo}", err=True)
raise typer.Exit(1)

config.Registry[vo].Groups[group] = new_group
config.registry[vo].groups[group] = new_group

update_config_and_commit(
repo_path=repo_path, config=config, message=f"Added group {group} in {vo}"
Expand All @@ -148,30 +148,30 @@ def add_user(
repo_path = get_repo_path(config_repo)
config = get_config_from_repo_path(repo_path)

new_user = UserConfig(PreferedUsername=preferred_username)
new_user = UserConfig(prefered_username=preferred_username)

if vo not in config.Registry:
if vo not in config.registry:
typer.echo(f"ERROR: Virtual Organization {vo} does not exist", err=True)
raise typer.Exit(1)

if sub in config.Registry[vo].Users:
if sub in config.registry[vo].users:
typer.echo(f"ERROR: User {sub} already exists", err=True)
raise typer.Exit(1)

config.Registry[vo].Users[sub] = new_user
config.registry[vo].users[sub] = new_user

if not groups:
groups = [config.Registry[vo].DefaultGroup]
groups = [config.registry[vo].default_group]

for group in set(groups):
if group not in config.Registry[vo].Groups:
if group not in config.registry[vo].groups:
typer.echo(f"ERROR: Group {group} does not exist in {vo}", err=True)
raise typer.Exit(1)
if sub in config.Registry[vo].Groups[group].Users:
if sub in config.registry[vo].groups[group].users:
typer.echo(f"ERROR: User {sub} already exists in group {group}", err=True)
raise typer.Exit(1)

config.Registry[vo].Groups[group].Users.add(sub)
config.registry[vo].groups[group].users.add(sub)

update_config_and_commit(
repo_path=repo_path,
Expand All @@ -187,7 +187,9 @@ def update_config_and_commit(repo_path: Path, config: Config, message: str):
yaml_path = repo_path / "default.yml"
typer.echo(f"Writing back configuration to {yaml_path}", err=True)
yaml_path.write_text(
yaml.safe_dump(config.model_dump(exclude_unset=True, mode="json"))
yaml.safe_dump(
config.model_dump(exclude_unset=True, mode="json", by_alias=True)
)
)
repo.index.add([yaml_path.relative_to(repo_path)])
repo.index.commit(message)
34 changes: 18 additions & 16 deletions diracx-cli/src/diracx/cli/internal/legacy.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,19 +34,19 @@


class IdPConfig(BaseModel):
URL: str
ClientID: str
url: str = Field(alias="URL")
client_id: str = Field(alias="ClientID")


class VOConfig(BaseModel):
DefaultGroup: str
IdP: IdPConfig
UserSubjects: dict[str, str]
Support: SupportInfo = Field(default_factory=SupportInfo)
default_group: str = Field(alias="DefaultGroup")
idp: IdPConfig = Field(alias="IdP")
user_subjects: dict[str, str] = Field(alias="UserSubjects")
support: SupportInfo = Field(default_factory=SupportInfo, alias="Support")


class ConversionConfig(BaseModel):
VOs: dict[str, VOConfig]
vos: dict[str, VOConfig] = Field(alias="VOs")


@app.command()
Expand Down Expand Up @@ -81,7 +81,9 @@ def cs_sync(old_file: Path, new_file: Path):
)[0].load()
config = config_class.model_validate(raw)
new_file.write_text(
yaml.safe_dump(config.model_dump(exclude_unset=True, mode="json"))
yaml.safe_dump(
config.model_dump(by_alias=True, exclude_unset=True, mode="json")
)
)


Expand Down Expand Up @@ -112,20 +114,20 @@ def _apply_fixes(raw):

# Check that we have the config for all the VOs
vos = set(raw["Registry"].get("VO", []))
if non_configured_vos := vos - set(conv_config.VOs):
if non_configured_vos := vos - set(conv_config.vos):
print(f"{non_configured_vos} don't have a migration config, ignoring")

# Modify the registry to be fully multi-VO
original_registry = raw.pop("Registry")
raw["Registry"] = {}

for vo, vo_meta in conv_config.VOs.items():
for vo, vo_meta in conv_config.vos.items():
raw["Registry"][vo] = {
"IdP": vo_meta.IdP.model_dump(),
"DefaultGroup": vo_meta.DefaultGroup,
"IdP": vo_meta.idp.model_dump(by_alias=True),
"DefaultGroup": vo_meta.default_group,
"Users": {},
"Groups": {},
"Support": vo_meta.Support,
"Support": vo_meta.support,
}
if "DefaultStorageQuota" in original_registry:
raw["Registry"][vo]["DefaultStorageQuota"] = original_registry[
Expand Down Expand Up @@ -154,14 +156,14 @@ def _apply_fixes(raw):
nicknames = {u.strip() for u in info["Users"].split(",") if u.strip()}
vo_users |= nicknames
raw["Registry"][vo]["Groups"][name]["Users"] = [
vo_meta.UserSubjects[n]
vo_meta.user_subjects[n]
for n in nicknames
if n in vo_meta.UserSubjects
if n in vo_meta.user_subjects
]
# Find the users that belong to this VO
for name, info in original_registry["Users"].items():
if name in vo_users:
if subject := vo_meta.UserSubjects.get(name):
if subject := vo_meta.user_subjects.get(name):
raw["Registry"][vo]["Users"][subject] = info | {
"PreferedUsername": name
}
Expand Down
42 changes: 21 additions & 21 deletions diracx-cli/tests/test_internal.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,10 @@ def test_generate_cs(tmp_path, protocol):
def test_add_vo(cs_repo):
config = ConfigSource.create_from_url(backend_url=cs_repo).read()

assert TEST_VO in config.Registry
assert config.Registry[TEST_VO].DefaultGroup == "user"
assert config.Registry[TEST_VO].IdP.URL == "https://idp.invalid"
assert config.Registry[TEST_VO].IdP.ClientID == "idp-client-id"
assert TEST_VO in config.registry
assert config.registry[TEST_VO].default_group == "user"
assert config.registry[TEST_VO].idp.url == "https://idp.invalid"
assert config.registry[TEST_VO].idp.client_id == "idp-client-id"

# Add a second VO to it
vo2 = "lhcb"
Expand All @@ -98,10 +98,10 @@ def test_add_vo(cs_repo):
config = ConfigSource.create_from_url(backend_url=cs_repo).read()
assert result.exit_code == 0, result.output

assert vo2 in config.Registry
assert config.Registry[vo2].DefaultGroup == "admin"
assert config.Registry[vo2].IdP.URL == "https://idp.example.invalid"
assert config.Registry[vo2].IdP.ClientID == "idp-client-id2"
assert vo2 in config.registry
assert config.registry[vo2].default_group == "admin"
assert config.registry[vo2].idp.url == "https://idp.example.invalid"
assert config.registry[vo2].idp.client_id == "idp-client-id2"

# Try to insert a VO that already exists
result = runner.invoke(
Expand All @@ -123,10 +123,10 @@ def test_add_group(cs_repo):

config = ConfigSource.create_from_url(backend_url=cs_repo).read()

assert TEST_USER_GROUP in config.Registry[TEST_VO].Groups
assert config.Registry[TEST_VO].Groups[TEST_USER_GROUP].JobShare == 1000
assert config.Registry[TEST_VO].Groups[TEST_USER_GROUP].Properties == {"NormalUser"}
assert config.Registry[TEST_VO].Groups[TEST_USER_GROUP].Users == set()
assert TEST_USER_GROUP in config.registry[TEST_VO].groups
assert config.registry[TEST_VO].groups[TEST_USER_GROUP].job_share == 1000
assert config.registry[TEST_VO].groups[TEST_USER_GROUP].properties == {"NormalUser"}
assert config.registry[TEST_VO].groups[TEST_USER_GROUP].users == set()

# Add a second group to it
result = runner.invoke(
Expand All @@ -146,13 +146,13 @@ def test_add_group(cs_repo):
config = ConfigSource.create_from_url(backend_url=cs_repo).read()
assert result.exit_code == 0, result.output

assert new_group in config.Registry[TEST_VO].Groups
assert config.Registry[TEST_VO].Groups[new_group].JobShare == 1000
assert config.Registry[TEST_VO].Groups[new_group].Properties == {
assert new_group in config.registry[TEST_VO].groups
assert config.registry[TEST_VO].groups[new_group].job_share == 1000
assert config.registry[TEST_VO].groups[new_group].properties == {
"AdminUser",
"NormalUser",
}
assert config.Registry[TEST_VO].Groups[new_group].Users == set()
assert config.registry[TEST_VO].groups[new_group].users == set()

# Try to insert a group that already exists
result = runner.invoke(
Expand Down Expand Up @@ -193,8 +193,8 @@ def test_add_user(cs_repo, vo, user_group):
config = ConfigSource.create_from_url(backend_url=cs_repo).read()

# Check the user isn't in it
if vo in config.Registry:
assert sub not in config.Registry[vo].Users
if vo in config.registry:
assert sub not in config.registry[vo].users

# Add a user to it
result = runner.invoke(
Expand All @@ -218,7 +218,7 @@ def test_add_user(cs_repo, vo, user_group):

config = ConfigSource.create_from_url(backend_url=cs_repo).read()
# check the user is defined
assert vo in config.Registry
assert sub in config.Registry[vo].Users
assert vo in config.registry
assert sub in config.registry[vo].users
for group in user_group or [TEST_USER_GROUP]:
assert config.Registry[vo].Groups[group].Users == {sub}
assert config.registry[vo].groups[group].users == {sub}
Loading
Loading