From aab9944b538f4a654a87d69ba918d33602d3fbec Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 30 Oct 2025 15:30:45 +0100 Subject: [PATCH 1/3] style: just blacked everything --- Pilot/pilotCommands.py | 5 +++-- Pilot/pilotTools.py | 2 +- Pilot/proxyTools.py | 6 +++--- Pilot/tests/Test_simplePilotLogger.py | 1 + 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 4815f44b..c2e6cb1e 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -834,7 +834,7 @@ def execute(self): archScript = self.pp.architectureScript if self.pp.architectureScript.split(" ")[0] == "dirac-apptainer-exec": archScript = " ".join(self.pp.architectureScript.split(" ")[1:]) - + architectureCmd = "%s %s -ddd" % (archScript, " ".join(cfg)) if self.pp.architectureScript.split(" ")[0] == "dirac-apptainer-exec": @@ -872,10 +872,12 @@ def execute(self): return localArchitecture + class ConfigureArchitectureWithoutCLI(CommandBase): """This command determines the platform. Separated from the ConfigureDIRAC command for easier extensibility. """ + def getPlatformString(self): # Modified to return our desired platform string, R. Graciani platformTuple = (platform.system(), platform.machine()) @@ -903,7 +905,6 @@ def execute(self): self.log.error("Configuration error [ERROR %s]" % str(e)) self.exitWithError(1) - cfg = ["-FDMH"] # force update, skip CA checks, skip CA download, skip VOMS if self.pp.useServerCertificate: cfg.append("--UseServerCertificate") diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 8afe0f62..b8e0f1ec 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -793,7 +793,7 @@ def executeAndGetOutput(self, cmd, environDict=None): outChunk = outChunk.decode("utf-8") # Strip unicode replacement characters # Ensure correct type conversion in Python 2 - outChunk = str(outChunk.replace(u"\ufffd", "")) + outChunk = str(outChunk.replace("\ufffd", "")) # Avoid potential str() issues in Py2 outChunk = unicode(outChunk) # pylint: disable=undefined-variable else: diff --git a/Pilot/proxyTools.py b/Pilot/proxyTools.py index a5fa652e..4aac355d 100644 --- a/Pilot/proxyTools.py +++ b/Pilot/proxyTools.py @@ -9,7 +9,7 @@ VOMS_FQANS_OID = b"1.3.6.1.4.1.8005.100.100.4" VOMS_EXTENSION_OID = b"1.3.6.1.4.1.8005.100.100.5" -RE_OPENSSL_ANS1_FORMAT = re.compile(br"^\s*\d+:d=(\d+)\s+hl=") +RE_OPENSSL_ANS1_FORMAT = re.compile(rb"^\s*\d+:d=(\d+)\s+hl=") def parseASN1(data): @@ -41,7 +41,7 @@ def getVO(proxy_data): str: A VO """ - chain = re.findall(br"-----BEGIN CERTIFICATE-----\n.+?\n-----END CERTIFICATE-----", proxy_data, flags=re.DOTALL) + chain = re.findall(rb"-----BEGIN CERTIFICATE-----\n.+?\n-----END CERTIFICATE-----", proxy_data, flags=re.DOTALL) for cert in chain: proc = Popen(["openssl", "x509", "-outform", "der"], stdin=PIPE, stdout=PIPE) out, _ = proc.communicate(cert) @@ -61,7 +61,7 @@ def getVO(proxy_data): if depth <= initial_depth: break # Look for a role, if it exists the VO is the first element - match = re.search(br"OCTET STRING\s+:/([a-zA-Z0-9]+)/Role=", line) + match = re.search(rb"OCTET STRING\s+:/([a-zA-Z0-9]+)/Role=", line) if match: return match.groups()[0].decode() raise NotImplementedError("Something went very wrong") diff --git a/Pilot/tests/Test_simplePilotLogger.py b/Pilot/tests/Test_simplePilotLogger.py index 1fc448ae..df2ac0c2 100644 --- a/Pilot/tests/Test_simplePilotLogger.py +++ b/Pilot/tests/Test_simplePilotLogger.py @@ -172,5 +172,6 @@ def test_executeAndGetOutput(self, popenMock, argvmock): self.stdout_mock.truncate() self.stderr_mock.truncate() + if __name__ == "__main__": unittest.main() From 3268f4bde04831fde85896bee2fab17f25c9d114 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 30 Oct 2025 15:52:56 +0100 Subject: [PATCH 2/3] style: json styling applied --- tests/CI/pilot_newSchema.json | 307 +++++++++++++++++----------------- tests/CI/pilot_oldSchema.json | 4 +- 2 files changed, 157 insertions(+), 154 deletions(-) diff --git a/tests/CI/pilot_newSchema.json b/tests/CI/pilot_newSchema.json index 0c8c1d32..ee793113 100644 --- a/tests/CI/pilot_newSchema.json +++ b/tests/CI/pilot_newSchema.json @@ -1,157 +1,160 @@ { - "timestamp": "2023-02-13T14:34:26.725499", - "CEs": { - "jenkins.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST" - }, - "jenkins-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL" - }, - "jenkins-mp.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-MP" - }, - "jenkins-mp-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL" - }, - "jenkins-singularity.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST", - "LocalCEType": "Singularity", - "InstallDIRACInContainer": "True" - }, - "jenkins-singularity-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL", - "LocalCEType": "Singularity" - }, - "jenkins-mp-pool.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "Queue": "VAR_JENKINS_QUEUE", - "GridCEType": "TEST-MP", - "LocalCEType": "Pool" - }, - "jenkins-mp-pool-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL", - "LocalCEType": "InProcess", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool" - } - }, - "jenkins-mp-pool-sudo.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-MP", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool/Sudo" - } - }, - "jenkins-mp-pool-sudo-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool/Sudo" - } - }, - "jenkins-mp-pool-singularity.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-MP", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool/Singularity" - } - }, - "jenkins-mp-pool-singularity-full.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-FULL", - "VAR_JENKINS_QUEUE": { - "LocalCEType": "Pool/Singularity" - } - }, - "jenkins-lhcb.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "LHCb" - }, - "jenkins-lhcb-d.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "LHCbD" - }, - "jenkins-lhcb-dd.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "LHCbDD" - }, + "timestamp": "2023-02-13T14:34:26.725499", + "CEs": { + "jenkins.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST" + }, + "jenkins-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL" + }, + "jenkins-mp.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-MP" + }, + "jenkins-mp-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL" + }, + "jenkins-singularity.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST", + "LocalCEType": "Singularity", + "InstallDIRACInContainer": "True" + }, + "jenkins-singularity-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL", + "LocalCEType": "Singularity" + }, + "jenkins-mp-pool.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "Queue": "VAR_JENKINS_QUEUE", + "GridCEType": "TEST-MP", + "LocalCEType": "Pool" + }, + "jenkins-mp-pool-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL", + "LocalCEType": "InProcess", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool" + } + }, + "jenkins-mp-pool-sudo.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-MP", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool/Sudo" + } + }, + "jenkins-mp-pool-sudo-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool/Sudo" + } + }, + "jenkins-mp-pool-singularity.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-MP", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool/Singularity" + } + }, + "jenkins-mp-pool-singularity-full.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-FULL", + "VAR_JENKINS_QUEUE": { + "LocalCEType": "Pool/Singularity" + } + }, + "jenkins-lhcb.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "LHCb" + }, + "jenkins-lhcb-d.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "LHCbD" + }, + "jenkins-lhcb-dd.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "LHCbDD" + }, "jenkins-revised.cern.ch": { - "Site": "VAR_JENKINS_SITE", - "GridCEType": "TEST-REVISED-COMMANDS" + "Site": "VAR_JENKINS_SITE", + "GridCEType": "TEST-REVISED-COMMANDS" } - }, - "Defaults": { - "Pilot": { - "RemoteLogging": "False", - "Commands": { - "TEST": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "TEST-MP": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "TEST-FULL": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements, LaunchAgent", - "LHCb": "CheckWorkerNode, LHCbInstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "LHCbD": "CheckWorkerNode, InstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "LHCbDD": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", - "TEST-REVISED-COMMANDS": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitectureWithoutCLI, ConfigureCPURequirements" + }, + "Defaults": { + "Pilot": { + "RemoteLogging": "False", + "Commands": { + "TEST": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "TEST-MP": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "TEST-FULL": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements, LaunchAgent", + "LHCb": "CheckWorkerNode, LHCbInstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "LHCbD": "CheckWorkerNode, InstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "LHCbDD": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements", + "TEST-REVISED-COMMANDS": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitectureWithoutCLI, ConfigureCPURequirements" } - } - }, - "gridpp": { - "Pilot": { - "Version": "VAR_DIRAC_VERSION", - "Extensions": "None", - "CheckVersion": "False", - "pilotFileServer": "lbcertifdirac70.cern.ch:8443", - "pilotRepoBranch": "does_not_matter", - "pilotRepo": "https://github.com/does_not_matter/Pilot.git", - "GenericPilotGroup": "gridpp_pilot", - "GenericPilotDN": "VAR_USERDN_GRIDPP", - "RemoteLogging": "False", - "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", - "UploadSE": "UKI-LT2-IC-HEP-disk", - "UploadPath": "/gridpp/pilotlogs/", - "RemoteLoggerTimerInterval": 0, - "LoggingShifterName": "GridPPLogManager", - "PilotLogLevel": "DEBUG", - "CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu" - } - }, - "dteam": { - "Pilot": { - "Version": "VAR_DIRAC_VERSION", - "CheckVersion": "False", - "pilotFileServer": "lbcertifdirac70.cern.ch:8443", - "pilotRepoBranch": "should_not_matter", - "pilotRepo": "https://github.com/should_not_matter/Pilot.git", - "GenericPilotGroup": "dteam_pilot", - "GenericPilotDN": "VAR_USERDN", - "RemoteLogging": "True", - "RemoteLoggerCEsWhiteList": "jenkins.cern.ch, jenkins-mp-pool.cern.ch", - "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", - "RemoteLoggerTimerInterval": 0, - "PilotLogLevel": "DEBUG", - "CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu" - } - }, - "LHCb": { - "Pilot": { - "Version": "VAR_DIRAC_VERSION", - "CheckVersion": "False", - "pilotFileServer": "lbcertifdirac70.cern.ch:8443", - "GenericPilotGroup": "dteam_pilot", - "GenericPilotDN": "VAR_USERDN", - "RemoteLogging": "False", - "RemoteLoggerTimerInterval": 0, - "PilotLogLevel": "DEBUG", - "CVMFS_locations": "/cvmfs/lhcb.cern.ch/, $VO_LHCB_SW_DIR" - } - }, - "ConfigurationServers": [ - "VAR_CS" - ] + } + }, + "gridpp": { + "Pilot": { + "Version": "VAR_DIRAC_VERSION", + "Extensions": "None", + "CheckVersion": "False", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", + "pilotRepoBranch": "does_not_matter", + "pilotRepo": "https://github.com/does_not_matter/Pilot.git", + "GenericPilotGroup": "gridpp_pilot", + "GenericPilotDN": "VAR_USERDN_GRIDPP", + "RemoteLogging": "False", + "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", + "UploadSE": "UKI-LT2-IC-HEP-disk", + "UploadPath": "/gridpp/pilotlogs/", + "RemoteLoggerTimerInterval": 0, + "LoggingShifterName": "GridPPLogManager", + "PilotLogLevel": "DEBUG", + "CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu" + } + }, + "dteam": { + "Pilot": { + "Version": "VAR_DIRAC_VERSION", + "CheckVersion": "False", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", + "pilotRepoBranch": "should_not_matter", + "pilotRepo": "https://github.com/should_not_matter/Pilot.git", + "GenericPilotGroup": "dteam_pilot", + "GenericPilotDN": "VAR_USERDN", + "RemoteLogging": "True", + "RemoteLoggerCEsWhiteList": "jenkins.cern.ch, jenkins-mp-pool.cern.ch", + "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", + "RemoteLoggerTimerInterval": 0, + "PilotLogLevel": "DEBUG", + "CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu" + } + }, + "LHCb": { + "Pilot": { + "Version": "VAR_DIRAC_VERSION", + "CheckVersion": "False", + "pilotFileServer": "lbcertifdirac70.cern.ch:8443", + "GenericPilotGroup": "dteam_pilot", + "GenericPilotDN": "VAR_USERDN", + "RemoteLogging": "False", + "RemoteLoggerTimerInterval": 0, + "PilotLogLevel": "DEBUG", + "CVMFS_locations": "/cvmfs/lhcb.cern.ch/, $VO_LHCB_SW_DIR" + } + }, + "ConfigurationServers": [ + "VAR_CS" + ], + "PreferredURLPatterns": [ + "VAR_PREF_URL_PATTERN" + ] } diff --git a/tests/CI/pilot_oldSchema.json b/tests/CI/pilot_oldSchema.json index 3502e3b5..163bb387 100644 --- a/tests/CI/pilot_oldSchema.json +++ b/tests/CI/pilot_oldSchema.json @@ -75,8 +75,8 @@ ] }, "Logging": { - "LoggingType":"LOCAL_FILE", - "LocalOutputFile":"myFile" + "LoggingType": "LOCAL_FILE", + "LocalOutputFile": "myFile" }, "NagiosProbes": "WN-mjf, WN-sft-csh, WN-sft-voms, WN-sft-vo-swdir", "Version": "VAR_DIRAC_VERSION", From 63571e3b40cddd0d9bfc0152104fac0f8197acfe Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 4 Nov 2025 15:17:02 +0100 Subject: [PATCH 3/3] feat: setting /DIRAC/PreferredURLPatterns if it is in pilot.json --- .github/workflows/integration.yml | 6 ++++++ Pilot/pilotCommands.py | 2 ++ Pilot/pilotTools.py | 3 +++ Pilot/proxyTools.py | 6 +++--- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ee252cfe..a3ff15fd 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -53,6 +53,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) @@ -115,6 +116,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) @@ -164,6 +166,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) @@ -221,6 +224,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) @@ -273,6 +277,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) @@ -334,6 +339,7 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + sed -i "s#VAR_PREF_URL_PATTERN#.*\.cern\.ch/.*#g" pilot.json g_job="testintegrationworkflow${GITHUB_JOB//-/}" pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)" pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev) diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index c2e6cb1e..5032a02d 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -508,6 +508,8 @@ def _getBasicsCFG(self): self.cfg.append('-S "%s"' % self.pp.setup) if self.pp.configServer: self.cfg.append('-C "%s"' % self.pp.configServer) + if self.pp.preferredURLPatterns: + self.cfg.append("-o /DIRAC/PreferredURLPatterns=%s" % quote(",".join(self.pp.preferredURLPatterns))) if self.pp.releaseProject: self.cfg.append('-e "%s"' % self.pp.releaseProject) self.cfg.append("-o /LocalSite/ReleaseProject=%s" % self.pp.releaseProject) diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index b8e0f1ec..11345f46 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -908,6 +908,7 @@ def __init__(self): self.site = "" self.setup = "" self.configServer = "" + self.preferredURLPatterns = "" self.ceName = "" self.ceType = "" self.queueName = "" @@ -1306,6 +1307,8 @@ def __initJSON2(self): # the generic one (a list): self.configServer = ",".join([str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"]]) + self.preferredURLPatterns = self.pilotJSON.get("PreferredURLPatterns", self.preferredURLPatterns) + # version(a comma separated values in a string). We take the first one. (the default value defined in the code) dVersion = pilotOptions.get("Version", self.releaseVersion) if dVersion: diff --git a/Pilot/proxyTools.py b/Pilot/proxyTools.py index 4aac355d..a5fa652e 100644 --- a/Pilot/proxyTools.py +++ b/Pilot/proxyTools.py @@ -9,7 +9,7 @@ VOMS_FQANS_OID = b"1.3.6.1.4.1.8005.100.100.4" VOMS_EXTENSION_OID = b"1.3.6.1.4.1.8005.100.100.5" -RE_OPENSSL_ANS1_FORMAT = re.compile(rb"^\s*\d+:d=(\d+)\s+hl=") +RE_OPENSSL_ANS1_FORMAT = re.compile(br"^\s*\d+:d=(\d+)\s+hl=") def parseASN1(data): @@ -41,7 +41,7 @@ def getVO(proxy_data): str: A VO """ - chain = re.findall(rb"-----BEGIN CERTIFICATE-----\n.+?\n-----END CERTIFICATE-----", proxy_data, flags=re.DOTALL) + chain = re.findall(br"-----BEGIN CERTIFICATE-----\n.+?\n-----END CERTIFICATE-----", proxy_data, flags=re.DOTALL) for cert in chain: proc = Popen(["openssl", "x509", "-outform", "der"], stdin=PIPE, stdout=PIPE) out, _ = proc.communicate(cert) @@ -61,7 +61,7 @@ def getVO(proxy_data): if depth <= initial_depth: break # Look for a role, if it exists the VO is the first element - match = re.search(rb"OCTET STRING\s+:/([a-zA-Z0-9]+)/Role=", line) + match = re.search(br"OCTET STRING\s+:/([a-zA-Z0-9]+)/Role=", line) if match: return match.groups()[0].decode() raise NotImplementedError("Something went very wrong")