From 3cc73649d1ee149c382ba56fdfd61d716eb78644 Mon Sep 17 00:00:00 2001 From: Janusz Martyniak Date: Mon, 2 Mar 2020 17:26:08 +0000 Subject: [PATCH] Dirac-Rucio integration - upload/download with REST --- DataManagementSystem/Client/DataManager.py | 7 + Resources/Catalog/FileCatalogClientBase.py | 16 +- Resources/Catalog/RucioFileCatalogClient.py | 376 ++++++++++++++++++ .../Catalog/RucioRESTClientApi/BaseClient.py | 349 ++++++++++++++++ .../Catalog/RucioRESTClientApi/DIDClient.py | 113 ++++++ .../Catalog/RucioRESTClientApi/RSEClient.py | 90 +++++ .../RucioRESTClientApi/ReplicaClient.py | 190 +++++++++ .../Catalog/RucioRESTClientApi/ScopeClient.py | 117 ++++++ Resources/Catalog/RucioRESTClientApi/Utils.py | 130 ++++++ .../Catalog/RucioRESTClientApi/__init__.py | 0 10 files changed, 1387 insertions(+), 1 deletion(-) create mode 100644 Resources/Catalog/RucioFileCatalogClient.py create mode 100644 Resources/Catalog/RucioRESTClientApi/BaseClient.py create mode 100644 Resources/Catalog/RucioRESTClientApi/DIDClient.py create mode 100644 Resources/Catalog/RucioRESTClientApi/RSEClient.py create mode 100644 Resources/Catalog/RucioRESTClientApi/ReplicaClient.py create mode 100644 Resources/Catalog/RucioRESTClientApi/ScopeClient.py create mode 100644 Resources/Catalog/RucioRESTClientApi/Utils.py create mode 100644 Resources/Catalog/RucioRESTClientApi/__init__.py diff --git a/DataManagementSystem/Client/DataManager.py b/DataManagementSystem/Client/DataManager.py index 198870549d2..716c3a2c04b 100644 --- a/DataManagementSystem/Client/DataManager.py +++ b/DataManagementSystem/Client/DataManager.py @@ -495,6 +495,13 @@ def putAndRegister(self, lfn, fileName, diracSE, guid=None, path=None, errStr = "Supplied file does not exist." log.debug(errStr, fileName) return S_ERROR(errStr) + # check if a catalog enforces a non standard path + if not path: + pfn = self.fileCatalog.getPfnFromLfn(lfn, diracSE) + if pfn['OK']: + path = pfn['Value'] if pfn['Value'] else None + else: + path = None # If the path is not provided then use the LFN path if not path: path = os.path.dirname(lfn) diff --git a/Resources/Catalog/FileCatalogClientBase.py b/Resources/Catalog/FileCatalogClientBase.py index 791d23d9427..eb891676b71 100644 --- a/Resources/Catalog/FileCatalogClientBase.py +++ b/Resources/Catalog/FileCatalogClientBase.py @@ -25,7 +25,7 @@ class FileCatalogClientBase(Client): """ # Default mandatory methods having all-allowing implementation in the base class - READ_METHODS = ['hasAccess', 'exists', 'getPathPermissions'] + READ_METHODS = ['hasAccess', 'exists', 'getPathPermissions', 'getPfnFromLfn'] # List of methods that can be complemented in the derived classes WRITE_METHODS = [] @@ -115,3 +115,17 @@ def getPathPermissions(self, lfns): """ return S_OK({'Failed': {}, 'Successful': dict.fromkeys(lfns, {'Write': True, "Read": True})}) + + def getPfnFromLfn(self, lfn, se): + """ + A default method to return an SE specific pfn which is non standard. + This method returns S_OK(None) to maintain a default DataManager.putAndRegister behaviour. + + :param lfn: logical file name + :type lfn: str + :param se: Storage Element name + :type se: str + :return: S_OK(None) + :rtype: dict + """ + return S_OK('') diff --git a/Resources/Catalog/RucioFileCatalogClient.py b/Resources/Catalog/RucioFileCatalogClient.py new file mode 100644 index 00000000000..ba50b05f16b --- /dev/null +++ b/Resources/Catalog/RucioFileCatalogClient.py @@ -0,0 +1,376 @@ +""" +Rucio File Catalog Client. +""" + +from __future__ import division + +import os +import os.path +import sys +import datetime + +from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations +from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup +from DIRAC.Core.Security.ProxyInfo import getProxyInfo +from DIRAC.Resources.Catalog.RucioRESTClientApi.BaseClient import BaseClient +from DIRAC.Resources.Catalog.RucioRESTClientApi.RSEClient import RSEClient +from DIRAC.Resources.Catalog.RucioRESTClientApi.DIDClient import DIDClient +from DIRAC.Resources.Catalog.RucioRESTClientApi.ScopeClient import ScopeClient +from DIRAC.Resources.Catalog.RucioRESTClientApi.ReplicaClient import ReplicaClient +from DIRAC.Resources.Catalog.RucioRESTClientApi import Utils +from DIRAC.Resources.Catalog.Utilities import checkCatalogArguments +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername, getVOMSAttributeForGroup, \ + getVOForGroup, getVOOption +from DIRAC.Resources.Catalog.FileCatalogClientBase import FileCatalogClientBase +from DIRAC.Core.Base.Client import Client +from DIRAC.Resources.Catalog.LcgFileCatalogClient import getClientCertInfo +from DIRAC.Core.Utilities.List import breakListIntoChunks + + +class RucioFileCatalogClient(FileCatalogClientBase): + """ + + """ + + READ_METHODS = FileCatalogClientBase.READ_METHODS + ['listDirectory', 'getUserDirectory', 'getPfnFromLfn', + 'getReplicas', 'getFileMetadata', 'isFile'] + + WRITE_METHODS = FileCatalogClientBase.WRITE_METHODS + ['addFile'] + + NO_LFN_METHODS = FileCatalogClientBase.NO_LFN_METHODS + ['getUserDirectory', 'createUserDirectory', + 'createUserMapping', 'removeUserDirectory'] + + ADMIN_METHODS = FileCatalogClientBase.ADMIN_METHODS + ['getUserDirectory'] + + def __init__(self, url=None, **options): + + super(RucioFileCatalogClient, self).__init__(url=url, **options) + + result = Operations(vo=getVOfromProxyGroup().get('Value', None)).\ + getOptionsDict('/Services/Catalogs/%s' % self.__class__.__name__.replace('Client', '')) + if result['OK']: + options.update(result['Value']) + + rucioHost = options.get('RucioHost', None) + authHost = options.get('AuthHost', None) + # giving ourselves a chance to have a VO-wide Rucio account, mainly for testing. + account = options.get('RucioAccount', None) + # ideally the Rucio account should match Dirac username, until we have a mapper + if account is None: + result = getProxyInfo() + if result['OK']: + account = result['Value'].get('username', 'root') + + clients = [ScopeClient, RSEClient, DIDClient, ReplicaClient] + for client in clients: + clientName = client.__name__.lower().replace("client", "Client") + setattr(self, clientName, client(rucioHost, authHost, account=account)) + result = getattr(self, clientName).authenticate() + if result['OK']: + gLogger.info('Rucio %s authentication successful' % clientName) + gLogger.debug("Rucio File Catalog %s client created with options: " % clientName, options) + else: + gLogger.error('Rucio client authentication failed', result) + + @checkCatalogArguments + def listDirectory(self, lfns, verbose=False): + gLogger.debug("Rucio list directory for lfns: ", lfns) + failed = {} + successful = {} + for path in lfns: + res = self.__getDirectoryContents(path, verbose) + if res['OK']: + successful[path] = res['Value'] + else: + failed[path] = res['Message'] + resDict = {'Failed': failed, 'Successful': successful} + gLogger.debug(resDict) + return S_OK(resDict) + + @checkCatalogArguments + def isFile(self, lfns, verbose=False): + """ + Check whether the supplied lfns are files. + + :param lfns: List of LFNs + :type lfns: list + :param verbose: verbose flag + :type verbose: bool + :return: Dirac S_OK object + :rtype: dict + """ + + failed = {} + successful = {} + + for lfn in lfns: + _, scope, didname = self.__getLfnElements(lfn) + metaresult = self.didClient.getMetadata(scope, didname) + if metaresult['OK']: + meta = metaresult['Value'] + if meta['did_type'] == 'FILE': + successful[lfn] = True + else: + successful[lfn] = False + else: + failed[lfn] = metaresult['Message'] + if verbose: + gLogger("Rucio isFile: ", {'Failed': failed, 'Successful': successful}) + return S_OK({'Failed': failed, 'Successful': successful}) + + def __getDirectoryContents(self, path, verbose): + """ + Get files and directories for a given path. + + :param path: Dirac logical file name, including VO and scope + :param verbose: + :return: a dictionary with files and their attributes + """ + subDirs = {} + links = {} + files = {} + # get the scope as a second element of the path - this is a convention + elements = os.path.normpath(path).split(os.sep) + + if len(elements) <= 2: + return S_ERROR(" The Rucio lfn path should contain at least 2 elements:" + " the VO, the scope (and an optional path):\n%s" % path) + scope = elements[2] + result = self.scopeClient.listScopes() + if result['OK']: + if scope not in result['Value']: + return S_ERROR("Rucio scope %s does not exist" % scope) + else: + return result + + result = self.didClient.scopeList(scope) + if result['OK']: + for elem in result['Value']: + # file only for a time being: + if elem['type'] == 'FILE': + name = elem['name'] + replicasListing = self.replicaClient.listReplicas([{'scope': elem['scope'], 'name': name}]) + if replicasListing['OK']: + replicas = replicasListing['Value'] + # only can get size and name, metadata rather limited + size = replicas['bytes'] + mtime = datetime.datetime(1970, 1, 1, 0, 0, 0) + metadata = {'MetaData': {'TYpe': 'File', 'Mode': 509, 'Size': size, 'ModificationDate': mtime}} + if len(name.split(os.sep)) == 1: + files[os.path.join(path, name)] = metadata + else: + return replicasListing + else: + return result + pathDict = {'Files': files, 'SubDirs': subDirs, 'Links': links} + return S_OK(pathDict) + + @checkCatalogArguments + def addFile(self, lfns): + """ + Upload and register a local file with Rucio file catalog. + + :param lfns: a list containing logical filenames + :return: Dirac S_OK object + """ + successful = {} + failed = {} + gLogger.debug("Rucio addFile (lfns): ", lfns) + for lfn in lfns: + lfnInfo = lfns[lfn] + pfn = lfnInfo['PFN'] + size = lfnInfo['Size'] + se = lfnInfo['SE'] + guid = lfnInfo['GUID'] + checksum = lfnInfo['Checksum'] + VO, scope, name = self.__getLfnElements(lfn) + rse = self.__dirac2RucioSE(se) + res = self.replicaClient.addReplica(rse, scope, name, size, checksum) + if res['OK']: + gLogger.debug(" Rucio replica %s registered successfully " % name) + successful[lfn] = True + else: + failed[lfn] = " Rucio replica %s registration failed " % name + return S_OK({'Failed': failed, 'Successful': successful}) + + @checkCatalogArguments + def exists(self, lfns): + """ + Check whether parts exists in Rucio. + + :param lfns: LFN list to check for existance + :return: Dirac S_OK object + """ + + failed = {} + successful = {} + + for lfn in lfns: + _, scope, didname = self.__getLfnElements(lfn) + metaresult = self.didClient.getMetadata(scope, didname) + successful[lfn] = metaresult['OK'] + return S_OK({'Failed': failed, 'Successful': successful}) + + @checkCatalogArguments + def getReplicas(self, lfns, allStatus=False, active=True): + """ + Get file replicas from Rucio. + + :param lfns: list of Dirac logical file names + :type lfns: list + :param allStatus: currently unused + :type allStatus: bool + :param active: look only at active SEs (currently unusedO + :type active: bool + :return: Dirac S_OK object + :rtype: dict + """ + lfnChunks = breakListIntoChunks(lfns, 1000) + failed = {} + successful = {} + fullDidList = [] + voDict = {} + + for lfnList in lfnChunks: + for lfn in lfnList: + if lfn: + did = self.__getDid(lfn) + fullDidList.append(did) + voDict[did['scope'] + ':' + did['name']] = self.__getLfnElements(lfn)[0] + + replicasListing = self.replicaClient.listReplicas(fullDidList) + if replicasListing['OK']: + rep = replicasListing['Value'] + if rep: + key = rep['scope'] + ':' + rep['name'] + lfn = os.path.join('/', voDict[key], rep['scope'], rep['name']) + for rse in rep['rses']: + se = self.__rucio2DiracSE(rse) + successful.setdefault(lfn, {})[se] = rep['rses'][rse][0] + else: + pass + # unclear failed.update({did['name']:'Error getting replicas from Did' for did in fullDidList}) + return S_OK({'Successful': successful, 'Failed': failed}) + + @checkCatalogArguments + def getFileMetadata(self, lfns): + """ + Get file metadata from the catalog. + + :param lfns: Logical file names (Dirac style) + :type lfns: list + :return: Dirac object with successful and failed metadata keyed by Dirac lfn + :rtype: dict + """ + successful = {} + failed = {} + + for lfn in lfns: + lfn = lfn.encode("ascii") + # need a Rucio lfn here: + rlfn = self.__diracLFN2RucioLFN(lfn) + scope, name = rlfn.split(':') + metaresult = self.didClient.getMetadata(scope, name) + if metaresult['OK']: + meta = metaresult['Value'] + if meta['did_type'] in ['DATASET', 'CONTAINER']: + pass + else: + successful[lfn] = {'Checksum': meta['adler32'].encode("ascii"), 'ChecksumType': 'Adler32', + 'CreationDate': meta['created_at'], 'GUID': meta['guid'], 'Mode': 436, + 'ModificationDate': meta['updated_at'], 'NumberOfLinks': 1, + 'Size': meta['bytes'], 'Status': '-'} + else: + failed[rlfn] = meta['Message'] + return S_OK({'Successful': successful, 'Failed': failed}) + + def getPfnFromLfn(self, lfn, se): + """ + Get a physical file name on a Rucio storage element from a supplied + logical file name. + + :param lfn: Dirac LFN to be translated to Rucio pfn + :type lfn: str + :param se: Dirac SE name + :type se: str + :return: a Dirac return object with pfn as a Value in case of success or an error object. + :rtype: dict + """ + lfn = self.__diracLFN2RucioLFN(lfn) + try: + pfn = self.client.lfns2pfns(self.__diracRucioSE(se), [lfn]) + return S_OK(pfn[lfn]) + except Exception as exc: + return S_ERROR(str(exc)) + + def __getLfnElements(self, lfn): + """ + Get the VO, scope and the did name from the Dirac lfn. + + :param lfn: Dirac LFN + :type lfn: str + """ + parts = lfn.split('/') + VO = parts[1] + scope = parts[2] + name = os.path.join('', *parts[3:]) + return VO, scope, name + + def __diracLFN2RucioLFN(self, lfn): + """ + Convert a Dirac LFN to a Rucio LFN. + + :param lfns: + :type lfns: + :return: + :rtype: + """ + VO, scope, name = self.__getLfnElements(lfn) + return scope + ':' + name + + def __dirac2RucioSE(self, se): + """ + Get Rucio RSE name from Dirac SE name. + + :param se: + :type se: + :return: + :rtype: + """ + se2rse = self.__diracRucioSEMap() + return se2rse.get(se, se) + + def __rucio2DiracSE(self, rse): + """ + Reverse RSE - SE mapping. + + :param rse: + :type rse: + :return: + :rtype: + """ + se2rse = self.__diracRucioSEMap() + rse2se = {se2rse[k]: k for k in se2rse} + return rse2se.get(rse, rse) + + def __diracRucioSEMap(self): + """ + Dirac SE to Rucio RSE map, has to be 1 to 1. + :return: + :rtype: + """ + se2rse = {'UKI-LT2-IC-HEP-disk': 'UKI-LT2-IC-HEP-DISK'} + return se2rse + + def __getDid(self, lfn): + """ + Convert a Dirac-style lfn to Rucio Did dictionary. + :param lfn: Dirac lfn + :type lfn: str + :return: Rucio Did + :rtype: dict + """ + + vo, scope, name = self.__getLfnElements(lfn) + return {'scope': scope, 'name': name} diff --git a/Resources/Catalog/RucioRESTClientApi/BaseClient.py b/Resources/Catalog/RucioRESTClientApi/BaseClient.py new file mode 100644 index 00000000000..d03211f02fd --- /dev/null +++ b/Resources/Catalog/RucioRESTClientApi/BaseClient.py @@ -0,0 +1,349 @@ +# -*- coding: utf-8 -*- +# Copyright 2012-2020 CERN +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Thomas Beermann , 2012-2020 +# - Vincent Garonne , 2012-2018 +# - Yun-Pin Sun , 2013 +# - Mario Lassnig , 2013-2020 +# - Cedric Serfon , 2014-2020 +# - Ralph Vigne , 2015 +# - Joaquín Bogado , 2015-2018 +# - Martin Barisits , 2016-2020 +# - Tobias Wegner , 2017 +# - Brian Bockelman , 2017-2018 +# - Robert Illingworth , 2018 +# - Hannes Hansen , 2018 +# - Tomas Javurek , 2019-2020 +# - Brandon White , 2019 +# - Ruturaj Gujar , 2019 +# - Eric Vaandering , 2019 +# - Jaroslav Guenther , 2019-2020 +# - Andrew Lister , 2019 +# - Eli Chadwick , 2020 +# - Patrick Austin , 2020 +# - Benedikt Ziemons , 2020 + +''' + Client class for callers of the Rucio system. Based on: + + https://github.com/rucio/rucio/blob/master/lib/rucio/client/baseclient.py + + Modified to be used by Dirac. Modifications include: + - keep only x509 authentication, + - eliminate any references to Rucio config file. All values needed to configure the client are + obtained from Dirac CS, + - avoid throwing exceptions. They are converted to Dirac S_ERROR or S_OK objects. + +''' + + +from __future__ import print_function + +import os +from os import environ, fdopen, makedirs, geteuid +from shutil import move +from tempfile import mkstemp +from urlparse import urlparse + +from requests import Session +from requests.status_codes import codes, _codes +from requests.exceptions import ConnectionError, RequestException + +from DIRAC.Core.Security import ProxyInfo +from DIRAC.Core.Security import Locations +from DIRAC import gLogger, S_OK, S_ERROR +from DIRAC.Resources.Catalog.RucioRESTClientApi import Utils + + +class BaseClient(object): + + AUTH_RETRIES, REQUEST_RETRIES = 2, 3 + TOKEN_PATH_PREFIX = Utils.getTempDir() + '/.rucio_' + TOKEN_PREFIX = 'auth_token_' + TOKEN_EXP_PREFIX = 'auth_token_exp_' + + def __init__(self, rucioHost=None, authHost=None, account=None, userAgent='rucio-clients', + timeout=600): + self.rucioHost = rucioHost + self.authHost = authHost + self.session = Session() + self.account = account + self.headers = {} + self.timeout = timeout + self.request_retries = self.REQUEST_RETRIES + self.tokenExpEpoch = None + self.tokenExpEpochFile = None + self.log = gLogger.getSubLogger("FileCatalog") + self.creds = {} + self.userAgent = userAgent + self.scriptID = 'python' + self.tokenPath = self.TOKEN_PATH_PREFIX + self.account + self.vo = None + + def authenticate(self): + """ + Performs X509 authentication. Gets a Dirac proxy and maintains a token. + + :return: S_OK or S_ERROR Dirac object. + :rtype: dict + """ + + # Get Dirac Proxy info: + proxyInfo = ProxyInfo.getProxyInfo() + if proxyInfo['OK']: + value = proxyInfo['Value'] + path = value['path'] + self.creds['client_proxy'] = path + timeleft = value['secondsLeft'] + if timeleft <= 0.0: + self.log.error("Proxy expired") + result = S_ERROR('Proxy expired') + self.vo = ProxyInfo.getVOfromProxyGroup().get('Value', None) + self.tokenPath += '@%s' % self.vo + self.tokenFile = self.tokenPath + '/' + self.TOKEN_PREFIX + self.account + else: + result = S_ERROR(proxyInfo.get('Message', 'Cannot find a proxy file (reson unavailable')) + + # scheme logic + rucio_scheme = urlparse(self.rucioHost).scheme + auth_scheme = urlparse(self.authHost).scheme + + if rucio_scheme != 'http' and rucio_scheme != 'https': + result = S_ERROR('rucio scheme \'%s\' not supported' % rucio_scheme) + + if auth_scheme != 'http' and auth_scheme != 'https': + result = S_ERROR('auth scheme \'%s\' not supported' % auth_scheme) + + # CA cert directory + self.caCertPath = Locations.getCAsLocation() + + # account ? + if self.account is None: + self.log.info('No account passed. Trying to get it from the environment') + try: + self.account = environ['RUCIO_ACCOUNT'] + except KeyError: + self.log.error("No account can be determined. Set the env varriable ?") + result = S_ERROR("No account can be determined. Set the env varriable ?") + + # Authenticate + result = self.__authenticate() + return result + + def __authenticate(self): + """ + Main method for authentication. It first tries to read a locally saved token. + If not available it requests a new one. + """ + + result = self.__read_token() + if result['OK']: + if not result['Value']: + result = self.__get_token() + return result + + def _sendRequest(self, url, headers=None, type='GET', data=None, params=None, stream=False): + """ + Helper method to send requests to the rucio server. + Gets a new token and retries if an unauthorized error is returned. + + :param url: the http url to use. + :param headers: additional http headers to send. + :param type: the http request type to use. + :param data: post data. + :param params: (optional) Dictionary or bytes to be sent in the url query string. + :return: the HTTP return body. + """ + hds = {'X-Rucio-Auth-Token': self.auth_token, 'X-Rucio-Account': self.account, 'X-Rucio-VO': self.vo, + 'Connection': 'Keep-Alive', 'User-Agent': self.userAgent, + 'X-Rucio-Script': self.scriptID} + + if headers is not None: + hds.update(headers) + + result = None + # + for retry in range(self.AUTH_RETRIES + 1): + try: + if type == 'GET': # was stream=True + result = self.session.get( + url, + headers=hds, + verify=self.caCertPath, + timeout=self.timeout, + params=params, + stream=False) + elif type == 'PUT': + result = self.session.put(url, headers=hds, data=data, verify=self.caCertPath, timeout=self.timeout) + elif type == 'POST': + result = self.session.post( + url, + headers=hds, + data=data, + verify=self.caCertPath, + timeout=self.timeout, + stream=stream) + elif type == 'DEL': + result = self.session.delete(url, headers=hds, data=data, verify=self.caCertPath, timeout=self.timeout) + else: + return + except ConnectionError as error: + self.log.error('ConnectionError: ' + str(error)) + if retry > self.request_retries: + return S_ERROR(str(error)) + continue + + if result is not None and result.status_code == codes.unauthorized: # pylint: disable-msg=E1101 + self.session = Session() + self.__get_token() + hds['X-Rucio-Auth-Token'] = self.auth_token + else: + break + + if result is None: + return S_ERROR('Rucio Server Connection Exception') + return result + + def __get_token(self): + """ + Calls the corresponding method to receive an auth token. + To be used if a 401 - Unauthorized error is received. + + :return: Dirac S_OK on success and S_ERROR in cas of an error. + """ + + self.log.debug('get a new token') + + for retry in range(self.AUTH_RETRIES + 1): + result = self.__get_token_x509() + if not result['OK']: + self.log.error('x509 authentication failed for account=%s with identity=%s' % (self.account, + self.creds)) + self.log.error(result['Message']) + + if self.auth_token is not None: + self.__write_token() + self.headers['X-Rucio-Auth-Token'] = self.auth_token + break + + if self.auth_token is None: + return S_ERROR('cannot get an auth token from server') + + return S_OK({}) + + def __get_token_x509(self): + """ + Sends a request to get an auth token from the server and stores it as a class + attribute. Uses x509 authentication. + + :return: S_OK f the token was successfully received. S_ERROR otherwise. + """ + + headers = {'X-Rucio-Account': self.account, 'X-Rucio-VO': self.vo} + + client_cert = None + url = os.path.join(self.authHost, 'auth/x509_proxy') + client_cert = self.creds['client_proxy'] + + if not os.path.exists(client_cert): + self.log.error('given proxy cert (%s) doesn\'t exist' % client_cert) + return S_ERROR('given proxy cert (%s) doesn\'t exist' % client_cert) + + result = None + for retry in range(self.AUTH_RETRIES + 1): + try: + result = self.session.get(url, headers=headers, cert=client_cert, verify=self.caCertPath) + break + except ConnectionError as error: + self.log.error(str(error)) + return S_ERROR(str(error)) + + # Note a response object for a failed request evaluates to false, so we cannot + # use "not result" here + if result is None: + self.log.error('Internal error: Request for authentication token returned no result!') + return S_ERROR('Internal error: Request for authentication token returned no result!') + + if result.status_code != codes.ok: # pylint: disable-msg=E1101 + return S_ERROR(self._getError(status_code=result.status_code, data=result.content)) + + self.auth_token = result.headers['x-rucio-auth-token'] + return S_OK() + + def __read_token(self): + """ + Checks if a local token file exists and reads the token from it. + + :return: True if a token could be read. False if no file exists. + """ + if not os.path.exists(self.tokenFile): + return S_OK(False) + + try: + tokenFile_handler = open(self.tokenFile, 'r') + self.auth_token = tokenFile_handler.readline() + self.headers['X-Rucio-Auth-Token'] = self.auth_token + except IOError as error: + return S_ERROR("I/O error({0}): {1}".format(error.errno, error.strerror)) + except Exception: + return S_ERROR("Exception when reading a token") + self.log.debug('got token from file') + return S_OK(True) + + def __write_token(self): + """ + Write the current auth_token to the local token file. + """ + # check if rucio temp directory is there. If not create it with permissions only for the current user + if not os.path.isdir(self.tokenPath): + try: + self.log.debug('rucio token folder \'%s\' not found. Create it.' % self.tokenPath) + makedirs(self.tokenPath, 0o700) + except Exception as exc: + return S_ERROR(str(exc)) + + # if the file exists check if the stored token is valid. If not request a + # new one and overwrite the file. Otherwise use the one from the file + try: + file_d, file_n = mkstemp(dir=self.tokenPath) + with fdopen(file_d, "w") as f_token: + f_token.write(self.auth_token) + move(file_n, self.tokenFile) + return S_OK() + except IOError as error: + return S_ERROR("I/O error({0}): {1}".format(error.errno, error.strerror)) + except Exception as exc: + return S_ERROR(str(exc)) + + def _getError(self, status_code=None, data=None): + """ + Obtain detailed error message and possibly an exception class name (rather than propagating + any Rucio exceptions) + + :param status_code: HTTP status code + :type status_code: int + :param data: exception data + :type data: str + :return: A combined error message + :rtype: str + """ + try: + data = Utils.parseResponse(data) + except ValueError: + data = {} + + message = data.get('ExceptionMessage', str(_codes.get(status_code, None))) + return data.get('ExceptionClass', 'Undefined') + ': ' + message diff --git a/Resources/Catalog/RucioRESTClientApi/DIDClient.py b/Resources/Catalog/RucioRESTClientApi/DIDClient.py new file mode 100644 index 00000000000..84a0eadff9d --- /dev/null +++ b/Resources/Catalog/RucioRESTClientApi/DIDClient.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# Copyright 2013-2020 CERN +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Vincent Garonne , 2013-2018 +# - Ralph Vigne , 2013-2015 +# - Mario Lassnig , 2013-2020 +# - Martin Barisits , 2013-2020 +# - Yun-Pin Sun , 2013 +# - Thomas Beermann , 2013 +# - Cedric Serfon , 2014-2020 +# - Joaquín Bogado , 2014-2018 +# - Brian Bockelman , 2018 +# - Eric Vaandering , 2018-2020 +# - asket , 2018 +# - Hannes Hansen , 2018 +# - Andrew Lister , 2019 +# - Eli Chadwick , 2020 +# - Aristeidis Fkiaras , 2020 +# - Alan Malta Rodrigues , 2020 +# - Benedikt Ziemons , 2020 + +""" + Based on: + + https://github.com/rucio/rucio/blob/master/lib/rucio/client/didclient.py + + Modified to be used by Dirac. Modifications include: + - only use a limited number of methods from the original class, which are needed for the Dirac Rucio + File Catalog Client. + - eliminate any references to Rucio config file. All values needed to configure the client are + obtained from Dirac CS, + - avoid throwing exceptions. They are converted to Dirac S_ERROR or S_OK objects. + +""" + +try: + from urllib import quote_plus +except ImportError: + from urllib.parse import quote_plus +import os +import random +import json +from requests.status_codes import codes + +from DIRAC.Resources.Catalog.RucioRESTClientApi.BaseClient import BaseClient +from DIRAC import gLogger, S_OK, S_ERROR +from DIRAC.Resources.Catalog.RucioRESTClientApi import Utils + + +class DIDClient(BaseClient): + + """DataIdentifier client class for working with data identifiers""" + + DIDS_BASEURL = 'dids' + ARCHIVES_BASEURL = 'archives' + + def __init__(self, rucioHost=None, authHost=None, account=None): + super(DIDClient, self).__init__(rucioHost, authHost, account) + + def scopeList(self, scope, name=None, recursive=False): + """ + List data identifiers in a scope. + + :param scope: The scope name. + :param name: The data identifier name. + :param recursive: boolean, True or False. + :return: Dirac S_OK holding the response or S_ERROR object in case of an error + """ + + payload = {} + path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), '']) + if name: + payload['name'] = name + if recursive: + payload['recursive'] = True + url = Utils.buildURL(self.rucioHost, path=path, params=payload) + + r = self._sendRequest(url, type='GET') + if r.status_code == codes.ok: + return S_OK([Utils.parseResponse(line) for line in r.text.split('\n') if line]) + else: + S_ERROR(self._getError(status_code=r.status_code, data=r.content)) + + def getMetadata(self, scope, name, plugin='DID_COLUMN'): + """ + Get data identifier metadata. + + :param scope: The scope name. + :param name: The data identifier name. + """ + path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'meta']) + url = os.path.join(self.rucioHost, path) + payload = {} + payload['plugin'] = plugin + r = self._sendRequest(url, type='GET', params=payload) + if r.status_code == codes.ok: + meta = Utils.parseResponse(r.text) # self._load_json_data(r) + return S_OK(meta) + else: + return S_ERROR(self._getError(status_code=r.status_code, data=r.content)) diff --git a/Resources/Catalog/RucioRESTClientApi/RSEClient.py b/Resources/Catalog/RucioRESTClientApi/RSEClient.py new file mode 100644 index 00000000000..d1f2fecd3d0 --- /dev/null +++ b/Resources/Catalog/RucioRESTClientApi/RSEClient.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# Copyright 2012-2020 CERN for the benefit of the ATLAS collaboration. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Vincent Garonne , 2012-2018 +# - Thomas Beermann , 2012 +# - Mario Lassnig , 2012-2020 +# - Ralph Vigne , 2013-2015 +# - Martin Barisits , 2013-2018 +# - Cedric Serfon , 2014 +# - Wen Guan , 2014 +# - Hannes Hansen , 2018 +# - Andrew Lister , 2019 +# + +""" + Based on: + + https://github.com/rucio/rucio/blob/master/lib/rucio/client/rseclient.py + + Modified to be used by Dirac. Modifications include: + - only use a limited number of methods from the original class, which are needed for the Dirac Rucio + File Catalog Client. + - eliminate any references to Rucio config file. All values needed to configure the client are + obtained from Dirac CS. + - avoid throwing exceptions. They are converted to Dirac S_ERROR or S_OK objects. +""" + +import random +import json +from requests.status_codes import codes + +from DIRAC.Resources.Catalog.RucioRESTClientApi.BaseClient import BaseClient +from DIRAC import gLogger, S_OK, S_ERROR +from DIRAC.Resources.Catalog.RucioRESTClientApi import Utils + + +class RSEClient(BaseClient): + """RSE client class for working with rucio RSEs""" + + RSE_BASEURL = 'rses' + + def __init__(self, rucioHost=None, authHost=None, account=None): + super(RSEClient, self).__init__(rucioHost, authHost, account) + + def lfns2pfns(self, rse, lfns, protocol_domain='ALL', operation=None, scheme=None): + """ + Dirac modified version. Returns S_OK with PFNs that should be used at a RSE, + corresponding to requested LFNs. + The PFNs are generated for the RSE *regardless* of whether a replica exists for the LFN. + + :param rse: the RSE name + :param lfns: A list of LFN strings to translate to PFNs. + :param protocol_domain: The scope of the protocol. Supported are 'LAN', 'WAN', and 'ALL' (as default). + :param operation: The name of the requested operation (read, write, or delete). + If None, all operations are queried. + :param scheme: The identifier of the requested protocol (gsiftp, https, davs, etc). + :returns: S_OK with a dictionary of LFN / PFN pair or S_ERROR Dirac object with an appropriate message. + """ + path = '/'.join([self.RSE_BASEURL, rse, 'lfns2pfns']) + params = [] + if scheme: + params.append(('scheme', scheme)) + if protocol_domain != 'ALL': + params.append(('domain', protocol_domain)) + if operation: + params.append(('operation', operation)) + for lfn in lfns: + params.append(('lfn', lfn)) + + url = Utils.buildURL(self.rucioHost, path=path, params=params, doseq=True) + + r = self._sendRequest(url, type='GET') + if r.status_code == codes.ok: + pfns = json.loads(r.text) + return S_OK(pfns) + else: + S_ERROR(self._getError(status_code=r.status_code, data=r.content)) diff --git a/Resources/Catalog/RucioRESTClientApi/ReplicaClient.py b/Resources/Catalog/RucioRESTClientApi/ReplicaClient.py new file mode 100644 index 00000000000..9970197a32a --- /dev/null +++ b/Resources/Catalog/RucioRESTClientApi/ReplicaClient.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +# Copyright 2013-2020 CERN for the benefit of the ATLAS collaboration. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Vincent Garonne , 2013-2018 +# - Mario Lassnig , 2013-2019 +# - Cedric Serfon , 2014-2015 +# - Ralph Vigne , 2015 +# - Brian Bockelman , 2018 +# - Martin Barisits , 2018 +# - Hannes Hansen , 2019 +# - Andrew Lister , 2019 +# - Luc Goossens , 2020 +# - Benedikt Ziemons , 2020 +# +""" + Based on: + + https://github.com/rucio/rucio/blob/master/lib/rucio/client/replicaclient.py + + Modified to be used by Dirac. Modifications include: + - only use a limited number of methods from the original class, which are needed for the Dirac Rucio + File Catalog Client. + - eliminate any references to Rucio config file. All values needed to configure the client are + obtained from Dirac CS, + - avoid throwing exceptions. They are converted to Dirac S_ERROR or S_OK objects. +""" + +try: + from urllib import quote_plus +except ImportError: + from urllib.parse import quote_plus + +import os +import random +from datetime import datetime +from json import dumps, loads +from requests.status_codes import codes + +from DIRAC.Resources.Catalog.RucioRESTClientApi.BaseClient import BaseClient +from DIRAC.Resources.Catalog.RucioRESTClientApi import Utils +from DIRAC import gLogger, S_OK, S_ERROR + + +class ReplicaClient(BaseClient): + """Replica client class for working with replicas""" + + REPLICAS_BASEURL = 'replicas' + + def __init__(self, rucioHost=None, authHost=None, account=None): + super(ReplicaClient, self).__init__(rucioHost, authHost, account) + + def listReplicas(self, dids, schemes=None, unavailable=False, + all_states=False, metalink=False, rse_expression=None, + client_location=None, sort=None, domain=None, + resolve_archives=True, resolve_parents=False, + updated_after=None): + """ + List file replicas for a list of data identifiers (DIDs). + + :param dids: The list of data identifiers (DIDs) like : + [{'scope': , 'name': }, {'scope': , 'name': }, ...] + :param schemes: A list of schemes to filter the replicas. (e.g. file, http, ...) + :param unavailable: Also include unavailable replicas in the list. + :param metalink: ``False`` (default) retrieves as JSON, + ``True`` retrieves as metalink4+xml. + :param rse_expression: The RSE expression to restrict replicas on a set of RSEs. + :param client_location: Client location dictionary for PFN modification {'ip', 'fqdn', 'site'} + :param sort: Sort the replicas: ``geoip`` - based on src/dst IP topographical distance + ``closeness`` - based on src/dst closeness + ``dynamic`` - Rucio Dynamic Smart Sort (tm) + :param domain: Define the domain. None is fallback to 'wan', otherwise 'wan, 'lan', or 'all' + :param resolve_archives: When set to True, find archives which contain the replicas. + :param resolve_parents: When set to True, find all parent datasets which contain the replicas. + :param updated_after: epoch timestamp or datetime object (UTC time), only return replicas updated after this time + :returns: A list of dictionaries with replica information. + """ + data = {'dids': dids, + 'domain': domain} + + if schemes: + data['schemes'] = schemes + if unavailable: + data['unavailable'] = True + data['all_states'] = all_states + + if rse_expression: + data['rse_expression'] = rse_expression + + if client_location: + data['client_location'] = client_location + + if sort: + data['sort'] = sort + + if updated_after: + if isinstance(updated_after, datetime): + # encode in UTC string with format '%Y-%m-%dT%H:%M:%S' e.g. '2020-03-02T12:01:38' + data['updated_after'] = updated_after.strftime('%Y-%m-%dT%H:%M:%S') + else: + data['updated_after'] = updated_after + + data['resolve_archives'] = resolve_archives + + data['resolve_parents'] = resolve_parents + + path = '/'.join([self.REPLICAS_BASEURL, 'list']) + url = os.path.join(self.rucioHost, path) + + headers = {} + if metalink: + headers['Accept'] = 'application/metalink4+xml' + + # pass json dict in querystring + r = self._sendRequest(url, headers=headers, type='POST', data=dumps(data), stream=False) + if r.status_code == codes.ok: + if not metalink: + return S_OK(Utils.parseResponse(r.text)) + return S_OK(r.text) + return S_ERROR(self._getError(status_code=r.status_code, data=r.content)) + + def addReplica(self, rse, scope, name, bytes, adler32, pfn=None, md5=None, meta={}): + """ + Add file replicas to a RSE. + + :param rse: the RSE name. + :param scope: The scope of the file. + :param name: The name of the file. + :param bytes: The size in bytes. + :param adler32: adler32 checksum. + :param pfn: PFN of the file for non deterministic RSE. + :param md5: md5 checksum. + :param meta: Metadata attributes. + :return: True if files were created successfully. + """ + dict = {'scope': scope, 'name': name, 'bytes': bytes, 'meta': meta, 'adler32': adler32} + if md5: + dict['md5'] = md5 + if pfn: + dict['pfn'] = pfn + return self.addReplicas(rse=rse, files=[dict]) + + def addReplicas(self, rse, files, ignore_availability=True): + """ + Bulk add file replicas to a RSE. + + :param rse: the RSE name. + :param files: The list of files. This is a list of DIDs like : + [{'scope': , 'name': }, {'scope': , 'name': }, ...] + :param ignore_availability: Ignore the RSE blacklisting. + :return: True if files were created successfully. + """ + path = self.REPLICAS_BASEURL + url = os.path.join(self.rucioHost, path) + data = {'rse': rse, 'files': files, 'ignore_availability': ignore_availability} + r = self._sendRequest(url, type='POST', data=Utils.render_json(**data)) + if r.status_code == codes.created: + return S_OK(True) + return S_ERROR(self._getError(status_code=r.status_code, data=r.content)) + + def deleteReplicas(self, rse, files, ignore_availability=True): + """ + Bulk delete file replicas from a RSE. + + :param rse: the RSE name. + :param files: The list of files. This is a list of DIDs like : + [{'scope': , 'name': }, {'scope': , 'name': }, ...] + :param ignore_availability: Ignore the RSE blacklisting. + :return: True if files have been deleted successfully. + """ + + path = self.REPLICAS_BASEURL + url = os.path.join(self.rucioHost, path) + data = {'rse': rse, 'files': files, 'ignore_availability': ignore_availability} + r = self._sendRequest(url, type='DEL', data=Utils.render_json(**data)) + if r.status_code == codes.ok: + return S_OK(True) + return S_ERROR(self._getError(status_code=r.status_code, data=r.content)) diff --git a/Resources/Catalog/RucioRESTClientApi/ScopeClient.py b/Resources/Catalog/RucioRESTClientApi/ScopeClient.py new file mode 100644 index 00000000000..e40516ce476 --- /dev/null +++ b/Resources/Catalog/RucioRESTClientApi/ScopeClient.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +# Copyright 2012-2018 CERN for the benefit of the ATLAS collaboration. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Thomas Beermann , 2012 +# - Vincent Garonne , 2012-2018 +# - Mario Lassnig , 2012 +# - Cedric Serfon , 2014 +# - Ralph Vigne , 2015 +# - Brian Bockelman , 2018 +# - Martin Barisits , 2018 +# - Andrew Lister , 2019 +# +# PY3K COMPATIBLE + +""" +Based on: + + https://github.com/rucio/rucio/blob/master/lib/rucio/client/scopeclient.py + + Modified to be used by Dirac. Modifications include: + - only use a limited number of methods from the original class, which are needed for the Dirac Rucio + File Catalog Client. + - eliminate any references to Rucio config file. All values needed to configure the client are + obtained from Dirac CS. + - avoid throwing exceptions. They are converted to Dirac S_ERROR or S_OK objects. +""" + +try: + from urllib import quote_plus +except ImportError: + from urllib.parse import quote_plus + +import os +import random +from json import loads +from requests.status_codes import codes + +from DIRAC.Resources.Catalog.RucioRESTClientApi.BaseClient import BaseClient +from DIRAC import gLogger, S_OK, S_ERROR + + +class ScopeClient(BaseClient): + + """Scope client class for working with rucio scopes""" + + SCOPE_BASEURL = 'accounts' + + def __init__(self, rucioHost=None, authHost=None, account=None): + super(ScopeClient, self).__init__(rucioHost, authHost, account) + + def addScope(self, account, scope): + """ + Sends the request to add a new scope. + + :param account: the name of the account to add the scope to. + :param scope: the name of the new scope. + :return: S_OK(True) if scope was created successfully or an S_ERROR object + with an appropriate message. + """ + + path = '/'.join([self.SCOPE_BASEURL, account, 'scopes', quote_plus(scope)]) + os.path.join(self.rucioHost, path) + r = self._sendRequest(url, type='POST') + if r.status_code == codes.created: + return S_OK(True) + else: + return S_ERROR(self._getError(status_code=r.status_code, data=r.content)) + + def listScopes(self): + """ + Sends the request to list all scopes. + + :return: Dirac S_OK object with a list containing the names of all scopes or + a S_ERROR object if case of a failure. + """ + + path = '/'.join(['scopes/']) + # # possibly os.path.join(choice(self.list_hosts),path) + url = os.path.join(self.rucioHost, path) + r = self._sendRequest(url) + if r.status_code == codes.ok: + scopes = loads(r.text) + return S_OK(scopes) + else: + return S_ERROR(self._getError(status_code=r.status_code, data=r.content)) + + def listScopesForAccount(self, account): + """ + Sends the request to list all scopes for a rucio account. + + :param account: the rucio account to list scopes for. + :return: a Dirac S_OK object with a list containing the names of all scopes for a rucio account. + or a S_ERROR object if case of a failure. + """ + + path = '/'.join([self.SCOPE_BASEURL, account, 'scopes/']) + + url = os.path.join(self.rucioHost, path) + r = self._sendRequest(url) + if r.status_code == codes.ok: + scopes = loads(r.text) + return S_OK(scopes) + else: + return S_ERROR(self._getError(status_code=r.status_code, data=r.content)) diff --git a/Resources/Catalog/RucioRESTClientApi/Utils.py b/Resources/Catalog/RucioRESTClientApi/Utils.py new file mode 100644 index 00000000000..f006c4f701d --- /dev/null +++ b/Resources/Catalog/RucioRESTClientApi/Utils.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# Copyright 2012-2020 CERN +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Vincent Garonne , 2012-2018 +# - Thomas Beermann , 2012-2018 +# - Mario Lassnig , 2012-2020 +# - Cedric Serfon , 2013-2020 +# - Ralph Vigne , 2013 +# - Joaquín Bogado , 2015-2018 +# - Martin Barisits , 2016-2020 +# - Brian Bockelman , 2018 +# - Tobias Wegner , 2018-2019 +# - Hannes Hansen , 2018-2019 +# - Tomas Javurek , 2019-2020 +# - Andrew Lister , 2019 +# - James Perry , 2019 +# - Gabriele Fronze' , 2019 +# - Jaroslav Guenther , 2019-2020 +# - Eli Chadwick , 2020 +# - Patrick Austin , 2020 +# - Benedikt Ziemons , 2020 +""" +original: https://github.com/rucio/rucio/blob/master/lib/rucio/common/utils.py + +A collection of utilities. These utilities are +used by modified Rucio client code to be used by Dirac. +Names modified to camelCase, as required by Dirac. + +""" +import os +import tempfile +import json +import re +from six import string_types +try: + # Python 2 + from urllib import urlencode, quote +except ImportError: + # Python 3 + from urllib.parse import urlencode, quote + + +def datetimeParser(dct): + """ datetime parser + """ + for k, v in list(dct.items()): + if isinstance(v, string_types) and re.search(" UTC", v): + try: + dct[k] = datetime.datetime.strptime(v, DATE_FORMAT) + except Exception: + pass + return dct + + +def parseResponse(data): + """ + JSON render function + """ + ret_obj = None + try: + ret_obj = data.decode('utf-8') + except AttributeError: + ret_obj = data + + return json.loads(ret_obj, object_hook=datetimeParser) + + +def buildURL(url, path=None, params=None, doseq=False): + """ + Utility function to build an url for requests to the rucio system. + If the optional parameter doseq is evaluates to True, individual key=value pairs + separated by '&' are generated for each element of the value sequence for the key. + """ + complete_url = url + if path is not None: + complete_url += "/" + path + if params is not None: + complete_url += "?" + if isinstance(params, str): + complete_url += quote(params) + else: + complete_url += urlencode(params, doseq=doseq) + return complete_url + + +def getTempDir(): + return os.path.abspath(tempfile.gettempdir()) + + +class APIEncoder(json.JSONEncoder): + """ + Proprietary JSONEconder subclass used by the json render function. + This is needed to address the encoding of special values. + """ + + def default(self, obj): # pylint: disable=E0202 + if isinstance(obj, datetime.datetime): + # convert any datetime to RFC 1123 format + return date_to_str(obj) + elif isinstance(obj, (datetime.time, datetime.date)): + # should not happen since the only supported date-like format + # supported at dmain schema level is 'datetime' . + return obj.isoformat() + elif isinstance(obj, datetime.timedelta): + return obj.days * 24 * 60 * 60 + obj.seconds + elif isinstance(obj, EnumSymbol): + return obj.description + elif isinstance(obj, (InternalAccount, InternalScope)): + return obj.external + return json.JSONEncoder.default(self, obj) + + +def render_json(**data): + """ + JSON render function + """ + return json.dumps(data, cls=APIEncoder) diff --git a/Resources/Catalog/RucioRESTClientApi/__init__.py b/Resources/Catalog/RucioRESTClientApi/__init__.py new file mode 100644 index 00000000000..e69de29bb2d