From fa04384a53c52399731d9040649e03bf471bd9ab Mon Sep 17 00:00:00 2001 From: Daniel Abercrombie Date: Tue, 22 Jan 2019 13:32:14 -0500 Subject: [PATCH 1/2] Updates that didn't make it to GitHub. --- lib/core/components/impl/mysqlstore.py | 18 ++++++++++++++++++ lib/policy/producers/weblock.py | 2 +- lib/source/impl/phedexreplicainfo.py | 7 ++++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/core/components/impl/mysqlstore.py b/lib/core/components/impl/mysqlstore.py index 0511029..6ecb0cc 100644 --- a/lib/core/components/impl/mysqlstore.py +++ b/lib/core/components/impl/mysqlstore.py @@ -2,10 +2,13 @@ import logging import fnmatch import hashlib +import os from dynamo.core.components.persistency import InventoryStore from dynamo.utils.interface.mysql import MySQL from dynamo.dataformat import Configuration, Partition, Dataset, Block, File, Site, SitePartition, Group, DatasetReplica, BlockReplica +from dynamo.policy.condition import Condition +from dynamo.policy.variables import site_variables LOG = logging.getLogger(__name__) @@ -17,6 +20,16 @@ def __init__(self, config): self._mysql = MySQL(config.db_params) + config_path = os.getenv('DYNAMO_SERVER_CONFIG', '/etc/dynamo/fom_config.json') + + self.fom_config = Configuration(config_path) + self.fom_conditions = [] + + for condition_text, module, conf in self.fom_config.rlfsm.transfer: + if condition_text is not None: # default + condition = Condition(condition_text, site_variables) + self.fom_conditions.append((condition, module, conf)) + def close(self): self._mysql.close() @@ -746,6 +759,10 @@ def _yield_sites(self, sites_tmp = None): #override sid = site_id ) + for cond, module, conf in self.fom_conditions: + if cond.match(site): + site.x509proxy = conf.x509proxy + all_chains = {} for protocol, chain_id, idx, lfn, pfn in self._mysql.xquery(mapping_sql, site_id): try: @@ -1119,6 +1136,7 @@ def delete_blockreplica(self, block_replica): #override return sql = 'DELETE FROM `block_replicas` WHERE `block_id` = %s AND `site_id` = %s' + self._mysql.query(sql, block_id, site_id) sql = 'DELETE FROM `block_replica_files` WHERE `block_id` = %s AND `site_id` = %s' diff --git a/lib/policy/producers/weblock.py b/lib/policy/producers/weblock.py index 1324b69..f0c17a7 100644 --- a/lib/policy/producers/weblock.py +++ b/lib/policy/producers/weblock.py @@ -136,7 +136,7 @@ def get_list(self, inventory): locked = False locks = source.make_request(lock_url[0].replace('`','"')) for lock in locks: - if lock == 1: + if lock: locked = True break if not locked: diff --git a/lib/source/impl/phedexreplicainfo.py b/lib/source/impl/phedexreplicainfo.py index 7aa7e51..6724290 100644 --- a/lib/source/impl/phedexreplicainfo.py +++ b/lib/source/impl/phedexreplicainfo.py @@ -91,6 +91,7 @@ def get_replicas(self, site = None, dataset = None, block = None): #override block_entries = self._phedex.make_request('blockreplicas', options, timeout = 7200) # Removing blocks that have only block_replicas with group None + ''' to_remove = set() for i,block_entry in enumerate(block_entries): @@ -111,7 +112,9 @@ def get_replicas(self, site = None, dataset = None, block = None): #override for i in sorted(to_remove, reverse=True): LOG.info("Discarding %s from list of block replicas" % block_entries[i]['name']) del block_entries[i] - + + ''' + parallelizer = Map() parallelizer.timeout = 7200 @@ -216,6 +219,7 @@ def get_replicas(self, site = None, dataset = None, block = None): #override replica.files = None else: # incomplete - since we cannot know what files are there, we'll just have to pretend there is none + LOG.info("Pretending for %s" % str(replica)) replica.size = 0 replica.files = tuple() @@ -420,6 +424,7 @@ def maker_blockreplicas(block, block_entry, site_check = None): # temporarily make this a list block_replica.file_ids = [] block_replica.size = 0 + LOG.info("Incomplete %s" % str(block_replica)) if 'file' in block_entry: for file_entry in block_entry['file']: From 4a7d2511ec267e405cb9868524d867e4df452287 Mon Sep 17 00:00:00 2001 From: Daniel Abercrombie Date: Thu, 21 Feb 2019 15:30:42 -0500 Subject: [PATCH 2/2] Another update --- lib/source/impl/phedexreplicainfo.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/source/impl/phedexreplicainfo.py b/lib/source/impl/phedexreplicainfo.py index 6724290..71e9e3f 100644 --- a/lib/source/impl/phedexreplicainfo.py +++ b/lib/source/impl/phedexreplicainfo.py @@ -219,7 +219,6 @@ def get_replicas(self, site = None, dataset = None, block = None): #override replica.files = None else: # incomplete - since we cannot know what files are there, we'll just have to pretend there is none - LOG.info("Pretending for %s" % str(replica)) replica.size = 0 replica.files = tuple()