crm# 250228-003676 pids

Author: markendr

pitschi/db/crud.py

@@ -93,16 +93,16 @@ def get_datasets(db: Session, username: str):
 
 def get_datasets_from_original(db: Session, username: str, origmachine: str, origpath: str):
     return db.query(models.Dataset).\
-            filter(models.Dataset.origionalmachine == origmachine).\
-            filter(models.Dataset.origionalpath == origpath).\
+            filter(models.Dataset.originalmachine == origmachine).\
+            filter(models.Dataset.originalpath == origpath).\
         join(models.Dataset.booking).\
             filter(or_(models.Booking.username == username, models.Booking.assistant == username)).all()
 
 def get_datasets_from_one_machine(db: Session, username: str, origmachine: str, date: datetime.date):
     return db.query(models.Dataset).\
             filter(and_(func.date(models.Dataset.modified) >= date),\
                     (func.date(models.Dataset.modified) <= date )).\
-            filter(models.Dataset.origionalmachine == origmachine).\
+            filter(models.Dataset.originalmachine == origmachine).\
         join(models.Dataset.booking).\
             filter(models.Booking.username == username).all()
 
@@ -314,7 +314,7 @@ def get_core(db: Session, coreid: int):
     return db.query(models.Core).\
             filter(models.Core.id == coreid).first()
 
-def create_core(db: Session, system: schemas.Core):
+def create_core(db: Session, core: schemas.Core):
     """
     Create a PPMS core, or update if needed
     """
@@ -344,6 +344,13 @@ def get_system(db: Session, systemid: int):
     return db.query(models.System).\
             filter(models.System.id == systemid).first()
 
+def get_system_pid(db: Session, systemid: int):
+    return db.query(models.System).\
+            filter(models.System.id == systemid).first().pid
+
+def get_system_ror(db: Session, systemid: int):
+    return get_core(db, get_system(db, systemid).coreid).rorid
+
 def get_system_byname(db: Session, systemname: str):
     return db.query(models.System).\
             filter(models.System.name == systemname).first()

pitschi/db/models.py

@@ -44,8 +44,8 @@ class PUser(Base):
 class Dataset(Base):
     __tablename__ = 'dataset'
     id = Column(Integer, primary_key=True, index=True)
-    origionalmachine = Column(String, unique=False, primary_key=False, index=False, nullable=False)
-    origionalpath = Column(String, unique=False, primary_key=False, index=False, nullable=False)
+    originalmachine = Column(String, unique=False, primary_key=False, index=False, nullable=False)
+    originalpath = Column(String, unique=False, primary_key=False, index=False, nullable=False)
     networkpath = Column(String, unique=False, primary_key=False, index=False, nullable=False)
     relpathfromrootcollection = Column(String, unique=False, primary_key=False, index=False, nullable=False)
     name = Column(String, unique=False, primary_key=False, index=False, nullable=False)

pitschi/db/schemas.py

@@ -67,8 +67,8 @@ class Config:
 #################################
 class DatasetBase(BaseModel):
     relpathfromrootcollection: str # /folder1/folder2 # get q collectio nfrom project
-    origionalmachine: str 
-    origionalpath: str   # C:\dekstop\folder1\folder2
+    originalmachine: str
+    originalpath: str   # C:\dekstop\folder1\folder2
     networkpath: str     # \\data.qbi.uq.edu.au\CMM4CEED-Q3504\folder1\folder2
     name: str
     received: Optional[datetime.datetime]

pitschi/ppms.py

@@ -140,6 +140,24 @@ def get_daily_training(date: datetime.date):
     return sessions
 
 
+def get_cores():
+    coreids = json.loads(config.get('ppms', 'coreids'))
+    logger.debug(f'get cores for core ids: {",".join(str(c) for c in coreids)}')
+    url = f"{config.get('ppms', 'ppms_url')}API2/"
+    payload=f"outformat=json&apikey={config.get('ppms', 'api2_key')}&action=Report2169"
+    headers = {
+      'Content-Type': 'application/x-www-form-urlencoded'
+    }
+    response = requests.request("POST", url, headers=headers, data=payload)
+    if response.ok:
+        if response.status_code != 204:
+            cores = response.json(strict=False)
+            if type(cores) == list and len(cores) > 0:
+                return [c for c in cores if c.get('Core ID') in coreids]
+    logger.warning(f'response status_code={response.status_code}, text="{response.text}"')
+    return []
+
+
 def get_system_pids():
     logger.debug(f'get all system pids')
     url = f"{config.get('ppms', 'ppms_url')}API2/"
@@ -158,7 +176,8 @@ def get_system_pids():
 
 
 def get_systems():
-    logger.debug("get all systems")
+    coreids = json.loads(config.get('ppms', 'coreids'))
+    logger.debug(f'get systems for core ids: {",".join(str(c) for c in coreids)}')
     url = f"{config.get('ppms', 'ppms_url')}pumapi/"
     payload=f"apikey={config.get('ppms', 'ppms_key')}&action=getsystems"
     headers = {
@@ -175,10 +194,11 @@ def get_systems():
             for row in _csv_reader:
                 if(len(row) > 3):
                     _coreid = int(row[0])
-                    _systemid = int(row[1])
-                    _systemtype = row[2]
-                    _systemname = row[3]
-                    systems.append({'coreid': _coreid, 'systemid': _systemid, 'systemtype': _systemtype, 'systemname': _systemname})
+                    if _coreid in coreids:
+                        _systemid = int(row[1])
+                        _systemtype = row[2]
+                        _systemname = row[3]
+                        systems.append({'coreid': _coreid, 'systemid': _systemid, 'systemtype': _systemtype, 'systemname': _systemname})
             return systems
     logger.warning(f'response status_code={response.status_code}, text="{response.text}"')
     return []
@@ -205,7 +225,8 @@ def get_system_rights(systemid: int):
 
 
 def get_projects():
-    logger.debug("get all projects")
+    coreids = json.loads(config.get('ppms', 'coreids'))
+    logger.debug(f'get projects for core ids: {",".join(str(c) for c in coreids)}')
     url = f"{config.get('ppms', 'ppms_url')}pumapi/"
     payload=f"apikey={config.get('ppms', 'ppms_key')}&action=getprojects&active=true&format=json"
     # payload=f"apikey={config.get('ppms', 'ppms_key')}&action=getprojects&format=json"
@@ -217,7 +238,7 @@ def get_projects():
         if response.status_code == 204:
             return []
         else:
-            return response.json(strict=False)
+            return [p for p in response.json(strict=False) if p.get('CoreFacilityRef') in coreids]
     else:
         return []
 
@@ -285,7 +306,6 @@ def get_rdm_collection(coreid: int, projectid: int):
     return ""
 
 def get_rdm_collections(coreid: int = None):
-    logger.debug("@get_rdm_collections: get all rdm collections")
     url = f"{config.get('ppms', 'ppms_url')}API2/"
     headers = { 'Content-Type': 'application/x-www-form-urlencoded' }
     action = config.get('ppms', 'qcollections_action')
@@ -295,6 +315,7 @@ def get_rdm_collections(coreid: int = None):
         coreids = json.loads(config.get('ppms', 'coreids'))
     else:
         coreids = [coreid]
+    logger.debug(f'get rdm collections for core ids: {",".join(str(c) for c in coreids)}')
     for coreid in coreids:
         payload = f"apikey={config.get('ppms', 'api2_key')}&action={action}&coreid={coreid}&outformat=json"
         response = requests.request("POST", url, headers=headers, data=payload)

pitschi/routers/clowder.py

@@ -8,7 +8,7 @@
 import pitschi.config as config
 import pitschi.utils as utils
 import pitschi.clowder_rest as clowderful
-import os, json
+import os
 
 router = APIRouter()
 logger = logging.getLogger('pitschixapi')

pitschi/routers/ppms.py

@@ -90,4 +90,8 @@ async def get_systems( systemid: int,
             headers={"WWW-Authenticate": "Basic"},
         )
     logger.debug(f"Querying systemid {systemid}")
-    return pdb.crud.get_system(db, systemid)
+    system = pdb.crud.get_system(db, systemid)
+    if not system:
+        return {}
+    ror = pdb.crud.get_system_ror(db, systemid)
+    return { 'id': system.id, 'coreid': system.coreid, 'type': system.type, 'name': system.name, 'pid': system.pid, 'ror': ror }

pitschi/routers/ppms_utils.py

@@ -70,9 +70,8 @@ def get_db_user(db: Session, login: str = None, userid: int = None, coreid: int
 def sync_cores(db: Session):
     cores = get_cores()
     for core in cores:
-        _id = core.get('Core ID')
         pdb.crud.create_core(db, pdb.schemas.Core(
-            id = _id,
+            id = core.get('Core ID'),
             institution = core.get('Institution'),
             shortname = core.get('Facility Short Name'),
             longname = core.get('Facility Long Name'),
@@ -182,8 +181,8 @@ def sync_ppms_projects(db: Session, alogger: logging.Logger = logger):
             id = _id,
             coreid = system.get('coreid'),
             type = system.get('systemtype'),
-            name = system.get('systemname')),
-            pid = pids.get(_id, '')
+            name = system.get('systemname'),
+            pid = pids.get(_id, ''))
         )
     sync_projects(db, alogger=alogger, alert=True)
     pdb.crud.set_stat(db, name='syncing_projects', value='False')

pitschi/routers/scheduledingest.py

@@ -101,7 +101,7 @@ def ingest_dataset_to_clowder(db, dataset, project, logger):
                         logger.error(f"@dataset {dataset} does not have any booking.: {dataset.bookingid}")
                         return (False, ["Fail to find dataset booking"])
                     _ds_metadata = {
-                        'system': dataset.origionalmachine,
+                        'system': dataset.originalmachine,
                         'author': _dataset_booking.username,
                         'projectid': _dataset_booking.projectid,
                         'bookingid': _dataset_booking.id,
@@ -110,7 +110,7 @@ def ingest_dataset_to_clowder(db, dataset, project, logger):
                     }
                     clowderful.upload_dataset_metadata(_clowder_key, _clowder_api_url, _ds_info.get('id'), _ds_metadata)
                     ### add tags
-                    _ds_tags = [ dataset.origionalmachine, _dataset_booking.username, str(_dataset_booking.projectid) ]
+                    _ds_tags = [ dataset.originalmachine, _dataset_booking.username, str(_dataset_booking.projectid) ]
                     clowderful.add_dataset_tags(_clowder_key, _clowder_api_url, _ds_info.get('id'), _ds_tags)
             else:
                 found = False
@@ -284,8 +284,8 @@ def send_email(db, datasetinfo, result, messages):
                 <p>Dear admins<br /></p>
                 <p>The following dataset failed to ingest:</p>
                 <ul>
-                    <li><b>Machine</b> {datasetinfo.origionalmachine}</li>
-                    <li><b>Location</b> {datasetinfo.origionalpath}</li>
+                    <li><b>Machine</b> {datasetinfo.originalmachine}</li>
+                    <li><b>Location</b> {datasetinfo.originalpath}</li>
                     <li><b>Booking id:</b> {datasetinfo.booking.id}</li>
                     <li><b>system id:</b> {datasetinfo.booking.systemid}</li>
                     <li><b>username:</b> {datasetinfo.booking.username}</li>

pitschi/routers/sync_ppms_bookings.py

@@ -4,7 +4,7 @@
 from fastapi import APIRouter, Depends, status
 from fastapi_utils.tasks import repeat_every
 import pitschi.db as pdb
-from pitschi.ppms import get_daily_bookings, get_daily_training, get_booking_details
+from pitschi.ppms import get_system_pids, get_daily_bookings, get_daily_training, get_booking_details
 from pitschi.routers import ppms_utils
 from sqlalchemy.orm import Session
 
@@ -31,6 +31,7 @@ def sync_ppms_bookings() -> None:
         logger.debug('query ppms bookings of today')
         _today_tz = datetime.datetime.now(pytz.timezone(config.get('ppms', 'timezone'))).date()
         _bookings = get_daily_bookings(_today_tz)
+        pids = {p['System ID']: p['PID'] for p in get_system_pids() if p.get('PID')}
         logger.debug(f'bookings: {len(_bookings)}')
         # training sessions can have multiple records -- use the record with
         # 'Training organised by' == 'User full Name' to set booking project/user
@@ -46,13 +47,15 @@ def sync_ppms_bookings() -> None:
                 continue
             _booking_details = _booking_details_list[0]
             _booking['systemId'] = None
-            if _booking_details.get('systemId'):
+            _id = _booking_details.get('systemId')
+            if _id:
                 try:
                     _system = pdb.crud.create_system(db, pdb.schemas.System(
-                        id = _booking_details.get('systemId'),
+                        id = _id,
                         coreid = _booking.get('coreid'),
                         type = _booking_details.get('systemType'),
-                        name = _booking_details.get('systemName'))
+                        name = _booking_details.get('systemName'),
+                        pid = pids.get(_id, ''))
                     )
                     _booking['systemId'] = _system.id
                 except: