added example of finding ccm workitem

Author: barny

elmclient/_ccm.py

@@ -251,7 +251,45 @@ def is_type_uri(self, uri):
         return False
 
     def resolve_uri_to_name(self, uri, trytouseasid=False):
-        return self.__super__.resolve_uri_to_name(self, uri, trytouseasid=True)
+        logger.info( f"resolve_uri_to_name {uri=}" )
+        if not uri:
+            result = None
+            return result
+        if not uri.startswith('http://') or not uri.startswith('https://'):
+        # try to remove prefix
+            uri1 = rdfxml.tag_to_uri(uri,noexception=True)
+            logger.debug(f"Trying to remove prefix {uri=} {uri1=}")
+            if uri1 is None:
+                return uri
+            if uri1 != uri:
+                logger.debug( f"Changed {uri} to {uri1}" )
+            else:
+                logger.debug( f"NOT Changed {uri} to {uri1}" )
+            # use the transformed URI
+            uri = uri1
+        if not uri.startswith(self.app.baseurl):
+            if self.server.jts.is_user_uri(uri):
+                result = self.server.jts.user_uritoname_resolver(uri)
+                logger.debug(f"returning user")
+                return result
+            uri1 = rdfxml.uri_to_prefixed_tag(uri,noexception=True)
+            logger.debug(f"No app base URL {self.app.baseurl=} {uri=} {uri1=}")
+            return uri1
+        elif not self.is_known_uri(uri):
+            if self.server.jts.is_user_uri(uri):
+                result = self.server.jts.user_uritoname_resolver(uri)
+            else:
+                if uri.startswith( "http://" ) or uri.startswith( "https://" ):
+                    uri1 = rdfxml.uri_to_prefixed_tag(uri)
+                    logger.debug( f"Returning the raw URI {uri} so changed it to prefixed {uri1}" )
+                    uri = uri1
+                result = uri
+            # ensure the result is in the types cache, in case it recurrs the result can be pulled from the cache
+            self.register_name(result,uri)
+        else:
+            result = self.get_uri_name(uri)
+        logger.info( f"Result {result=}" )
+        return result
 
 #################################################################################################
 

elmclient/_project.py

@@ -93,6 +93,7 @@ def report_type_system( self ):
                 shortname +=  " (default)" if self.app.default_query_resource is not None and k==rdfxml.tag_to_uri(self.app.default_query_resource) else ""
                 rows.append( [shortname,k,app_qcdetails[k]])
             # print in a nice table with equal length columns
+            print( f"{rows=}" )
             report += utils.print_in_html(rows,['Short Name', 'URI', 'Query Capability URI'])
 
         report += "<H2>Project Queryable Resource Types, short name and URI</H2>\n"

elmclient/_rm.py

@@ -1066,7 +1066,7 @@ def deliver_changeset( self ):
 #################################################################################################
 
 @utils.mixinomatic
-class _RMApp(_app._App, _typesystem.No_Type_System_Mixin):
+class _RMApp(_app._App, oslcqueryapi._OSLCOperations_Mixin, _typesystem.Type_System_Mixin):
     domain = 'rm'
     project_class = _RMProject
     supports_configs = True
@@ -1116,6 +1116,40 @@ def _get_headers(self, headers=None):
         logger.info( f"rm gh {result=}" )
         return result
 
+    # load the typesystem using the OSLC shape resources listed for all the creation factories and query capabilities
+    def load_types(self, force=False):
+        self._load_types(force)
+
+    # load the typesystem using the OSLC shape resources
+    def _load_types(self,force=False):
+        logger.debug( f"load type {self=} {force=}" )
+
+        # if already loaded, try to avoid reloading
+        if self.typesystem_loaded and not force:
+            return
+
+        self.clear_typesystem()
+
+        # get the services.xml
+        sx = self.retrieve_oslc_catalog_xml()
+        if sx:
+            shapes_to_load = rdfxml.xml_find_elements(sx, './/oslc:resourceShape')
+            print( f"{shapes_to_load=}" )
+            burp
+            
+            pbar = tqdm.tqdm(initial=0, total=len(shapes_to_load),smoothing=1,unit=" results",desc="Loading ERM/DN shapes")
+
+            for el in shapes_to_load:
+                self._load_type_from_resource_shape(el)
+                pbar.update(1)
+
+            pbar.close()
+        else:
+            raise Exception( "services xml not found!" )
+
+        self.typesystem_loaded = True
+        return None
+
     @classmethod
     def add_represt_arguments( cls, subparsers, common_args ):
         '''

elmclient/examples/OSLCQUERY.md

@@ -26,7 +26,7 @@ Of course there are things about a query that can't be elided - you still have t
 Overview - OSLC Query
 =====================
 
-For full details on the OSLC Query specification see https://docs.oasis-open.org/oslc-core/oslc-query/v3.0/oslc-query-v3.0.html
+For full details on the OSLC Query specification see https://docs.oasis-open-projects.org/oslc-op/query/v3.0/os/oslc-query.html
 
 IBM DOORS Next (DN), Engineering Test Manager (ETM), Engineering Workflow Manager (EWM) and Global Configuration Management (GCM) have supported OSLC query for many years now.
 

elmclient/examples/ccm_simple_attachmentdownload.py

@@ -14,6 +14,17 @@
 import elmclient.utils as utils
 import elmclient.rdfxml as rdfxml
 
+# setup logging - see levels in utils.py
+#loglevel = "INFO,INFO"
+loglevel = "TRACE,OFF"
+levels = [utils.loglevels.get(l,-1) for l in loglevel.split(",",1)]
+if len(levels)<2:
+    # assert console logging level OFF if not provided
+    levels.append(None)
+if -1 in levels:
+    raise Exception( f'Logging level {loglevel} not valid - should be comma-separated one or two values from DEBUG, INFO, WARNING, ERROR, CRITICAL, OFF' )
+utils.setup_logging( filelevel=levels[0], consolelevel=levels[1] )
+
 logger = logging.getLogger(__name__)
 
 jazzhost = 'https://jazz.ibm.com:9443'

elmclient/examples/ccm_simple_findworkitem.py

@@ -0,0 +1,79 @@
+##
+## © Copyright 2021- IBM Inc. All rights reserved
+# SPDX-License-Identifier: MIT
+##
+
+# example of using the elmclient package to find a work item using OSLC Query for the id, then getting the RDF XML for it
+# use log2seq to get a sequence diagram of the interactions with EWM from the logs produced
+
+import csv
+import logging
+
+import lxml.etree as ET
+
+import elmclient.server as elmserver
+import elmclient.utils as utils
+import elmclient.rdfxml as rdfxml
+
+# setup logging - see levels in utils.py
+#loglevel = "INFO,INFO"
+loglevel = "TRACE,OFF"
+levels = [utils.loglevels.get(l,-1) for l in loglevel.split(",",1)]
+if len(levels)<2:
+    # assert console logging level OFF if not provided
+    levels.append(None)
+if -1 in levels:
+    raise Exception( f'Logging level {loglevel} not valid - should be comma-separated one or two values from DEBUG, INFO, WARNING, ERROR, CRITICAL, OFF' )
+utils.setup_logging( filelevel=levels[0], consolelevel=levels[1] )
+
+logger = logging.getLogger(__name__)
+
+jazzhost = 'https://jazz.ibm.com:9443'
+username = 'ibm'
+password = 'ibm'
+
+jtscontext = 'jts' # specifies /jts change to e.g. jts:jts23 if your jts is on a different context root such as /jts23
+ccmcontext  = 'ccm' # specifies /ccm change to e.g. ccm:ccm2 if your ccm is on a different context root such as /ccm2
+
+proj = "SGC Planning and Tasks"
+
+workitemid=38
+
+outfile = "ccm_simple_findworkitem_output.csv"
+
+# caching control
+# 0=fully cached (but code below specifies queries aren't cached)
+# 1=clear cache initially then continue with cache enabled
+# 2=clear cache and disable caching
+caching = 0
+
+# create our "server" which is how we connect to EWM
+# first enable the proxy so if a proxy is running it can monitor the communication with server (this is ignored if proxy isn't running)
+elmserver.setupproxy(jazzhost,proxyport=8888)
+theserver = elmserver.JazzTeamServer(jazzhost, username, password, verifysslcerts=False, jtsappstring=f"jts:{jtscontext}", appstring='rm', cachingcontrol=caching)
+
+# create the CMM application interface
+ccmapp = theserver.find_app( f"ccm:{ccmcontext}", ok_to_create=True )
+
+p = ccmapp.find_project( proj )
+
+qcbase = p.get_query_capability_uri("oslc_cm1:ChangeRequest")
+
+# query
+results = p.execute_oslc_query(
+    qcbase,
+    whereterms=[['dcterms:identifier','=',f'"{workitemid}"']],
+    select=['*'],
+#    prefixes={rdfxml.RDF_DEFAULT_PREFIX["dcterms"]:'dcterms'} # note this is reversed - url to prefix
+    )
+
+workitem_u = list(results.keys())[0]
+
+print( f"Work item {workitemid} uri is {workitem_u}" )
+
+# now retrieve it
+workitem_x = p.execute_get_xml( workitem_u, intent="Retrieve the workitem content" )
+
+print( ET.tostring( workitem_x ) )
+
+print( "Finished" )

elmclient/examples/dn_simple_createartifact.py

@@ -16,7 +16,7 @@
 # then you POST some basic content compliant with the shape to the factory URL
 # this must also specify the folder where you want the artifact to be created - which means you need to find the folder URL
 # folders are found using a OSLC Query capability for folders - this returns one level at a time
-# sowill likely need a series of ueries to find an existing folder
+# so will likely need a series of queries to find an existing folder
 
 import logging
 import os.path

elmclient/examples/dn_simple_modulestructure.py

@@ -6,9 +6,9 @@
 # example of accessing the module structure API https://jazz.net/wiki/bin/view/Main/DNGModuleAPI
 # also see https://jazz.net/wiki/bin/view/Main/DNGModuleApiOverview
 # prints the module content with indenting corresponding to headings and calculated section number
-# NOTE NOTE NOTE the section number calculation has not been fully verified/checked - it seems to work after superficial inspection
+# NOTE NOTE NOTE the section number calculation has not been fully verified/checked - it seems to work after superficial inspection but you probably should verify its correctness yourself
 
-# provide on the commandline the id of an artifact in the same component and a new binding for it will be created in the structure - location hardcoded!
+# if you provide on the commandline the id of an artifact in the same component then a new binding for it will be created in the structure - location hardcoded!
 
 import csv
 import logging
@@ -346,7 +346,7 @@ def iterator():
                 raise Exception( "Root not at start of structure!" )
             # toinsert is already prepared
             # get the etag
-            response,etag = c.execute_get_rdf_xml(
+            response,etag = c.execute_get_json(
                 structure_u,
                 cacheable=False,
                 headers={
@@ -394,6 +394,7 @@ def iterator():
 
             # get the structure again afer the update
             modstructure_j = c.execute_get_json(structure_u, cacheable=False, headers={'vvc.configuration': config_u,'DoorsRP-Request-Type':'public 2.0', 'OSLC-Core-Version': None, 'Configuration-Context': None}, intent="Retrieve module structure (JSON) after update"  ) # have to remove the OSLC-Core-Version and Configuration-Context headers, and provide vvc.configuration header
+            print( "Structure updated - finishing now" )
             sys.exit(0)
         # scan all the entries into a dictionary keyed by the URL
         entries = {}

elmclient/examples/oslcquery.py

@@ -70,7 +70,7 @@ def do_oslc_query(inputargs=None):
     # setup arghandler
     parser = argparse.ArgumentParser(description="Perform OSLC query on a Jazz application, with results output to CSV (and other) formats - use -h to get some basic help")
 
-    parser.add_argument('-f', '--searchterms', action='append', default=[], help='**APPS MAY NOT FULLY SUPPORT THIS** A word or phrase to search, returning ranked results"')
+    parser.add_argument('-f', '--searchterms', action='append', default=[], help='**APPS MAY NOT FULLY SUPPORT THIS** A word or phrase to search, returning ranked results - use once for each term and oslcquery will insert the quotes and commas"')
     parser.add_argument('-n', '--null', action='append', default=[], help='Post-filter: A property that must be null (empty) for the resource to be included in the results - you can specify this option more than once')
     parser.add_argument('-o', '--orderby', default='', help='**APPS MAY NOT FULLY SUPPORT THIS** A comma-separated list of properties to sort by - prefix with "+" for ascending, "-" for descending- if -f/--searchterms is specified this orders items with the same oslc:score - to speciy a leading -, use = e.g. -o=-dcterms:title')
     parser.add_argument('-p', '--projectname', default=None, help='Name of the project - omit to run a query on the application')
@@ -80,6 +80,7 @@ def do_oslc_query(inputargs=None):
     parser.add_argument('-u', '--unique', action="store_true", help="Post-filter: Remove results with an rm_nav:parent value which are not-unique in the results on dcterms:identifier - this keeps module artifacts (which don't have rm_nav:parent) and artifacts for modules (which don't have a module artifact)) - RELEVANT ONLY FOR DOORS Next!")
     parser.add_argument('-v', '--value', action='append', default=[], help='Post-filter: A property name that must have a value for the resource to be included in the results - you can specify this option more than once')
     parser.add_argument('-A', '--appstrings', default=None, help=f'A comma-seperated list of apps, the query goes to the first entry, default "{APPSTRINGS}". Each entry must be a domain or domain:contextroot e.g. rm or rm:rm1 - Default can be set using environemnt variable QUERY_APPSTRINGS')
+    parser.add_argument('-B', '--browser', default=None, help='Save results in HTML file and open in a browser')
     parser.add_argument('-C', '--component', help='The local component (optional, you *have* to specify the local configuration using -F)')
     parser.add_argument('-D', '--delaybetweenpages', type=float,default=0.0, help="Delay in seconds between each page of results - use this to reduce overall server load particularly for large result sets or when retrieving many properties")
     parser.add_argument('-E', '--globalproject', default=None, help="The global configuration project - optional if the globalconfiguration is unique in the gcm app")
@@ -187,6 +188,7 @@ def do_oslc_query(inputargs=None):
             serverport=80
         else:
             raise Exception( "Unknown scheme in jazzurl {args.jazzurl}" )
+            
     # now try to connect
     if not server.tcp_can_connect_to_url(serverhost, serverport, timeout=2.0):
         raise Exception( f"Server not contactable {args.jazzurl}" )
@@ -643,7 +645,7 @@ def safeint(s,nonereturns=0):
     print( f"Query result has {len(results.keys())} {resultsentries}" )
 
     # COMPARE IS UNTESTED!
-    if args.outputfile or args.compareresults:
+    if args.outputfile or args.browser or args.compareresults:
         # write to CSV and/or compare with CSV
         # FIRST merge columns with same name - this merges types across components based on name
         # which is only need for queries in a GC. NOTE this doesn't attempt to use RDF URIs which it probably should :-o
@@ -682,13 +684,33 @@ def safeint(s,nonereturns=0):
                     v[sk1] = otherexisting if otherexisting else existing
 
         fieldnames = sorted(headings)
+        
         if args.outputfile:
+            # produce a CSV file
             with open(args.outputfile, 'w', newline='', encoding='utf-8-sig') as csvfile:
                 writer = csv.DictWriter(csvfile, fieldnames=fieldnames, restval='')
                 writer.writeheader()
                 for k, v in results.items():
                     writer.writerow(v)
 
+        if args.browser:
+            # produce an html file and also open it in your browser
+            with open(args.browser, 'w', newline='', encoding='utf-8-sig') as htmlfile:
+                htmlfile.write( "<html><body><table><tr>")
+                for fieldname in fieldnames:
+                    htmlfile.write( f"<th>{fieldname}</th>" )
+                htmlfile.write( "</tr>" )
+                for k, v in results.items():
+                    htmlfile.write( "<tr>" )
+                    for fieldname in fieldnames:
+                        htmlfile.write( f"<td>{v.get( fieldname,"" )}</td>" )
+                    htmlfile.write( "</tr>" )
+                htmlfile.write( "</table>" )
+                htmlfile.write( "</body></html>" )
+            # display the report
+            url = f'file://{os.path.abspath(args.browser)}'
+            webbrowser.open(url, new=2)  # open in new tab
+            
         if args.saveprocessedresults:
             open(args.saveprocessedresults+"_after.json","wt").write(json.dumps(results))
 
@@ -762,7 +784,7 @@ def safeint(s,nonereturns=0):
 #            print( f"creating {outputpath=}" )
             os.makedirs( outputpath, exist_ok=True)
 
-        # basically for RM: retrieve all the result resources (as RDF-XML) and store to one file per resource
+        # intended for RM (not sure what it will do for the other apps): retrieve all the result resources (as RDF-XML) and store to one file per resource
         unknownid = 1
         retrieved = {}
         for k in results.keys():

elmclient/httpops.py

@@ -196,13 +196,15 @@ def execute_delete(self, reluri, *, params=None, headers=None, **kwargs):
         response = request.execute( **kwargs )
         return response
 
-    def execute_get_json(self, reluri, *, params=None, headers=None, **kwargs):
+    def execute_get_json(self, reluri, *, params=None, headers=None, return_etag = False, **kwargs):
         reqheaders = {'Accept': 'application/json'}
         if headers is not None:
             reqheaders.update(headers)
         request = self._get_get_request(reluri=reluri, params=params, headers=reqheaders)
         response = request.execute( **kwargs )
         result = json.loads(response.content)
+        if return_etag:
+            return (result,response.headers['ETag'])
         return result
 
     def execute_get_binary( self, reluri, *, params=None, headers=None, **kwargs):