rm configyuration loading can no wbe incremental - load until the config is found - and oslcquery with -V npow shows a curl equivalent (not very tested!)

Author: barny

elmclient/_rm.py

@@ -84,7 +84,8 @@ def __init__(self, name, project_uri, app, is_optin=False, singlemode=False,defa
         self.default_query_resource = "oslc_rm:Requirement"
         self._iscomponent=False
         self._confs_to_load = []
-
+        self._confstoparent = []
+        
     # save a folder details, and return the new folder instance
     def _savefolder( self, parent, fname, folderuri ):
         ROOTNAME = "+-+root+-+"
@@ -511,13 +512,17 @@ def add_external_component(self,compu):
         self._components[compu]['component'] = c
         return c
 
-    def load_configs(self, cacheable=True ):
-        logger.debug( f"Loading configs {self._confs_to_load=}" )
+    # if given stopatnameoruri returns True if found, False if not (which means all configs have been loaded and it isn't there!)
+    def load_configs(self, cacheable=True, stopatnameoruri=None, verbose=False, incremental=False ):
+        logger.debug( f"Loading configs {self._confs_to_load=}, {stopatnameoruri=}" )
         # load configurations
         # and build a tree with initial baseline as root, alternating baseline and stream nodes each with a list of children, so it can be walked if needed
         self.configTree = anytree.AnyNode(name='theroot',title='root', created=None, typesystem=None, ismutable=False, ischangeset=False )
-        confstoparent = []
+        result = False
+            
         while True:
+            if verbose:
+                print( ".",end="" )
             if not self._confs_to_load:
                 break
             confu = self._confs_to_load.pop()
@@ -546,6 +551,8 @@ def load_configs(self, cacheable=True ):
             confmembers_x = list( set( rdfxml.xml_find_elements(configs_xml, './/oslc_config:Configuration') + rdfxml.xml_find_elements(configs_xml, './/oslc_config:Stream') + rdfxml.xml_find_elements(configs_xml, './/oslc_config:Baseline') + rdfxml.xml_find_elements(configs_xml, './/oslc_config:ChangeSet') ) )
 
             for confmember_x in confmembers_x:
+                if verbose:
+                    print( ">",end="",flush=True )
 #                print( f"========================\n{confmember_x=}" )
 #                print( f"{confmember_x.tag=}" )
 #                print( "XML=",ET.tostring( confmember_x ) )
@@ -563,6 +570,10 @@ def load_configs(self, cacheable=True ):
                 ismutable = False
                 ischangeset = False
                 conftitle = rdfxml.xmlrdf_get_resource_text(confmember_x, './dcterms:title')
+                
+                if stopatnameoruri and ( conftitle==stopatnameoruri or thisconfu==stopatnameoruri ):
+                    result = True
+                    
                 created = rdfxml.xmlrdf_get_resource_uri(confmember_x, './dcterms:created', exceptionifnotfound=True)
 #                print( f"{conftitle=}" )
 #                print( f"{created=}" )
@@ -620,18 +631,21 @@ def load_configs(self, cacheable=True ):
                     thisnode = anytree.AnyNode( None, name=thisconfu, title=conftitle, conftype=conftype, created=created, typesystem=_newtypesystem.TypeSystem(conftitle, thisconfu), ismutable=ismutable ) #TypeSystem(conftitle, thisconfu), ismutable=ismutable, ischangeset=ischangeset )
                     if parentnode is None:
                         # do this one later
-                        confstoparent.append( ( thisnode, theparent_u ) )
+                        self._confstoparent.append( ( thisnode, theparent_u ) )
 #                        print( f"Saved for later {confstoparent[-1]=}" )
                     else:
                         # parent is known so attach to it
                         thisnode.parent = parentnode
 #                        print( f"Config {conftitle} Added config {thisconfu} parent={parentnode}" )
+            # now check if stopatnameoururi is present and if so set result True
+            if result:
+                break
 
         # now iterate over the unparented nodes repeatedly finding their parents until there are none left because all have been parented
-        while confstoparent:
+        while self._confstoparent:
             # we're going to copy unparented node into this new list, then copy the new list into confstoparent
             newconfstoparent = []
-            for (i,nodedetails) in enumerate(confstoparent):
+            for (i,nodedetails) in enumerate(self._confstoparent):
                 (node,theparent_u) = nodedetails
                 # if we find the parent, attach it - if not found, add it to this list of those still needing parent
                 parentnode = anytree.search.find( self.configTree, filter_=lambda n: n.name==theparent_u )
@@ -640,10 +654,14 @@ def load_configs(self, cacheable=True ):
                     thisnode.parent = parentnode
 #                    print( f"Parented {node}" )
                 else:
-                    # remember this still needs parenting
-                    newconfstoparent.append( (node,parent) )
-            confstoparent = newconfstoparent
-
+                    # remember this still needs parenting! Will be found on a later pass
+                    newconfstoparent.append( (node,theparent_u) )
+            self._confstoparent = newconfstoparent
+            
+        if verbose:
+            print( "" )
+            
+        return result
 
     def load_configtree( self, *, fromconfig_u=None, loadbaselines=False, followsubstreams=False, loadchangesets=False, alwayscaching=False ):
         # show the config tree
@@ -703,7 +721,7 @@ def load_configtree( self, *, fromconfig_u=None, loadbaselines=False, followsubs
                         raise Exception( f"Unkown type {typedetails[1]}" )
 
 
-    def get_local_config(self, name_or_uri, global_config_uri=None):
+    def get_local_config(self, name_or_uri, global_config_uri=None, verbose=False, incremental=False ):
         logger.info( f"GLC {self=} {name_or_uri=}" )
 #        print( f"GLC {self=} {name_or_uri=} {global_config_uri=}" )
         if global_config_uri:
@@ -719,7 +737,6 @@ def get_local_config(self, name_or_uri, global_config_uri=None):
                 raise Exception( 'Cannot find configuration [%s] in project [%s]' % (name_or_uri, self.uri))
             result = config_uri
         else:
-            self.load_configs()
             result = None
             filter = None
             if name_or_uri.startswith("S:"):
@@ -731,14 +748,19 @@ def get_local_config(self, name_or_uri, global_config_uri=None):
             elif name_or_uri.startswith("C:"):
                 filter="ChangeSet"
                 name_or_uri = name_or_uri[2:]
-            for cu, cd in self._configurations.items():
-#                print( f"{cu=} {cd=}" )
-                if filter and cd['conftype'] != filter:
-                    continue
-                if cu == name_or_uri or cd['name'] == name_or_uri:
-                    if result:
-                        raise Exception( f"Config {name_or_uri} isn't unique - you could try prefixing it with S: for stream, B: for baseline, or C: for changeset")
-                    result = cu
+
+            while result is None:
+                for cu, cd in self._configurations.items():
+    #                print( f"{cu=} {cd=}" )
+                    if filter and cd['conftype'] != filter:
+                        continue
+                    if cu == name_or_uri or cd['name'] == name_or_uri:
+                        if result:
+                            raise Exception( f"Config {name_or_uri} isn't unique - you could try prefixing it with S: for stream, B: for baseline, or C: for changeset")
+                        result = cu
+                if result is None:
+                    if not self.load_configs( stopatnameoruri=name_or_uri, verbose=verbose, incremental=incremental ):
+                        break
 #        print( f"GLC {result} {self=} {name_or_uri=}" )
 
         return result

elmclient/examples/oslcquery.py

@@ -3,6 +3,10 @@
 # SPDX-License-Identifier: MIT
 ##
 
+#
+# app to exercise OSLC Query capabilities in  (relatively) human-friendly way - the key thing is to allow user to specify things by name and the app finds the corresponding URI
+# See OSLC Query 3.0 standard https://docs.oasis-open-projects.org/oslc-op/query/v3.0/os/oslc-query.html
+#
 
 import argparse
 import csv

elmclient/httpops.py

@@ -11,7 +11,9 @@
 import json
 import logging
 import lxml.etree as ET
+import platform
 import re
+import shlex
 import time
 import urllib
 
@@ -22,6 +24,66 @@
 
 logger = logging.getLogger(__name__)
 
+is_windows = any(platform.win32_ver())
+
+
+def quote( s ):
+    if is_windows:
+        for c in s:
+            # only quote if there's a shell-unfriendly character!
+            if c not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._':
+                return f'"{s}"'
+        return s
+    else:
+        return shlex.quote(s)
+
+# curl login example (non-JAS):
+# curl -k -c cookies.txt "https://jazz.ibm.com:9443/jts/authenticated/identity"
+# curl -k -L -b cookies.txt -c cookies.txt -d j_username=ibm -d j_password=ibm "https://jazz.ibm.com:9443/jts/authenticated/j_security_check"
+
+# derived from https://github.com/ofw/curlify/blob/master/curlify.py but with quoting fixed so on Windows it generates Windows-friendly curl commands
+def to_curl(request, compressed=False, verify=True):
+    """
+    Returns string with curl command by provided request object
+
+    Parameters
+    ----------
+    compressed : bool
+        If `True` then `--compressed` argument will be added to result
+    """
+    parts = [
+        ('curl', None),
+        ('-X', request.method),
+        ('-b','cookies.txt'),
+        ('-c','cookies.txt'),
+    ]
+
+    for k, v in sorted(request.headers.items()):
+        parts += [('-H', '{0}: {1}'.format(k, v))]
+
+    if request.body:
+        body = request.body
+        if isinstance(body, bytes):
+            body = body.decode('utf-8')
+        parts += [('-d', body)]
+
+    if compressed:
+        parts += [('--compressed', None)]
+
+    if not verify:
+        parts += [('--insecure', None)]
+
+    parts += [(None, request.url)]
+
+    flat_parts = []
+    for k, v in parts:
+        if k:
+            flat_parts.append(quote(k))
+        if v:
+            flat_parts.append(quote(v))
+
+    return ' '.join(flat_parts)
+
 # prefix on a password that indicates it's an application password structure
 AP_PREFIX  ="ap:"
 
@@ -559,7 +621,7 @@ def get_auth_path(self, request_url, response):
     #  1. if the response indicates login is required then login and try the request again
     #  2. if request is rejected for various reasons retry with the CSRF header applied
     # supports Jazz Form authorization and Jazz Authorization Server login
-    def _execute_one_request_with_login( self, *, no_error_log=False, close=False, donotlogbody=False, retry_get_after_login=True, remove_headers=None, remove_parameters=None, intent=None, action = None, automaticlogin=True ):
+    def _execute_one_request_with_login( self, *, no_error_log=False, close=False, donotlogbody=False, retry_get_after_login=True, remove_headers=None, remove_parameters=None, intent=None, action = None, automaticlogin=True, showcurl=False ):
 #        if intent is None:
 #            raise Exception( "No intent provided!" )
         intent = intent or ""
@@ -575,6 +637,8 @@ def _execute_one_request_with_login( self, *, no_error_log=False, close=False, d
             # if Configuration-Context is not None:
 #            print( f"Copied header Configuration-Context to parameter oslc_config.context" )
             request.params['oslc_config.context'] = request.headers['Configuration-Context']
+#            del request.headers['Configuration-Context']
+#            print( f"Deleted C-C" )
 
         # ensure keep-alive/close
         if close:
@@ -591,6 +655,7 @@ def _execute_one_request_with_login( self, *, no_error_log=False, close=False, d
 
         # this is for generic API debugging to be able to remove any header before it's actually sent!
         if remove_headers:
+            print( f"{remove_headers=}" )
             for h in remove_headers:
 #                print( f"{request.headers=}" )
                 if h in request.headers:
@@ -602,6 +667,9 @@ def _execute_one_request_with_login( self, *, no_error_log=False, close=False, d
         try:
             prepped = self._session.prepare_request( request )
 
+            if showcurl:
+                print( f"Curl command (assuming auth cookies are in cookies.txt) is {to_curl(prepped)}" )
+
             # check for us using an appp password for this url (context root) and if so extend the User-Agent header 
             prepped.headers['User-Agent'] += addhdr
 
@@ -611,6 +679,8 @@ def _execute_one_request_with_login( self, *, no_error_log=False, close=False, d
 
             response.raise_for_status()
 
+            
+
             if not automaticlogin:
 #                print( f"No auto login {response}" )
                 return response

elmclient/oslcqueryapi.py

@@ -615,7 +615,7 @@ def _execute_vanilla_oslc_query(self, querycapabilityuri, query_params, orderby=
                 intent = f"Retrieve {utils.nth(page)} page of OSLC query results"
 
             # request this page
-            this_result_xml = self.execute_get_rdf_xml(query_url, params=params, headers=headers, cacheable=cacheable, intent=intent)
+            this_result_xml = self.execute_get_rdf_xml(query_url, params=params, headers=headers, cacheable=cacheable, intent=intent, showcurl=verbose)
             queryurls.append(query_url)
 
             if saverawresults: