oslcquery now doesn't fail when confgurations have been archived

Author: barny

elmclient/_ccm.py

@@ -281,7 +281,7 @@ def resolve_uri_to_name(self, uri, trytouseasid=False):
                 result = self.server.jts.user_uritoname_resolver(uri)
             else:
                 if uri.startswith( "http://" ) or uri.startswith( "https://" ):
-                    uri1 = rdfxml.uri_to_prefixed_tag(uri)
+                    uri1 = rdfxml.uri_to_prefixed_tag(uri, oktocreate=False,noexception=True)
                     logger.debug( f"Returning the raw URI {uri} so changed it to prefixed {uri1}" )
                     uri = uri1
                 result = uri

elmclient/_queryparser.py

@@ -16,7 +16,7 @@
 
 #
 # This enhanced OSLC Query grammar supports "||", "&&" and bracketed evaluation
-# This is based on the EBNF in OSLC Query 3.0 standard https://tools.oasis-open.org/version-control/browse/wsvn/oslc-core/trunk/specs/oslc-query.html
+# This is based on the EBNF in OSLC Query 3.0 standard https://docs.oasis-open-projects.org/oslc-op/query/v3.0/os/oslc-query.html
 # Differences/enhancements from the OSLC 3.0 definition:
 #  You can combine compound_terms using && and || - note when using these the compound_term MUST be in ( ) (which also makes reading the expression easier)
 #  also accepts just a single compound_term with no ( ) needed (i.e. this is basically the vanilla OSLC Query 3.0 syntax)

elmclient/_rm.py

@@ -358,7 +358,11 @@ def load_components_and_configurations(self,force=False, cacheable=True):
             configs = self.execute_get_xml(compuri+"/configurations", intent="Retrieve project/component's list of all configurations", cacheable=cacheable)
             for conf in rdfxml.xml_find_elements(configs,'.//rdfs:member'):
                 confu = rdfxml.xmlrdf_get_resource_uri(conf)
-                thisconfx = self.execute_get_xml(confu, intent="Retrieve a configuration definition", cacheable=cacheable)
+                try:
+                    thisconfx = self.execute_get_xml(confu, intent="Retrieve a configuration definition", cacheable=cacheable)
+                except:
+                    logger.info( f"Singlemode config ERROR probably archived {confu} !!!!!!!" )
+                    continue
                 conftitle= rdfxml.xmlrdf_get_resource_text(thisconfx,'.//dcterms:title')
                 created = rdfxml.xmlrdf_get_resource_uri(thisconfx, './/dcterms:created')
                 # e.g. http://open-services.net/ns/config#Stream
@@ -396,7 +400,11 @@ def load_components_and_configurations(self,force=False, cacheable=True):
                     confs = configs["http://www.w3.org/2000/01/rdf-schema#member"]
                 for aconf in confs:
                     confu = aconf['@id']
-                    confx = self.execute_get_xml(confu, intent="Retrieve configuration definition RDF", cacheable=cacheable)
+                    try:
+                        confx = self.execute_get_xml(confu, intent="Retrieve configuration definition RDF", cacheable=cacheable)
+                    except:
+                        logger.info( f"Old optin config ERROR probably archived {confu} !!!!!!!" )
+                        continue
                     conftitle = rdfxml.xmlrdf_get_resource_text(confx,'.//dcterms:title')
                     conftype = 'Stream' if 'stream' in confu else 'Baseline'
                     created = rdfxml.xmlrdf_get_resource_uri(confx, './/dcterms:created')
@@ -522,7 +530,7 @@ def load_configs(self, cacheable=True ):
             try:
                 configs_xml = self.execute_get_rdf_xml(confu, intent="Retrieve a configuration definition", cacheable=cacheable)
             except:
-                logger.info( f"Config ERROR {thisconfu} !!!!!!!" )
+                logger.info( f"Config ERROR {thisconfu} ignored (the config was probably archived) !!!!!!!" )
                 continue
 
             confmemberx = rdfxml.xml_find_elements(configs_xml, './/rdfs:member[@rdf:resource]')

elmclient/examples/oneperminute.py

@@ -0,0 +1,95 @@
+##
+## © Copyright 2021- IBM Inc. All rights reserved
+# SPDX-License-Identifier: MIT
+##
+
+# Test for authentication timeout
+# retrieve a protected resource with login, then
+# sits retreiving it every minute with login disabled, i.e. until login fails
+# due to authentication timeout
+
+#import csv
+import logging
+
+import time
+
+import lxml.etree as ET
+
+import elmclient.server as elmserver
+import elmclient.utils as utils
+import elmclient.rdfxml as rdfxml
+
+# setup logging - see levels in utils.py
+#loglevel = "INFO,INFO"
+loglevel = "TRACE,OFF"
+levels = [utils.loglevels.get(l,-1) for l in loglevel.split(",",1)]
+if len(levels)<2:
+    # assert console logging level OFF if not provided
+    levels.append(None)
+if -1 in levels:
+    raise Exception( f'Logging level {loglevel} not valid - should be comma-separated one or two values from DEBUG, INFO, WARNING, ERROR, CRITICAL, OFF' )
+utils.setup_logging( filelevel=levels[0], consolelevel=levels[1] )
+
+logger = logging.getLogger(__name__)
+
+POLLINTERVAL=10
+
+jazzhost = 'https://jazz.ibm.com:9443'
+username = 'ibm'
+password = 'ibm'
+#username = 'tokenuser'
+#password = 'tokenuser'
+
+protected_u = jazzhost+"/rm/process/project-areas"
+
+jtscontext = 'jts' # specifies /jts change to e.g. jts:jts23 if your jts is on a different context root such as /jts23
+rmcontext  = 'rm' # specifies /ccm change to e.g. ccm:ccm2 if your ccm is on a different context root such as /ccm2
+
+#proj = "SGC Planning and Tasks"
+
+#workitemid=38
+
+#outfile = "ccm_simple_findworkitem_output.csv"
+
+# caching control
+# 0=fully cached (but code below specifies queries aren't cached)
+# 1=clear cache initially then continue with cache enabled
+# 2=clear cache and disable caching
+caching = 2
+
+# create our "server" which is how we connect to EWM
+# first enable the proxy so if a proxy is running it can monitor the communication with server (this is ignored if proxy isn't running)
+elmserver.setupproxy(jazzhost,proxyport=8888)
+#theserver = elmserver.JazzTeamServer(jazzhost, username, password, verifysslcerts=False, jtsappstring=f"jts:{jtscontext}", appstring='rm', cachingcontrol=caching)
+theserver = elmserver.JazzTeamServer(jazzhost, username, password, verifysslcerts=False, jtsappstring=f"jts:{jtscontext}", cachingcontrol=caching)
+
+# create the RM application interface
+#rmapp = theserver.find_app( f"rm:{rmcontext}", ok_to_create=True )
+
+#p = ccmapp.find_project( proj )
+
+starttime = time.perf_counter()
+nextendtime = starttime
+autologin = True
+
+while True:
+	# now retrieve the protected resource
+    elapsedsecs = time.perf_counter() - starttime
+    print( f"Runtime {int(elapsedsecs/60)}m {int(elapsedsecs%60):02d}s" )
+
+    resource_x = theserver.execute_get_xml( protected_u, intent="Retrieve protected resource", automaticlogin=autologin, close=True )
+
+    autologin = False
+    
+    while nextendtime < time.perf_counter():
+        nextendtime += POLLINTERVAL
+        
+    time.sleep( nextendtime - time.perf_counter() )
+    
+    if utils.kbhit():
+        ch = utils.getch()
+        print( f"{ch=}" )
+        if ch == b'\x1b':
+            break
+
+print( "Finished" )

elmclient/examples/oslcquery.py

@@ -207,7 +207,7 @@ def do_oslc_query(inputargs=None):
 
     utils.log_commandline( os.path.basename(sys.argv[0]),inputargs )
 
-    if args.password is None:
+    if args.password is None or args.password=="PROMPT":
         args.password = getpass.getpass(prompt=f'Password for user {args.username}: ')
 
     # request proxy config if appropriate

elmclient/httpops.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: MIT
 ##
 
+
 import codecs
 import html.parser
 import http
@@ -558,7 +559,7 @@ def get_auth_path(self, request_url, response):
     #  1. if the response indicates login is required then login and try the request again
     #  2. if request is rejected for various reasons retry with the CSRF header applied
     # supports Jazz Form authorization and Jazz Authorization Server login
-    def _execute_one_request_with_login( self, *, no_error_log=False, close=False, donotlogbody=False, retry_get_after_login=True, remove_headers=None, remove_parameters=None, intent=None, action = None ):
+    def _execute_one_request_with_login( self, *, no_error_log=False, close=False, donotlogbody=False, retry_get_after_login=True, remove_headers=None, remove_parameters=None, intent=None, action = None, automaticlogin=True ):
 #        if intent is None:
 #            raise Exception( "No intent provided!" )
         intent = intent or ""
@@ -610,6 +611,13 @@ def _execute_one_request_with_login( self, *, no_error_log=False, close=False, d
 
             response.raise_for_status()
 
+            if not automaticlogin:
+#                print( f"No auto login {response}" )
+                return response
+            else:
+#                print( f"auto login allowed {response}" )
+                pass
+                
             # check for a non-error response which also indicates that authentication is needed using
             # a special header (in which case the response is not the data requested)
             if 'X-com-ibm-team-repository-web-auth-msg' in response.headers:
@@ -622,6 +630,9 @@ def _execute_one_request_with_login( self, *, no_error_log=False, close=False, d
                     retry_after_login_needed = True
 
         except requests.HTTPError as e:
+            if not automaticlogin:
+                # the only real reason to be here is due to needing auth!
+                raise
             if not no_error_log:
                 logger.trace( f"HTTPError {e}" )
             if e.response.status_code == 401 and 'X-jazz-web-oauth-url' in e.response.headers:
@@ -756,11 +767,12 @@ def _jsa_login(self, auth_url, ap_redirect_url, url):
             # step 2 - check for response indicating
 #            if auth_url_response.status_code != 200 or 'X-JSA-LOGIN-REQUIRED' not in auth_url_response.headers:
 
-            if auth_url_response.status_code != 200 and not ap_redirect-Url:
+            if auth_url_response.status_code != 200 and not ap_redirect_url:
                 return auth_url_response  # no more auth required
             if ap_redirect_url and auth_url_response.status_code==401:
                 if not authurl_response.headers.get( 'WWW-Authenticate',"" ).startswith( "Negotiate"):
                     return auth_url_response  # no more auth required
+                    
             if ap_redirect_url and 'X-JSA-LOGIN-REQUIRED' not in auth_url_response.headers:
                 # app password login
                 # decide if SAML or OIDC, or perhaps we are authenticated and there's nothing else to do!
@@ -779,7 +791,7 @@ def _jsa_login(self, auth_url, ap_redirect_url, url):
                 # do the login
                 username, password = self.get_user_password(auth_url)
                 appassword = self.get_app_password( url )
-                
+#                print( f"{username=} {password=} {appassword=}" )
                 # if redirects are automatically followed on this call to authenticate with the OP, the GET of the original protected resource fails, and so the authentication fails.
                 # this may be because this GET doesn't have headers like OSLC-Core-Version.
                 # Solution is not to follow redirects and ensure that the original GET is repeated, i.e. with the correct headers :-)
@@ -796,6 +808,7 @@ def _jsa_login(self, auth_url, ap_redirect_url, url):
                 if auth_url_response.status_code == 200:
                     # use basic auth - 3iii in https://jazz.net/wiki/bin/view/Main/NativeClientAuthentication
                     username, password = self.get_user_password(auth_url)
+                    print( f"{username=} {password=}" )
                     auth_url_response = self._session.get( str(auth_url), auth=(username, password) )  # Load up them cookies!
         else:
             logger.error('''Something about JSA OIDC login has changed since this script was written. I can no longer determine where to authorize myself.''')