Skip to content

Commit 5c612fc

Browse files
barnybarny
committed
made httpops more resistant to failed network, made _project.py _get_typeuri_rdf resist a 404 exception which is allowed ot be swallowed
1 parent a4e2e20 commit 5c612fc

24 files changed

+9100
-992
lines changed

‎elmclient/_project.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,11 +74,16 @@ def _get_typeuri_rdf(self,uri):
7474
# print( f"Retrieving {uri}" )
7575
logger.info( utils.callers() )
7676
try:
77-
self._gettypecache[realuri] = self.execute_get_rdf_xml(uri, intent="Retrieve type definition") if uri.startswith( "https://") else None
77+
# try to retrieve the rdf using no_error_log=True becase some urls don't exist and we don't really mind if it can't be retrieved, e.g. /jts/users/unassigned which never exits in JTS
78+
self._gettypecache[realuri] = self.execute_get_rdf_xml(uri, intent="Retrieve type definition", no_error_log=True) if uri.startswith( "https://") else None
7879
logger.info( f"Retrieved:" )
7980
except ET.XMLSyntaxError:
8081
self._gettypecache[realuri] = None
8182
logger.info( "Bad result - ignoring" )
83+
except requests.HTTPError:
84+
self._gettypecache[realuri] = None
85+
logger.info( "Bad result - ignoring" )
86+
8287
# print( f"Returning {realuri} {self._gettypecache[realuri]} {ET.tostring(self._gettypecache[realuri])}" )
8388
return self._gettypecache[realuri]
8489

‎elmclient/httpops.py‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -513,30 +513,40 @@ def execute( self, no_error_log=False, close=False, **kwargs ):
513513

514514
# execute the request, retrying with increasing delays (login isn't handled at this level but at lower level)
515515
def _execute_request( self, *, no_error_log=False, close=False, cacheable=True, **kwargs ):
516-
for wait_dur in [2, 5, 10, 20, 30, 60, 0]:
516+
for wait_dur in [2, 5, 10, 0]: # 20, 30, 60,
517+
result=None
517518
try:
518519
if not self._session.alwayscache and not cacheable:
519520
# add a header so the response isn't cached
520521
self._req.headers['Cache-Control'] = "no-store, max-age=0"
521522
result = self._execute_one_request_with_login( no_error_log=no_error_log, close=close, **kwargs)
522523
return result
523524
except requests.RequestException as e:
525+
# print( f"ERROR {e=}" )
526+
if no_error_log:
527+
raise
524528
# ALWAYS retry until all retry delays have been tried, the raise
525-
# logger.info( f"Got error on HTTP request. URL: {self._req.url}, {e.response.status_code}, {e.response.text}")
526529
logger.exception( f"Httpops RequestException was thrown for URL: {self._req.url} exception: {repr(e)}" )
527-
if wait_dur == 0
528-
logger.error( "HTTPOPS not succeeded after all timeouts! - giving up!"
530+
531+
if not self._is_retryable_error( e, result ):
532+
raise
533+
534+
if wait_dur == 0:
535+
logger.error( "HTTPOPS not succeeded after all timeouts! - giving up!" )
529536
raise
530-
logger.error( f"HTTPOPS pausing for {wait_dur} then retrying"
537+
logger.error( f"HTTPOPS pausing for {wait_dur} then retrying" )
531538
time.sleep(wait_dur)
532-
except: Exception as e:
539+
except Exception as e:
540+
print( f"Unexpected exception was thrown for URL: {self._req.url}" )
533541
logger.exception( f"Unexpected exception was thrown for URL: {self._req.url}" )
534542
raise
535543

544+
logger.error( "HTTPOPS not succeeded after all timeouts! - giving up!" )
536545
raise Exception('programming error this point should never be reached')
537546

538547

539-
logger.error( "HTTPOPS not succeeded after all timeouts! - giving up!" # log a request/response, which may be the result of one or more redirections, so first log each of their request/response
548+
549+
# log a request/response, which may be the result of one or more redirections, so first log each of their request/response
540550
def log_redirection_history( self, response, intent, action=None, donotlogbody=False ):
541551
thisintent = intent
542552
after = ""
@@ -679,11 +689,13 @@ def _log_response( self, response, action=None ):
679689
return logtext
680690

681691
# categorize a Requests .send() exception e as to whether is retriable
682-
def _is_retryable_error( self, e ):
692+
def _is_retryable_error( self, e, resp ):
683693
if self._session.auto_retry:
684-
if e.response is None:
685-
return True
686-
if e.response.status_code in [
694+
# print( f"{e=} {resp=}" )
695+
696+
if resp is None or resp.response is None:
697+
return False
698+
if resp.response.status_code in [
687699
http.client.REQUEST_TIMEOUT,
688700
http.client.LOCKED,
689701
# http.client.INTERNAL_SERVER_ERROR,

‎elmclient/tests/results/ccm301.html‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1388,12 +1388,6 @@ <h2>Shapes<h2>
13881388
<TD>https://jazz.ibm.com:9443/jts/users/tokenuser</TD>
13891389
<TD>https://jazz.ibm.com:9443/jts/users/tokenuser</TD>
13901390
</TR>
1391-
<TR> <TD></TD>
1392-
<TD></TD>
1393-
<TD>https://jazz.ibm.com:9443/jts/users/unassigned</TD>
1394-
<TD>https://jazz.ibm.com:9443/jts/users/unassigned</TD>
1395-
<TD>https://jazz.ibm.com:9443/jts/users/unassigned</TD>
1396-
</TR>
13971391
<TR> <TD></TD>
13981392
<TD></TD>
13991393
<TD>https://jazz.ibm.com:9443/jts/users/ursula</TD>

‎elmclient/tests/results/gcm401.html‎

Lines changed: 33 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ <h2>Shapes<h2>
163163
<TD></TD>
164164
</TR>
165165
<TR> <TD></TD>
166-
<TD>'RDF predicate URI'</TD>
166+
<TD>'RDF data type URI'</TD>
167167
<TD>sameAs</TD>
168168
<TD>owl:sameAs</TD>
169169
<TD></TD>
@@ -229,7 +229,7 @@ <h2>Shapes<h2>
229229
<TD></TD>
230230
</TR>
231231
<TR> <TD></TD>
232-
<TD>'RDF predicate URI'</TD>
232+
<TD>'RDF data type URI'</TD>
233233
<TD>sameAs</TD>
234234
<TD>owl:sameAs</TD>
235235
<TD></TD>
@@ -337,7 +337,7 @@ <h2>Shapes<h2>
337337
<TD></TD>
338338
</TR>
339339
<TR> <TD></TD>
340-
<TD>'RDF predicate URI'</TD>
340+
<TD>'RDF data type URI'</TD>
341341
<TD>sameAs</TD>
342342
<TD>owl:sameAs</TD>
343343
<TD></TD>
@@ -397,7 +397,7 @@ <h2>Shapes<h2>
397397
<TD></TD>
398398
</TR>
399399
<TR> <TD></TD>
400-
<TD>'RDF predicate URI'</TD>
400+
<TD>'RDF data type URI'</TD>
401401
<TD>sameAs</TD>
402402
<TD>owl:sameAs</TD>
403403
<TD></TD>
@@ -452,8 +452,8 @@ <h2>Shapes<h2>
452452
</TR>
453453
<TR> <TD></TD>
454454
<TD>Member</TD>
455-
<TD></TD>
456-
<TD>https://jazz.ibm.com:9443/gc/type/shape/datatype/enumerated#A8</TD>
455+
<TD>allowedValue</TD>
456+
<TD>oslc:allowedValue</TD>
457457
<TD></TD>
458458
</TR>
459459
<TR> <TD></TD>
@@ -469,7 +469,7 @@ <h2>Shapes<h2>
469469
<TD></TD>
470470
</TR>
471471
<TR> <TD></TD>
472-
<TD>'RDF predicate URI'</TD>
472+
<TD>'RDF data type URI'</TD>
473473
<TD>sameAs</TD>
474474
<TD>owl:sameAs</TD>
475475
<TD></TD>
@@ -507,7 +507,7 @@ <h2>Shapes<h2>
507507
<TR> <TD></TD>
508508
<TD>Member</TD>
509509
<TD></TD>
510-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/type/datatypes/enumerated#A2</TD>
510+
<TD>https://jazz.ibm.com:9443/gc/type/shape/datatype/enumerated#A11</TD>
511511
<TD></TD>
512512
</TR>
513513
<TR> <TD></TD>
@@ -523,7 +523,7 @@ <h2>Shapes<h2>
523523
<TD></TD>
524524
</TR>
525525
<TR> <TD></TD>
526-
<TD>'RDF predicate URI'</TD>
526+
<TD>'RDF data type URI'</TD>
527527
<TD>sameAs</TD>
528528
<TD>owl:sameAs</TD>
529529
<TD></TD>
@@ -559,7 +559,7 @@ <h2>Shapes<h2>
559559
<TD></TD>
560560
</TR>
561561
<TR> <TD></TD>
562-
<TD>'RDF predicate URI'</TD>
562+
<TD>'RDF data type URI'</TD>
563563
<TD>sameAs</TD>
564564
<TD>owl:sameAs</TD>
565565
<TD></TD>
@@ -585,13 +585,13 @@ <h2>Shapes<h2>
585585
<TR> <TD></TD>
586586
<TD>Contributor</TD>
587587
<TD></TD>
588-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/components#A3</TD>
588+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/components/_WhBhQXaLEfCsldegwSKHYQ#A5</TD>
589589
<TD></TD>
590590
</TR>
591591
<TR> <TD></TD>
592592
<TD>Creator</TD>
593593
<TD></TD>
594-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/components#A5</TD>
594+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/components/_WhBhQXaLEfCsldegwSKHYQ#A1</TD>
595595
<TD></TD>
596596
</TR>
597597
<TR> <TD></TD>
@@ -621,7 +621,7 @@ <h2>Shapes<h2>
621621
<TR> <TD></TD>
622622
<TD>'Project area'</TD>
623623
<TD></TD>
624-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/components#A10</TD>
624+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/components/_WhBhQXaLEfCsldegwSKHYQ#A0</TD>
625625
<TD></TD>
626626
</TR>
627627
<TR> <TD></TD>
@@ -663,13 +663,13 @@ <h2>Shapes<h2>
663663
<TR> <TD></TD>
664664
<TD>Contributor</TD>
665665
<TD></TD>
666-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/components/_WhBhQXaLEfCsldegwSKHYQ#A9</TD>
666+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/components#A6</TD>
667667
<TD></TD>
668668
</TR>
669669
<TR> <TD></TD>
670670
<TD>Creator</TD>
671671
<TD></TD>
672-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/components/_WhBhQXaLEfCsldegwSKHYQ#A0</TD>
672+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/components#A4</TD>
673673
<TD></TD>
674674
</TR>
675675
<TR> <TD></TD>
@@ -699,7 +699,7 @@ <h2>Shapes<h2>
699699
<TR> <TD></TD>
700700
<TD>'Project area'</TD>
701701
<TD></TD>
702-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/components/_WhBhQXaLEfCsldegwSKHYQ#A2</TD>
702+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/components#A2</TD>
703703
<TD></TD>
704704
</TR>
705705
<TR> <TD></TD>
@@ -843,31 +843,31 @@ <h2>Shapes<h2>
843843
<TR> <TD></TD>
844844
<TD>Committer</TD>
845845
<TD></TD>
846-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A7</TD>
846+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A5</TD>
847847
<TD></TD>
848848
</TR>
849849
<TR> <TD></TD>
850850
<TD>Component</TD>
851851
<TD></TD>
852-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A0</TD>
852+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A1</TD>
853853
<TD></TD>
854854
</TR>
855855
<TR> <TD></TD>
856856
<TD>Contribution</TD>
857857
<TD></TD>
858-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A21</TD>
858+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A12</TD>
859859
<TD></TD>
860860
</TR>
861861
<TR> <TD></TD>
862862
<TD>Contributor</TD>
863863
<TD></TD>
864-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A20</TD>
864+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A13</TD>
865865
<TD></TD>
866866
</TR>
867867
<TR> <TD></TD>
868868
<TD>Creator</TD>
869869
<TD></TD>
870-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A15</TD>
870+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A19</TD>
871871
<TD></TD>
872872
</TR>
873873
<TR> <TD></TD>
@@ -879,7 +879,7 @@ <h2>Shapes<h2>
879879
<TR> <TD></TD>
880880
<TD>'Derived from'</TD>
881881
<TD></TD>
882-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A14</TD>
882+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A7</TD>
883883
<TD></TD>
884884
</TR>
885885
<TR> <TD></TD>
@@ -903,13 +903,13 @@ <h2>Shapes<h2>
903903
<TR> <TD></TD>
904904
<TD>'Previous baseline'</TD>
905905
<TD></TD>
906-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A10</TD>
906+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A15</TD>
907907
<TD></TD>
908908
</TR>
909909
<TR> <TD></TD>
910910
<TD>'Project area'</TD>
911911
<TD></TD>
912-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A1</TD>
912+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations/_WhBhQXaLEfCsldegwSKHYQ#A3</TD>
913913
<TD></TD>
914914
</TR>
915915
<TR> <TD></TD>
@@ -981,25 +981,25 @@ <h2>Shapes<h2>
981981
<TR> <TD></TD>
982982
<TD>Committer</TD>
983983
<TD></TD>
984-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A22</TD>
984+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A3</TD>
985985
<TD></TD>
986986
</TR>
987987
<TR> <TD></TD>
988988
<TD>Component</TD>
989989
<TD></TD>
990-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A16</TD>
990+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A22</TD>
991991
<TD></TD>
992992
</TR>
993993
<TR> <TD></TD>
994994
<TD>Contribution</TD>
995995
<TD></TD>
996-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A5</TD>
996+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A21</TD>
997997
<TD></TD>
998998
</TR>
999999
<TR> <TD></TD>
10001000
<TD>Contributor</TD>
10011001
<TD></TD>
1002-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A17</TD>
1002+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A10</TD>
10031003
<TD></TD>
10041004
</TR>
10051005
<TR> <TD></TD>
@@ -1017,7 +1017,7 @@ <h2>Shapes<h2>
10171017
<TR> <TD></TD>
10181018
<TD>'Derived from'</TD>
10191019
<TD></TD>
1020-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A20</TD>
1020+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A5</TD>
10211021
<TD></TD>
10221022
</TR>
10231023
<TR> <TD></TD>
@@ -1041,13 +1041,13 @@ <h2>Shapes<h2>
10411041
<TR> <TD></TD>
10421042
<TD>'Previous baseline'</TD>
10431043
<TD></TD>
1044-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A14</TD>
1044+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A12</TD>
10451045
<TD></TD>
10461046
</TR>
10471047
<TR> <TD></TD>
10481048
<TD>'Project area'</TD>
10491049
<TD></TD>
1050-
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A4</TD>
1050+
<TD>https://jazz.ibm.com:9443/gc/oslc-query/shape/configurations#A14</TD>
10511051
<TD></TD>
10521052
</TR>
10531053
<TR> <TD></TD>
@@ -1153,7 +1153,7 @@ <h2>Shapes<h2>
11531153
<TD></TD>
11541154
</TR>
11551155
<TR> <TD></TD>
1156-
<TD>'RDF predicate URI'</TD>
1156+
<TD>'RDF data type URI'</TD>
11571157
<TD>sameAs</TD>
11581158
<TD>owl:sameAs</TD>
11591159
<TD></TD>
@@ -1207,7 +1207,7 @@ <h2>Shapes<h2>
12071207
<TD></TD>
12081208
</TR>
12091209
<TR> <TD></TD>
1210-
<TD>'RDF predicate URI'</TD>
1210+
<TD>'RDF data type URI'</TD>
12111211
<TD>sameAs</TD>
12121212
<TD>owl:sameAs</TD>
12131213
<TD></TD>

0 commit comments

Comments
 (0)