diff --git a/bin/cafyrun b/bin/cafyrun
new file mode 100644
index 0000000..458d293
--- /dev/null
+++ b/bin/cafyrun
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+import pytest
+import _pytest.main
+import sys
+
+# Customize messages for pytest exit codes
+msg = {_pytest.main.EXIT_OK: 'OK',
+ _pytest.main.EXIT_TESTSFAILED: 'Tests failed',
+ _pytest.main.EXIT_INTERRUPTED: 'Interrupted',
+ _pytest.main.EXIT_INTERNALERROR: 'Internal error',
+ _pytest.main.EXIT_USAGEERROR: 'Usage error',
+ _pytest.main.EXIT_NOTESTSCOLLECTED: 'No tests collected'}
+
+exitcode = pytest.main(sys.argv[1:])
+
+
+
+
+print(msg[exitcode])
+sys.exit(exitcode)
+
diff --git a/pytest_cafy/plugin.py b/pytest_cafy/plugin.py
index 74ff9c3..3031253 100755
--- a/pytest_cafy/plugin.py
+++ b/pytest_cafy/plugin.py
@@ -21,11 +21,12 @@
import pytest
from _pytest.terminal import TerminalReporter
-from _pytest.runner import runtestprotocol
+from _pytest.runner import runtestprotocol, TestReport
from _pytest.mark import MarkInfo
from enum import Enum
from tabulate import tabulate
+from pprint import pprint, pformat
from shutil import copyfile
from configparser import ConfigParser
from datetime import datetime
@@ -122,7 +123,7 @@ def pytest_addoption(parser):
group.addoption('--work-dir', dest="workdir", metavar="DIR", default=None,
help="Path for work dir")
- group.addoption('--report-dir', dest="reportdir",
+ group.addoption('-R','--report-dir', dest="reportdir",
metavar="DIR",
default=None,
help="Path for report dir")
@@ -133,7 +134,7 @@ def pytest_addoption(parser):
type=lambda x: is_valid_param(x, file_type='topology_file'),
help='Filename of your testbed')
- group.addoption('--test-input-file', action='store', dest='test_input_file',
+ group.addoption('-I', '--test-input-file', action='store', dest='test_input_file',
metavar='test_input_file',
type=lambda x: is_valid_param(x, file_type='input_file'),
help='Filename of your test input file')
@@ -534,7 +535,7 @@ def pytest_collection_modifyitems(session, config, items):
log.info("url: {}".format(url))
log.info("Calling API service for live logging of reg_id ")
params = {"reg_id": CafyLog.registration_id}
- response = requests.patch(url, json=params, headers=headers)
+ response = requests.patch(url, json=params, headers=headers, timeout=120)
if response.status_code == 200:
log.info("Calling API service for live logging of reg_id successful")
else:
@@ -558,7 +559,7 @@ def pytest_collection_modifyitems(session, config, items):
url = '{0}/api/runs/{1}/cases'.format(os.environ.get('CAFY_API_HOST'), os.environ.get('CAFY_RUN_ID'))
log.info("url: {}".format(url))
log.info("Calling API service for live logging of collected testcases ")
- response = requests.post(url, json=CafyLog.collected_testcases, headers=headers)
+ response = requests.post(url, json=CafyLog.collected_testcases, headers=headers, timeout=120)
if response.status_code == 200:
log.info("Calling API service for live logging of collected testcases successful")
else:
@@ -789,7 +790,7 @@ def initiate_analyzer(self, reg_id, test_case, debug_server):
try:
url = "http://{0}:5001/initiate_analyzer/".format(CafyLog.debug_server)
self.log.info("Calling registration service (url:%s) to initialize analyzer" % url)
- response = requests.post(url, data=params)
+ response = requests.post(url, data=params, timeout=300)
if response.status_code == 200:
self.log.info("Analyzer initialized")
return True
@@ -813,7 +814,7 @@ def pytest_runtest_teardown(self, item, nextitem):
self.log.set_testcase("Teardown")
else:
testcase_name = self.get_test_name(nextitem.nodeid)
- self.log.set_testcase(testcase_name)
+ # self.log.set_testcase(testcase_name)
testcase_name = self.get_test_name(item.nodeid)
self.log.info('Teardown module for testcase {}'.format(testcase_name))
@@ -863,7 +864,7 @@ def check_analyzer_status(self, params, headers):
try:
url = "http://{0}:5001/end_test_case/".format(CafyLog.debug_server)
self.log.info("Calling registration service (url:%s) to check analyzer status" % url)
- response = requests.get(url, data=params)
+ response = requests.get(url, data=params, timeout=60)
if response.status_code == 200:
return response.json()['analyzer_status']
else:
@@ -894,7 +895,7 @@ def pytest_runtest_logreport(self, report):
try:
url = 'http://{0}:5001/registertest/'.format(CafyLog.debug_server)
self.log.info("Calling registration service to start handshake(url:%s" % url)
- response = requests.post(url, json=params, headers=headers)
+ response = requests.post(url, json=params, headers=headers, timeout=300)
if response.status_code == 200:
self.log.info("Handshake part of registration service was successful")
else:
@@ -905,14 +906,6 @@ def pytest_runtest_logreport(self, report):
if report.when == 'teardown':
- if self.reg_dict:
- reg_id = self.reg_dict.get('reg_id')
- test_class = report.nodeid.split('::')[1]
- if (test_class not in self.analyzer_testcase.keys()) or self.analyzer_testcase.get(test_class) == 1:
- analyzer_status = self.post_testcase_status(reg_id, testcase_name, CafyLog.debug_server)
- self.log.info('Analyzer Status is {}'.format(analyzer_status))
- else:
- self.log.info('Analyzer is not invoked as testcase failed in setup')
status = "unknown"
if testcase_name in self.testcase_dict:
status = self.testcase_dict[testcase_name]
@@ -1034,6 +1027,47 @@ def pytest_runtest_logreport(self, report):
else:
self.testcase_failtrace_dict[testcase_name] = None
+ # Add the testcase status to testcase_dict as error if the test failed in setup
+ try:
+ if report.when == 'setup' and report.outcome == 'failed':
+ testcase_name = self.get_test_name(report.nodeid)
+ self.testcase_dict[testcase_name] = 'error'
+ except Exception as e:
+ self.log.error("Error getting the testcase status for setup failure: {}".format(e))
+
+
+ @pytest.hookimpl(hookwrapper=True, trylast=True)
+ def pytest_runtest_makereport(self, item, call):
+ outcome = (yield)
+ if call.when =='call':
+ report = outcome.get_result()
+ testcase_name = self.get_test_name(report.nodeid)
+ if self.reg_dict:
+ reg_id = self.reg_dict.get('reg_id')
+ test_class = report.nodeid.split('::')[1]
+ if (test_class not in self.analyzer_testcase.keys()) or self.analyzer_testcase.get(test_class) == 1:
+ analyzer_status = self.post_testcase_status(reg_id, testcase_name, CafyLog.debug_server)
+ self.log.info('Analyzer Status is {}'.format(analyzer_status))
+ else:
+ self.log.info('Analyzer is not invoked as testcase failed in setup')
+ if isinstance(analyzer_status, bool):
+ return
+ failures = json.loads(analyzer_status.get('failures',[]))
+ if len(failures):
+ self.log.error('Test case failed due to crash/traceback {}'.format(pformat(failures)))
+ test_outcome = 'failed'
+ report = TestReport(
+ report.nodeid,
+ report.location,
+ report.keywords,
+ test_outcome,
+ report.longrepr,
+ report.when,
+ report.sections,
+ report.duration,
+ )
+ outcome.force_result(report)
+
def check_call_report(self, item, nextitem):
"""
@@ -1284,7 +1318,7 @@ def invoke_reg_on_failed_testcase(self, params, headers):
try:
url = "http://{0}:5001/startdebug/".format(CafyLog.debug_server)
self.log.info("Calling registration service (url:%s) to start collecting" % url)
- response = requests.post(url, json=params, headers=headers)
+ response = requests.post(url, json=params, headers=headers, timeout=1500)
if response.status_code == 200:
return response
else:
@@ -1301,7 +1335,7 @@ def invoke_rc_on_failed_testcase(self, params, headers):
try:
url = "http://{0}:5003/startrootcause/".format(CafyLog.debug_server)
self.log.info("Calling RC engine to start rootcause (url:%s)" % url)
- response = requests.post(url, json=params, headers=headers)
+ response = requests.post(url, json=params, headers=headers, timeout=300)
if response.status_code == 200:
return response
else:
@@ -1330,13 +1364,17 @@ def pytest_terminal_summary(self, terminalreporter):
if junitxml_file_path != _junitxml_filename:
copyfile(_junitxml_filename, junitxml_file_path)
os.chmod(junitxml_file_path, 0o775)
+
+ temp_list = []
+ terminalreporter.write_line("\n TestCase Summary Status Table")
+ for k,v in self.testcase_dict.items():
+ temp_list.append((k,v))
+ print (tabulate(temp_list, headers=['Testcase_name', 'Status'], tablefmt='grid'))
if not self.no_email:
- self._sendemail()
- terminalreporter.write_line("\n TestCase Summary Status Table")
- temp_list = []
- for k,v in self.testcase_dict.items():
- temp_list.append((k,v))
- print (tabulate(temp_list, headers=['Testcase_name', 'Status'], tablefmt='grid'))
+ try:
+ self._sendemail()
+ except Exception as err:
+ self.log.error("Error when sending email: {err}".format(err=str(err)))
#Unset environ variables cafykit_mongo_learn & cafykit_mongo_read if set
@@ -1403,7 +1441,7 @@ def _get_analyzer_log(self):
"debug_server_name": CafyLog.debug_server}
url = 'http://{0}:5001/get_analyzer_log/'.format(CafyLog.debug_server)
try:
- response = requests.get(url, data=params)
+ response = requests.get(url, data=params, timeout=300)
if response is not None and response.status_code == 200:
if response.text:
if 'Content-Disposition' in response.headers:
@@ -1435,15 +1473,22 @@ def pytest_sessionfinish(self):
url = 'http://{0}:5001/uploadcollectorlogfile/'.format(CafyLog.debug_server)
print("url = ", url)
self.log.info("Calling registration upload collector logfile service (url:%s)" %url)
- response = requests.post(url, json=params, headers=headers)
+ response = requests.post(url, json=params, headers=headers, timeout=300)
if response is not None and response.status_code == 200:
if response.text:
- self.log.info ("Debug Collector logs: %s" %(response.text))
+ summary_log = response.text
+ if '+'*120 in response.text:
+ summary_log, verbose_log = response.text.split('+'*120)
+ self.log.info ("Debug Collector logs: %s" %(summary_log))
if 'Content-Disposition' in response.headers:
debug_collector_log_filename = response.headers['Content-Disposition'].split('filename=')[-1]
collector_log_file_full_path = os.path.join(CafyLog.work_dir,debug_collector_log_filename)
with open(collector_log_file_full_path, 'w') as f:
- f.write(response.text)
+ f.write(summary_log)
+ verbose_log_file_path = collector_log_file_full_path.replace("debug_collection.log",
+ "verbose_collection.log")
+ with open(verbose_log_file_path, 'w') as f:
+ f.write(verbose_log)
try:
DebugLibrary.convert_collector_logs_to_json(collector_log_file_full_path)
except:
@@ -1453,7 +1498,7 @@ def pytest_sessionfinish(self):
url = 'http://{0}:5001/deleteuploadedfiles/'.format(CafyLog.debug_server)
self.log.info("Calling registration delete upload file service (url:%s)" % url)
- response = requests.post(url, json=params, headers=headers)
+ response = requests.post(url, json=params, headers=headers, timeout=300)
if response.status_code == 200:
self.log.info("Topology and input files deleted from registration server")
else:
@@ -1537,7 +1582,11 @@ def __init__(self, terminalreporter, testcase_dict, testcase_failtrace_dict, arc
# Run Info
self.exec_host = platform.node()
self.python_version = platform.python_version()
- self. platform = platform.platform()
+ self.platform = platform.platform()
+ try:
+ self.cafykit_release = os.path.basename(os.environ.get("VIRTUAL_ENV"))
+ except:
+ self.cafykit_release = None
self.testbed = None
self.registration_id = CafyLog.registration_id
self.submitter = EmailReport.USER
@@ -1545,7 +1594,7 @@ def __init__(self, terminalreporter, testcase_dict, testcase_failtrace_dict, arc
self.topo_file = topo_file
self.run_dir = self.terminalreporter.startdir.strpath
try:
- self.git_commit_id = subprocess.check_output(['git', 'rev-parse', 'origin/master']).decode("utf-8").replace('\n', '')
+ self.git_commit_id = subprocess.check_output(['git', 'rev-parse', 'origin/master'], timeout=5).decode("utf-8").replace('\n', '')
except Exception:
self.git_commit_id = None
self.archive = CafyLog.work_dir
diff --git a/pytest_cafy/resources/mail_template.html b/pytest_cafy/resources/mail_template.html
index 65ba9f0..7d2bb62 100644
--- a/pytest_cafy/resources/mail_template.html
+++ b/pytest_cafy/resources/mail_template.html
@@ -1,38 +1,5 @@
-{% macro detailed_header(report) -%}
-
-Detailed Results
-
- 🕐 {{report.run_time}}
-
-
- View allure report >
-
-
-
- View Summary report >
-
-
-
-{%- endmacro %}
-
-
-{% macro detailed_data(report) -%}
-{% for name, status in report.testcase_dict.items() %}
-
- | {{name}} |
- {{status}} |
- {{report.testcase_failtrace_dict[name]}} |
-
-{% endfor %}
-{%- endmacro %}
-
-
{% macro summary(report) -%}
@@ -45,6 +12,15 @@ Detailed Results
{%- endmacro %}
+{% macro detailed_data(report) -%}
+{% for name, status in report.testcase_dict.items() %}
+
+ | {{name}} |
+ {{status}} |
+ {{report.testcase_failtrace_dict[name]}} |
+
+{% endfor %}
+{%- endmacro %}
@@ -57,6 +33,24 @@ Detailed Results
{{report.script_list}}
+
+ Summary
+
+
+
+ | Passed |
+ Xpassed |
+ Failed |
+ Skipped |
+ Xfailed |
+ Total |
+
+
+
+ {{ summary(report) }}
+
+
+
Run Info
@@ -67,10 +61,12 @@ Run Info
TestBed |
{{report.testbed}} |
+ {% if report.registration_id %}
| Registration ID |
{{report.registration_id}} |
+ {% endif %}
| Submitter |
{{report.submitter}} |
@@ -79,12 +75,18 @@ Run Info
Run directory |
{{report.run_dir}} |
-
+ {% if report.git_commit_id %}
| Git commit id |
{{report.git_commit_id}} |
-
+ {% endif %}
+ {% if report.cafykit_release %}
+
+ | Cafykit Release |
+ {{report.cafykit_release}} |
+
+ {% endif %}
| Script list |
{{report.script_list}} |
@@ -117,19 +119,10 @@ Run Info
Cafy repo |
{{report.cafy_repo}} |
-
| Exec host |
{{report.exec_host}} |
-
- | Python version |
- {{report.python_version}} |
-
-
- | Exec platform |
- {{report.platform}} |
-
| Topology file |
{{report.topo_file}} |
@@ -145,23 +138,6 @@ Run Info
-
- Summary
-
-
-
- | Passed |
- Xpassed |
- Failed |
- Skipped |
- Xfailed |
- Total |
-
-
-
- {{ summary(report) }}
-
-
Testcase Status Summary
@@ -178,43 +154,6 @@ Testcase Status Summary
-
-
- Build Info
-
- Image
- {{report.image}}
-
-
-
-
- | XR EFR |
- {{report.xr_efr}} |
-
-
- | XR workspace |
- {{report.xr_ws}} |
-
-
- | Calvados EFR |
- {{report.cal_efr}} |
-
-
- | Calvados workspace |
- {{report.cal_ws}} |
-
- {% if report.jenkins_url %}
-
- | Jenkins URL |
- {{report.jenkins_url}} |
-
- {% endif %}
-
-
-
-
-
- {{ detailed_header(report) }}