Create cafyrun alias to run pytest scripts JIRA issue CAFY-260

bin/cafyrun

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+import pytest
+import _pytest.main
+import sys
+
+# Customize messages for pytest exit codes
+msg = {_pytest.main.EXIT_OK: 'OK',
+       _pytest.main.EXIT_TESTSFAILED: 'Tests failed',
+       _pytest.main.EXIT_INTERRUPTED: 'Interrupted',
+       _pytest.main.EXIT_INTERNALERROR: 'Internal error',
+       _pytest.main.EXIT_USAGEERROR: 'Usage error',
+       _pytest.main.EXIT_NOTESTSCOLLECTED: 'No tests collected'}
+
+exitcode = pytest.main(sys.argv[1:])
+
+
+
+
+print(msg[exitcode])
+sys.exit(exitcode)
+

pytest_cafy/plugin.py

@@ -21,11 +21,12 @@
 import pytest
 
 from _pytest.terminal import TerminalReporter
-from _pytest.runner import runtestprotocol
+from _pytest.runner import runtestprotocol, TestReport
 from _pytest.mark import MarkInfo
 
 from enum import Enum
 from tabulate import tabulate
+from pprint import pprint, pformat
 from shutil import copyfile
 from configparser import ConfigParser
 from datetime import datetime
@@ -122,7 +123,7 @@ def pytest_addoption(parser):
     group.addoption('--work-dir', dest="workdir", metavar="DIR", default=None,
                     help="Path for work dir")
 
-    group.addoption('--report-dir', dest="reportdir",
+    group.addoption('-R','--report-dir', dest="reportdir",
                     metavar="DIR",
                     default=None,
                     help="Path for report dir")
@@ -133,7 +134,7 @@ def pytest_addoption(parser):
                     type=lambda x: is_valid_param(x, file_type='topology_file'),
                     help='Filename of your testbed')
 
-    group.addoption('--test-input-file', action='store', dest='test_input_file',
+    group.addoption('-I', '--test-input-file', action='store', dest='test_input_file',
                     metavar='test_input_file',
                     type=lambda x: is_valid_param(x, file_type='input_file'),
                     help='Filename of your test input file')
@@ -534,7 +535,7 @@ def pytest_collection_modifyitems(session, config, items):
                 log.info("url: {}".format(url))
                 log.info("Calling API service for live logging of reg_id ")
                 params = {"reg_id": CafyLog.registration_id}
-                response = requests.patch(url, json=params, headers=headers)
+                response = requests.patch(url, json=params, headers=headers, timeout=120)
                 if response.status_code == 200:
                     log.info("Calling API service for live logging of reg_id successful")
                 else:
@@ -558,7 +559,7 @@ def pytest_collection_modifyitems(session, config, items):
             url = '{0}/api/runs/{1}/cases'.format(os.environ.get('CAFY_API_HOST'), os.environ.get('CAFY_RUN_ID'))
             log.info("url: {}".format(url))
             log.info("Calling API service for live logging of collected testcases ")
-            response = requests.post(url, json=CafyLog.collected_testcases, headers=headers)
+            response = requests.post(url, json=CafyLog.collected_testcases, headers=headers, timeout=120)
             if response.status_code == 200:
                 log.info("Calling API service for live logging of collected testcases successful")
             else:
@@ -789,7 +790,7 @@ def initiate_analyzer(self, reg_id, test_case, debug_server):
             try:
                 url = "http://{0}:5001/initiate_analyzer/".format(CafyLog.debug_server)
                 self.log.info("Calling registration service (url:%s) to initialize analyzer" % url)
-                response = requests.post(url, data=params)
+                response = requests.post(url, data=params, timeout=300)
                 if response.status_code == 200:
                     self.log.info("Analyzer initialized")
                     return True
@@ -813,7 +814,7 @@ def pytest_runtest_teardown(self, item, nextitem):
             self.log.set_testcase("Teardown")
         else:
             testcase_name = self.get_test_name(nextitem.nodeid)
-            self.log.set_testcase(testcase_name)
+            # self.log.set_testcase(testcase_name)
         testcase_name = self.get_test_name(item.nodeid)
         self.log.info('Teardown module for testcase {}'.format(testcase_name))
 
@@ -863,7 +864,7 @@ def check_analyzer_status(self, params, headers):
             try:
                 url = "http://{0}:5001/end_test_case/".format(CafyLog.debug_server)
                 self.log.info("Calling registration service (url:%s) to check analyzer status" % url)
-                response = requests.get(url, data=params)
+                response = requests.get(url, data=params, timeout=60)
                 if response.status_code == 200:
                     return response.json()['analyzer_status']
                 else:
@@ -894,7 +895,7 @@ def pytest_runtest_logreport(self, report):
                     try:
                         url = 'http://{0}:5001/registertest/'.format(CafyLog.debug_server)
                         self.log.info("Calling registration service to start handshake(url:%s" % url)
-                        response = requests.post(url, json=params, headers=headers)
+                        response = requests.post(url, json=params, headers=headers, timeout=300)
                         if response.status_code == 200:
                             self.log.info("Handshake part of registration service was successful")
                         else:
@@ -905,14 +906,6 @@ def pytest_runtest_logreport(self, report):
 
 
         if report.when == 'teardown':
-            if self.reg_dict:
-                reg_id = self.reg_dict.get('reg_id')
-                test_class = report.nodeid.split('::')[1]
-                if (test_class not in self.analyzer_testcase.keys()) or self.analyzer_testcase.get(test_class) == 1:
-                    analyzer_status = self.post_testcase_status(reg_id, testcase_name, CafyLog.debug_server)
-                    self.log.info('Analyzer Status is {}'.format(analyzer_status))
-                else:
-                    self.log.info('Analyzer is not invoked as testcase failed in setup')
             status = "unknown"
             if testcase_name in self.testcase_dict:
                 status = self.testcase_dict[testcase_name]
@@ -1034,6 +1027,47 @@ def pytest_runtest_logreport(self, report):
                     else:
                         self.testcase_failtrace_dict[testcase_name] = None
 
+        # Add the testcase status to testcase_dict as error if the test failed in setup
+        try:
+            if report.when == 'setup' and report.outcome == 'failed':
+                testcase_name = self.get_test_name(report.nodeid)
+                self.testcase_dict[testcase_name] = 'error'
+        except Exception as e:
+            self.log.error("Error getting the testcase status for setup failure: {}".format(e))
+
+
+    @pytest.hookimpl(hookwrapper=True, trylast=True)
+    def pytest_runtest_makereport(self, item, call):
+        outcome = (yield)
+        if call.when =='call':
+            report = outcome.get_result()
+            testcase_name = self.get_test_name(report.nodeid)
+            if self.reg_dict:
+                reg_id = self.reg_dict.get('reg_id')
+                test_class = report.nodeid.split('::')[1]
+                if (test_class not in self.analyzer_testcase.keys()) or self.analyzer_testcase.get(test_class) == 1:
+                    analyzer_status = self.post_testcase_status(reg_id, testcase_name, CafyLog.debug_server)
+                    self.log.info('Analyzer Status is {}'.format(analyzer_status))
+                else:
+                    self.log.info('Analyzer is not invoked as testcase failed in setup')
+                if isinstance(analyzer_status, bool):
+                    return
+                failures = json.loads(analyzer_status.get('failures',[]))
+                if len(failures):
+                    self.log.error('Test case failed due to crash/traceback {}'.format(pformat(failures)))
+                    test_outcome = 'failed'
+                    report = TestReport(
+                        report.nodeid,
+                        report.location,
+                        report.keywords,
+                        test_outcome,
+                        report.longrepr,
+                        report.when,
+                        report.sections,
+                        report.duration,
+                    )
+                    outcome.force_result(report)
+
 
     def check_call_report(self, item, nextitem):
         """
@@ -1284,7 +1318,7 @@ def invoke_reg_on_failed_testcase(self, params, headers):
             try:
                 url = "http://{0}:5001/startdebug/".format(CafyLog.debug_server)
                 self.log.info("Calling registration service (url:%s) to start collecting" % url)
-                response = requests.post(url, json=params, headers=headers)
+                response = requests.post(url, json=params, headers=headers, timeout=1500)
                 if response.status_code == 200:
                     return response
                 else:
@@ -1301,7 +1335,7 @@ def invoke_rc_on_failed_testcase(self, params, headers):
             try:
                 url = "http://{0}:5003/startrootcause/".format(CafyLog.debug_server)
                 self.log.info("Calling RC engine to start rootcause (url:%s)" % url)
-                response = requests.post(url, json=params, headers=headers)
+                response = requests.post(url, json=params, headers=headers, timeout=300)
                 if response.status_code == 200:
                     return response
                 else:
@@ -1330,13 +1364,17 @@ def pytest_terminal_summary(self, terminalreporter):
                 if junitxml_file_path != _junitxml_filename:
                     copyfile(_junitxml_filename, junitxml_file_path)
                     os.chmod(junitxml_file_path, 0o775)
+
+        temp_list = []
+        terminalreporter.write_line("\n TestCase Summary Status Table")
+        for k,v in self.testcase_dict.items():
+            temp_list.append((k,v))
+        print (tabulate(temp_list, headers=['Testcase_name', 'Status'], tablefmt='grid'))
         if not self.no_email:
-            self._sendemail()
-            terminalreporter.write_line("\n TestCase Summary Status Table")
-            temp_list = []
-            for k,v in self.testcase_dict.items():
-                temp_list.append((k,v))
-            print (tabulate(temp_list, headers=['Testcase_name', 'Status'], tablefmt='grid'))
+            try: 
+                self._sendemail()
+            except Exception as err: 
+                self.log.error("Error when sending email: {err}".format(err=str(err)))
 
 
         #Unset environ variables cafykit_mongo_learn & cafykit_mongo_read if set
@@ -1403,7 +1441,7 @@ def _get_analyzer_log(self):
                   "debug_server_name": CafyLog.debug_server}
         url = 'http://{0}:5001/get_analyzer_log/'.format(CafyLog.debug_server)
         try:
-            response = requests.get(url, data=params)
+            response = requests.get(url, data=params, timeout=300)
             if response is not None and response.status_code == 200:
                 if response.text:
                     if 'Content-Disposition' in response.headers:
@@ -1435,15 +1473,22 @@ def pytest_sessionfinish(self):
                 url = 'http://{0}:5001/uploadcollectorlogfile/'.format(CafyLog.debug_server)
                 print("url = ", url)
                 self.log.info("Calling registration upload collector logfile service (url:%s)" %url)
-                response = requests.post(url, json=params, headers=headers)
+                response = requests.post(url, json=params, headers=headers, timeout=300)
                 if response is not None and response.status_code == 200:
                     if response.text:
-                        self.log.info ("Debug Collector logs: %s" %(response.text))
+                        summary_log = response.text
+                        if '+'*120 in response.text:
+                            summary_log, verbose_log = response.text.split('+'*120)
+                        self.log.info ("Debug Collector logs: %s" %(summary_log))
                         if 'Content-Disposition' in response.headers:
                             debug_collector_log_filename = response.headers['Content-Disposition'].split('filename=')[-1]
                             collector_log_file_full_path = os.path.join(CafyLog.work_dir,debug_collector_log_filename)
                             with open(collector_log_file_full_path, 'w') as f:
-                                f.write(response.text)
+                                f.write(summary_log)
+                            verbose_log_file_path = collector_log_file_full_path.replace("debug_collection.log",
+                                                                                         "verbose_collection.log")
+                            with open(verbose_log_file_path, 'w') as f:
+                                f.write(verbose_log)
                             try:
                                 DebugLibrary.convert_collector_logs_to_json(collector_log_file_full_path)
                             except:
@@ -1453,7 +1498,7 @@ def pytest_sessionfinish(self):
 
                 url = 'http://{0}:5001/deleteuploadedfiles/'.format(CafyLog.debug_server)
                 self.log.info("Calling registration delete upload file service (url:%s)" % url)
-                response = requests.post(url, json=params, headers=headers)
+                response = requests.post(url, json=params, headers=headers, timeout=300)
                 if response.status_code == 200:
                     self.log.info("Topology and input files deleted from registration server")
                 else:
@@ -1537,15 +1582,19 @@ def __init__(self, terminalreporter, testcase_dict, testcase_failtrace_dict, arc
         # Run Info
         self.exec_host = platform.node()
         self.python_version = platform.python_version()
-        self. platform = platform.platform()
+        self.platform = platform.platform()
+        try:
+            self.cafykit_release = os.path.basename(os.environ.get("VIRTUAL_ENV"))
+        except:
+            self.cafykit_release = None
         self.testbed = None
         self.registration_id = CafyLog.registration_id
         self.submitter = EmailReport.USER
         self.cafy_repo = EmailReport.CAFY_REPO
         self.topo_file = topo_file
         self.run_dir = self.terminalreporter.startdir.strpath
         try:
-            self.git_commit_id = subprocess.check_output(['git', 'rev-parse', 'origin/master']).decode("utf-8").replace('\n', '')
+            self.git_commit_id = subprocess.check_output(['git', 'rev-parse', 'origin/master'], timeout=5).decode("utf-8").replace('\n', '')
         except Exception:
             self.git_commit_id = None
         self.archive = CafyLog.work_dir