webapp: add API to get all header files of project (#1552)

* webapp: check for constructor file

Signed-off-by: David Korczynski <david@adalogics.com>

* webapp: add API to get all header files of project

Signed-off-by: David Korczynski <david@adalogics.com>

---------

Signed-off-by: David Korczynski <david@adalogics.com>

Author: DavidKorczynski

tools/web-fuzzing-introspection/app/static/assets/db/web_db_creator_from_summary.py

@@ -289,6 +289,27 @@ def extract_local_project_data(project_name, oss_fuzz_path,
     debug_report = oss_fuzz.extract_local_introspector_debug_info(
         project_name, oss_fuzz_path)
 
+    if debug_report:
+        all_files_in_project = debug_report.get('all_files_in_project', [])
+        all_header_files_in_project = set()
+        for elem in all_files_in_project:
+            source_file = elem.get('source_file', '')
+            if source_file.endswith('.h'):
+                normalized_file = os.path.normpath(source_file)
+                if '/usr/local/' in normalized_file or '/usr/include/' in normalized_file:
+                    continue
+                all_header_files_in_project.add(normalized_file)
+
+        all_header_files = {
+            'project': project_name.split('###')[0],
+            'all-header-files': list(all_header_files_in_project)
+        }
+    else:
+        all_header_files = {
+            'project': project_name.split('###')[0],
+            'all-header-files': list()
+        }
+
     # Refine the data
     all_function_list = oss_fuzz.extract_local_introspector_function_list(
         project_name, oss_fuzz_path)
@@ -351,6 +372,7 @@ def extract_local_project_data(project_name, oss_fuzz_path,
         'project_timestamp': project_timestamp,
         "introspector-data-dict": introspector_data_dict,
         "coverage-data-dict": code_coverage_data_dict,
+        'all-header-files': all_header_files,
     }
 
 
@@ -449,6 +471,27 @@ def extract_project_data(project_name, date_str, should_include_details,
     else:
         debug_report = None
 
+    if debug_report:
+        all_files_in_project = debug_report.get('all_files_in_project', [])
+        all_header_files_in_project = set()
+        for elem in all_files_in_project:
+            source_file = elem.get('source_file', '')
+            if source_file.endswith('.h'):
+                normalized_file = os.path.normpath(source_file)
+                if '/usr/local/' in normalized_file or '/usr/include/' in normalized_file:
+                    continue
+                all_header_files_in_project.add(normalized_file)
+
+        all_header_files = {
+            'project': project_name.split('###')[0],
+            'all-header-files': list(all_header_files_in_project)
+        }
+    else:
+        all_header_files = {
+            'project': project_name.split('###')[0],
+            'all-header-files': list()
+        }
+
     # Currently, we fail if any of code_coverage_summary of introspector_report is
     # None. This should later be adjusted such that we can continue if we only
     # have code coverage but no introspector data. However, we need to adjust
@@ -551,6 +594,7 @@ def extract_project_data(project_name, date_str, should_include_details,
         'project_timestamp': project_timestamp,
         "introspector-data-dict": introspector_data_dict,
         "coverage-data-dict": code_coverage_data_dict,
+        'all-header-files': all_header_files,
     }
     return
 
@@ -571,6 +615,7 @@ def analyse_list_of_projects(date, projects_to_analyse,
     function_list = list()
     constructor_list = list()
     project_timestamps = list()
+    all_header_files = list()
     accummulated_fuzzer_count = 0
     accummulated_function_count = 0
     accummulated_covered_functions = 0
@@ -645,6 +690,9 @@ def analyse_list_of_projects(date, projects_to_analyse,
                 'function_coverage_estimate'] += introspector_dictionary[
                     'functions_covered_estimate']
 
+            all_header_files.append(
+                analyses_dictionary[project_key]['all-header-files'])
+
         coverage_dictionary = analyses_dictionary[project_key].get(
             'coverage-data-dict', None)
         if coverage_dictionary != None:
@@ -662,7 +710,7 @@ def analyse_list_of_projects(date, projects_to_analyse,
     # - all constructors (maybe empty)
     # - a list of project timestamps
     # - the DB timestamp
-    return function_list, constructor_list, project_timestamps, db_timestamp
+    return function_list, constructor_list, project_timestamps, db_timestamp, all_header_files
 
 
 def extend_db_timestamps(db_timestamp, output_directory):
@@ -752,9 +800,13 @@ def extend_func_db(function_list, output_directory, target):
         json.dump(all_functions, f)
 
 
-def update_db_files(db_timestamp, project_timestamps, function_list,
-                    constructor_list, output_directory,
-                    should_include_details):
+def update_db_files(db_timestamp,
+                    project_timestamps,
+                    function_list,
+                    constructor_list,
+                    output_directory,
+                    should_include_details,
+                    all_header_files=dict()):
     logger.info(
         "Updating the database with DB snapshot. Number of functions in total: %d"
         % (db_timestamp['function_count']))
@@ -763,6 +815,9 @@ def update_db_files(db_timestamp, project_timestamps, function_list,
         extend_func_db(constructor_list, output_directory,
                        DB_JSON_ALL_CONSTRUCTORS)
 
+    with open('all-header-files.json', 'w') as f:
+        f.write(json.dumps(all_header_files))
+
     extend_db_json_files(project_timestamps, output_directory)
     extend_db_timestamps(db_timestamp, output_directory)
 
@@ -840,14 +895,15 @@ def analyse_set_of_dates(dates, projects_to_analyse, output_directory,
         if force_creation:
             is_end = True
 
-        function_list, constructor_list, project_timestamps, db_timestamp = analyse_list_of_projects(
+        function_list, constructor_list, project_timestamps, db_timestamp, all_header_files = analyse_list_of_projects(
             date, projects_to_analyse, should_include_details=is_end)
         update_db_files(db_timestamp,
                         project_timestamps,
                         function_list,
                         constructor_list,
                         output_directory,
-                        should_include_details=is_end)
+                        should_include_details=is_end,
+                        all_header_files=all_header_files)
 
 
 def get_date_at_offset_as_str(day_offset=-1):
@@ -1074,13 +1130,18 @@ def create_local_db(oss_fuzz_path):
         extract_local_project_data(project, oss_fuzz_path, analyses_dictionary)
 
     # Accummulate the data from all the projects.
+    all_header_files = []
     for project_key in analyses_dictionary:
         # Append project timestamp to the list of timestamps
         project_timestamp = analyses_dictionary[project_key][
             'project_timestamp']
         project_timestamps.append(project_timestamp)
         db_timestamp['fuzzer_count'] += project_timestamp['fuzzer-count']
 
+        # Extend all header files
+        all_header_files.append(
+            analyses_dictionary[project_key]['all-header-files'])
+
         # Accummulate all function list and branch blockers
         introspector_dictionary = project_timestamp.get(
             'introspector-data', None)
@@ -1120,7 +1181,8 @@ def create_local_db(oss_fuzz_path):
                     function_list,
                     constructor_list,
                     os.getcwd(),
-                    should_include_details=True)
+                    should_include_details=True,
+                    all_header_files=all_header_files)
 
 
 def create_db(max_projects, days_to_analyse, output_directory, input_directory,

tools/web-fuzzing-introspection/app/webapp/__init__.py

@@ -32,8 +32,11 @@ def load_db():
     project_currents = os.path.join(
         os.path.dirname(__file__),
         "../static/assets/db/all-project-current.json")
+
     projects_build_status = os.path.join(
         os.path.dirname(__file__), "../static/assets/db/build-status.json")
+    all_header_files_file = os.path.join(
+        os.path.dirname(__file__), "../static/assets/db/all-header-files.json")
 
     if len(data_storage.DB_TIMESTAMPS) > 0:
         return
@@ -140,6 +143,11 @@ def load_db():
                     coverage_build_log=project_dict['cov-build-log'],
                     fuzz_build_log=project_dict['fuzz-build-log']))
 
+    if os.path.isfile(all_header_files_file):
+        with open(all_header_files_file, 'r') as f:
+            all_header_files = json.load(f)
+        data_storage.ALL_HEADER_FILES = all_header_files
+
     return
 
 

tools/web-fuzzing-introspection/app/webapp/data_storage.py

@@ -24,6 +24,8 @@
 
 PROJECT_DEBUG_DATA = []
 
+ALL_HEADER_FILES = []
+
 
 def get_projects():
     return PROJECTS

tools/web-fuzzing-introspection/app/webapp/routes.py

@@ -1351,6 +1351,25 @@ def shutdown():
         return {'result': 'failed', 'msg': 'not a local server'}
 
 
+@blueprint.route('/api/all-header-files')
+def all_project_header_files():
+    project = request.args.get('project', None)
+    if project == None:
+        return {
+            'result': 'error',
+            'extended_msgs': ['Please provide project name']
+        }
+
+    for elem in data_storage.ALL_HEADER_FILES:
+        if elem['project'] == project:
+            return {
+                'result': 'success',
+                'all-header-files': elem['all-header-files']
+            }
+
+    return {'result': 'failed', 'msg': 'did not find project'}
+
+
 @blueprint.route('/api/addr-to-recursive-dwarf-info')
 def type_at_addr():
     project = request.args.get('project', None)