Merge pull request #52 from mattip/multiple-exes

mattip · web-flow · commit c79de33cead1 · 2024-08-19T12:11:35.000+03:00
Allow multiple exes, baselines
diff --git a/codespeed/commits/git.py b/codespeed/commits/git.py
@@ -64,8 +64,8 @@ def getlogs(endrev, startrev):
     p, stdout, stderr = execute_command(cmd, working_copy)
 
     if p.returncode != 0:
-        raise CommitLogError("%s returned %s: %s" % (
-                             " ".join(cmd), p.returncode, stderr))
+        raise CommitLogError(f"'%s' in '%s' returned %s: %s" % (
+                             " ".join(cmd), working_copy, p.returncode, stderr))
     logs = []
     for log in filter(None, stdout.split('\x1e')):
         (short_commit_id, commit_id, date_t, author_name, author_email,
diff --git a/codespeed/results.py b/codespeed/results.py
@@ -73,7 +73,7 @@ def save_result(data, update_repo=True):
         b.save()
 
     try:
-        rev = branch.revisions.get(commitid=data['commitid'])
+        rev = branch.revisions.get(commitid=data['commitid'].split(":")[-1])
     except Revision.DoesNotExist:
         rev_date = data.get("revision_date")
         # "None" (as string) can happen when we urlencode the POST data
diff --git a/codespeed/views.py b/codespeed/views.py
@@ -69,17 +69,17 @@ def get_context_data(self, **kwargs):
         context = super(HomeView, self).get_context_data(**kwargs)
         context['show_reports'] = settings.SHOW_REPORTS
         context['show_historical'] = settings.SHOW_HISTORICAL
-        historical_settings = ['SHOW_HISTORICAL', 'DEF_BASELINE', 'DEF_EXECUTABLE']
+        historical_settings = ['SHOW_HISTORICAL', 'DEF_BASELINES', 'DEF_EXECUTABLES']
         if not all(getattr(settings, var) for var in historical_settings):
             context['show_historical'] = False
             return context
 
         try:
             baseline_exe = Executable.objects.get(
-                name=settings.DEF_BASELINE['executable'])
+                name=settings.DEF_BASELINES[0]['executable'])
             context['baseline'] = baseline_exe
-            def_name = settings.DEF_EXECUTABLE['name']
-            def_project = Project.objects.get(name=settings.DEF_EXECUTABLE['project'])
+            def_name = settings.DEF_EXECUTABLES[0]['name']
+            def_project = Project.objects.get(name=settings.DEF_EXECUTABLES[0]['project'])
             default_exe = Executable.objects.get(name=def_name,
                                                  project=def_project,
                                                 )
@@ -101,44 +101,55 @@ def gethistoricaldata(request):
         env = env.first()
 
     # Fetch Baseline data, filter by executable
-    baseline_exe = Executable.objects.get(
-        name=settings.DEF_BASELINE['executable'])
-    baseline_revs = Revision.objects.filter(
-        branch__project=baseline_exe.project).order_by('-date')
-    baseline_lastrev = baseline_revs[0]
-    for rev in baseline_revs:
-        baseline_results = Result.objects.filter(
-            executable=baseline_exe, revision=rev, environment=env)
-        if baseline_results:
-            baseline_lastrev = rev
-            break
-    if len(baseline_results) == 0:
-        logger.error('Could not find results for {} rev="{}" env="{}"'.format(
-                baseline_exe, baseline_lastrev, env))
-    data['baseline'] = '{} {}'.format(
-        settings.DEF_BASELINE['executable'], baseline_lastrev.tag)
-
-    def_name = settings.DEF_EXECUTABLE['name']
-    def_project = Project.objects.get(name=settings.DEF_EXECUTABLE['project'])
-    default_exe = Executable.objects.get(name=def_name, project=def_project)
-    default_branch = Branch.objects.get(
-        name=default_exe.project.default_branch,
-        project=default_exe.project)
+    baseline_results = []
+    for b in settings.DEF_BASELINES:
+        baseline_exe = Executable.objects.get(
+            name=b['executable'])
+        tag=b['revision']
+        rev = Revision.objects.filter(branch__project=baseline_exe.project, tag=tag)
+        if len(rev) < 1:
+            return HttpResponse(json.dumps(
+                f"Could not find {tag=} for {b['executable']} in database")
+            )
+        rev0 = rev[0]
+        resname = '{} {}'.format(b['executable'], rev0.tag)
+        baseline_results.append((resname, Result.objects.filter(
+            executable=baseline_exe, revision=rev0, environment=env)))
+        if not baseline_results[-1][1]:
+            logger.error('Could not find results for {} rev="{}" env="{}"'.format(
+                    baseline_exe, rev0, env))
 
-    # Fetch tagged revisions for executable
-    default_taggedrevs = Revision.objects.filter(
-            branch=default_branch
-        ).exclude(tag="").order_by('date')
     default_results = {}
-    for rev in default_taggedrevs:
-        res = Result.objects.filter(
-            executable=default_exe, revision=rev, environment=env)
-        if not res:
-            logger.info('no results for %s %s %s' % (str(default_exe), str(rev), str(env)))
-            continue
-        default_results[rev.tag] = res
-    data['tagged_revs'] = [rev.tag for rev in default_taggedrevs if rev.tag in default_results]
+    all_taggedrevs = []
+    for executable in settings.DEF_EXECUTABLES:
+        _def_name = executable['name']
+        _def_project = Project.objects.get(name=executable['project'])
+        _default_exe = Executable.objects.get(name=_def_name, project=_def_project)
+        _default_branch = Branch.objects.get(
+            name=_default_exe.project.default_branch,
+            project=_default_exe.project)
+
+        # Fetch tagged revisions for executable
+        default_taggedrevs = Revision.objects.filter(
+                branch=_default_branch
+            ).exclude(tag="").order_by('date')
+        all_taggedrevs += default_taggedrevs
+        for rev in default_taggedrevs:
+            res = Result.objects.filter(
+                executable=_default_exe, revision=rev, environment=env)
+            if not res:
+                logger.info("no results for '%s' '%s' '%s'" % (str(_default_exe), str(rev), str(env)))
+                continue
+            default_results[rev.tag] = res
+    data['tagged_revs'] = [rev.tag for rev in all_taggedrevs if rev.tag in default_results]
     # Fetch data for latest results
+    executable = settings.DEF_EXECUTABLES[0]
+    def_name = executable['name']
+    def_project = Project.objects.get(name=executable['project'])
+    default_exe = Executable.objects.get(name=def_name, project=def_project)
+    default_branch = Branch.objects.get(
+            name=default_exe.project.default_branch,
+            project=default_exe.project)
     revs = Revision.objects.filter(
         branch=default_branch).order_by('-date')[:100]
     default_lastrev = None
@@ -153,17 +164,30 @@ def gethistoricaldata(request):
 
     # Collect data
     benchmarks = []
-    for res in baseline_results:
+    # Collate first baseline and all the default_results
+    resset = baseline_results[0][1]
+    resname = baseline_results[0][0]
+    data['baseline'] = resname
+    for res in resset:
         if res == 0:
             continue
         benchmarks.append(res.benchmark.name)
-        data['results'][res.benchmark.name] = {data['baseline']: res.value}
+        data['results'][res.benchmark.name] = {resname: res.value}
         for rev_name in default_results:
             val = 0
             for default_res in default_results[rev_name]:
                 if default_res.benchmark.name == res.benchmark.name:
                     val = default_res.value
-            data['results'][res.benchmark.name][rev_name] = val
+                data['results'][res.benchmark.name][rev_name] = val
+    # Collate other baseline
+    for resname, resset in baseline_results[1:]:
+        for res in resset:
+            if res == 0:
+                continue
+            if not res.benchmark.name in data['results']:
+                continue
+            data['results'][res.benchmark.name][resname] = res.value
+        data['tagged_revs'].insert(0, resname)
     benchmarks.sort()
     data['benchmarks'] = benchmarks
     return HttpResponse(json.dumps(data))
@@ -896,7 +920,7 @@ def displaylogs(request):
         log['commit_browse_url'] = project.commit_browsing_url.format(**log)
 
     return render(
-        request, 
+        request,
         'codespeed/changes_logs.html',
         {
             'error': error, 'logs': logs,
diff --git a/codespeed/views_data.py b/codespeed/views_data.py
@@ -83,10 +83,10 @@ def getbaselineexecutables(include_tags=None):
                 'name': name,
             })
     # move default to first place
-    if hasattr(settings, 'DEF_BASELINE') and settings.DEF_BASELINE is not None:
+    if hasattr(settings, 'DEF_BASELINES') and settings.DEF_BASELINES is not None:
         try:
-            exename = settings.DEF_BASELINE['executable']
-            commitid = settings.DEF_BASELINE['revision']
+            exename = settings.DEF_BASELINES[0]['executable']
+            commitid = settings.DEF_BASELINES[0]['revision']
             for base in baseline:
                 if base['key'] == "none":
                     continue
@@ -104,11 +104,11 @@ def getbaselineexecutables(include_tags=None):
 
 def getdefaultexecutable():
     default = None
-    if (hasattr(settings, 'DEF_EXECUTABLE') and
-            settings.DEF_EXECUTABLE is not None):
+    if (hasattr(settings, 'DEF_EXECUTABLES') and
+            settings.DEF_EXECUTABLES is not None):
         try:
-            def_name = settings.DEF_EXECUTABLE['name']
-            def_project = Project.objects.get(name=settings.DEF_EXECUTABLE['project'])
+            def_name = settings.DEF_EXECUTABLES[0]['name']
+            def_project = Project.objects.get(name=settings.DEF_EXECUTABLES[0]['project'])
             default = Executable.objects.get(name=def_name, project=def_project)
         except Executable.DoesNotExist:
             pass
diff --git a/speed_pypy/settings.py b/speed_pypy/settings.py
@@ -89,8 +89,14 @@
 
 SHOW_REPORTS = False
 SHOW_HISTORICAL = True
-DEF_BASELINE = {'executable': 'cpython', 'revision': '3.6.8'}
-DEF_EXECUTABLE = {'name': 'pypy3.9-jit-64', 'project': 'PyPy3.9'}
+DEF_BASELINES = [
+                 {'executable': 'cpython', 'revision': '3.7.19'},
+                 {'executable': 'cpython', 'revision': '3.11.9'},
+                ]
+DEF_EXECUTABLES = [
+                   {'name': 'pypy3.10-jit-64', 'project': 'PyPy3.10'},
+                   {'name': 'pypy3.9-jit-64', 'project': 'PyPy3.9'},
+                  ]
 DEF_ENVIRONMENT = 'benchmarker'
 
 
diff --git a/speed_pypy/templates/home.html b/speed_pypy/templates/home.html
@@ -33,7 +33,7 @@ <h3>How fast is {{ default_exe.project }}?</h3>
     <p class="plot-caption">Plot 1: The above plot represents {{ default_exe.project }} ({{ default_exe }}) benchmark times normalized to {{ baseline }}. Smaller is better.</p>
     <p>It depends greatly on the type of task being performed. The geometric average of all benchmarks is <span id="geomean"></span> or <strong id="geofaster"></strong> times <em>faster</em> than {{ baseline }}</p>
 
-    <h3>How has {{ default_exe.project }} performance evolved over time?</h3>
+    <h3>How has PyPy performance evolved over time?</h3>
     <div id="historical-plot"></div>
     <p class="plot-caption">Plot 2: Speedup compared to {{ baseline }}, using the inverse of the geometric average of normalized times, out of <span id="num_of_benchs"></span> benchmarks (see <a href="http://dl.acm.org/citation.cfm?id=5673" title="How not to lie with statistics: the correct way to summarize benchmark results">paper</a> on why the geometric mean is better for normalized results).</p>
   </div>
@@ -81,6 +81,10 @@ <h3>How has {{ default_exe.project }} performance evolved over time?</h3>
             $("#baseline-comparison-plot").html(getLoadText('Error retrieving data', 0));
             return 1;
         }
+        if (typeof data === 'string' || data instanceof String) {
+            $("#baseline-comparison-plot").html(getLoadText('Error retrieving data ' + data, 0));
+            return 1;
+        }
         var trunk_geomean = 1;
         var tagged_data = new Array();
         for (i in data['tagged_revs']) {
@@ -108,7 +112,7 @@ <h3>How has {{ default_exe.project }} performance evolved over time?</h3>
             plotdata[0].push(relative_value);
             plotdata[1].push(1.0);
             labels.push(relative_value.toFixed(2));
-            if (relative_value > 0) {
+            if (relative_value > 0 && !isNaN(relative_value)) {
                 trunk_geomean *= relative_value;
             }
         }
@@ -158,7 +162,10 @@ <h3>How has {{ default_exe.project }} performance evolved over time?</h3>
             num_of_benchs = tagged_data[i].length;
             var tempgeo = 1;
             for (var j in tagged_data[i]) {
-                tempgeo *= tagged_data[i][j];
+				value = tagged_data[i][j];
+				if (value > 0 && !isNaN(value)) {
+					tempgeo *= value;
+				}
             }
             tempgeo = Math.pow(tempgeo, 1/tagged_data[i].length);
             tempgeo = 1/tempgeo;
@@ -169,7 +176,7 @@ <h3>How has {{ default_exe.project }} performance evolved over time?</h3>
         for (var i in data['tagged_revs']) {
             ticks.push(data['tagged_revs'][i]);
         }
-        ticks.push('latest');
+        ticks.push('latest {{ default_exe.project }}');
         var geolabels = new Array();
         for (var i in geomeans) {
             geolabels.push(geomeans[i].toFixed(2) + "x");
@@ -197,7 +204,7 @@ <h3>How has {{ default_exe.project }} performance evolved over time?</h3>
                 },
                 yaxis:{
                     min: 0,
-                    tickOptions:{formatString:'%.2f'}
+                    tickOptions:{formatString:'%.1f'}
                 }
             }
         };
