Merge pull request #55 from mattip/cpython-baseline

compare to cpython3.11, fix some tools

Author: mattip

speed_pypy/settings.py

@@ -90,11 +90,11 @@
 SHOW_REPORTS = False
 SHOW_HISTORICAL = True
 DEF_BASELINES = [
-                 {'executable': 'cpython', 'revision': '3.7.19'},
                  {'executable': 'cpython', 'revision': '3.11.9'},
                  {'executable': 'cpython', 'revision': '3.12.4'},
                 ]
 DEF_EXECUTABLES = [
+                   {'name': 'pypy3.11-jit-64', 'project': 'PyPy3.11'},
                    {'name': 'pypy3.10-jit-64', 'project': 'PyPy3.10'},
                    {'name': 'pypy3.9-jit-64', 'project': 'PyPy3.9'},
                   ]

tools/pypy/import_from_json.py

@@ -2,64 +2,88 @@
 ################################################################################
 # This script imports PyPy's result data from json files located on the server #
 ################################################################################
-import simplejson, urllib2
+import simplejson
+from urllib.request import urlopen
+from urllib.error import URLError
 import sys
 from xml.dom.minidom import parse
 from datetime import datetime
-import saveresults, savecpython
+import saveresults
 
-RESULTS_URLS = {
-    'pypy-c-jit': 'http://buildbot.pypy.org/bench_results/',
-    'pypy-c':  'http://buildbot.pypy.org/bench_results_nojit/',
-}
-START_REV = 79485
-END_REV = 79485
-PROJECT = "PyPy"
+URL = 'http://buildbot.pypy.org/benchmark-results/'
+START_REV = 186137
+END_REV = 200000
 
-for INTERP in RESULTS_URLS:
-    RESULTS_URL = RESULTS_URLS[INTERP]
-    # get json filenames
-    filelist = []
-    try:
-        datasource = urllib2.urlopen(RESULTS_URL)
-        dom = parse(datasource)
-        for elem in dom.getElementsByTagName('td'):
-            for e in elem.childNodes:
-                if len(e.childNodes):
-                    filename = e.firstChild.toxml()
-                    if e.tagName == "a" and ".json" in filename:
-                        if int(filename.replace(".json", "")) >= START_REV and\
-                            int(filename.replace(".json", "")) <= END_REV:
-                            filelist.append(filename)
-    except urllib2.URLError, e:
-        response = "None"
-        if hasattr(e, 'reason'):
-            response = '\n  We failed to reach ' + RESULTS_URL + '\n'
-            response += '  Reason: ' + str(e.reason)
-        elif hasattr(e, 'code'):
-            response = '\n  The server couldn\'t fulfill the request\n'
-            response += '  Error code: ' + str(e)
-        print "Results Server (%s) response: %s\n" % (RESULTS_URL, response)
-        sys.exit(1)
-    finally:
-        datasource.close()
 
-    # read json result and save to speed.pypy.org
-    for filename in filelist:
-        print "Reading %s..." % filename
-        f = urllib2.urlopen(RESULTS_URL + filename)
-        result = simplejson.load(f)
-        f.close()
-        proj = PROJECT
-        revision = result['revision']
-        interpreter = INTERP
-        int_options = ""
-        options = ""
-        if 'options' in result:
-            options = result['options']
+speed_url = "http://127.0.0.1:8000/"
 
-        host = 'tannit'
-        #saveresults.save(proj, revision, result['results'], options, interpreter, host)
-        if filename == filelist[len(filelist)-1]:
-            savecpython.save('cpython', '100', result['results'], options, 'cpython', host)
-print "\nOK"
+# get json filenames
+filelist = []
+try:
+    datasource = urlopen(URL)
+    dom = parse(datasource)
+    for elem in dom.getElementsByTagName('td'):
+        for e in elem.childNodes:
+            if len(e.childNodes):
+                filename = e.firstChild.toxml()
+                if e.tagName == "a" and ".json" in filename:
+                    rev_int = int(filename.split(":")[0])
+                    if START_REV <= rev_int <= END_REV:
+                        filelist.append(filename)
+except URLError as e:
+    response = "None"
+    if hasattr(e, 'reason'):
+        response = '\n  We failed to reach ' + URL + '\n'
+        response += '  Reason: ' + str(e.reason)
+    elif hasattr(e, 'code'):
+        response = '\n  The server couldn\'t fulfill the request\n'
+        response += '  Error code: ' + str(e)
+    print("Results Server (%s) response: %s\n" % (URL, response))
+    sys.exit(1)
+finally:
+    datasource.close()
+
+# read json result and save to speed.pypy.org
+skipped = []
+for filename in filelist:
+    print(f"Reading {filename}...")
+    f = urlopen(URL + filename)
+    if f.length < 10:
+        skipped.append(filename)
+        continue
+    result = simplejson.load(f)
+    f.close()
+    revision = result['revision']
+    int_options = ""
+    options = ""
+    if 'options' in result:
+        options = result['options']
+
+    host = 'benchmarker'
+    if revision and len(revision) >10:
+        branch = result['branch']
+        if branch in ("default", "main"):
+            proj = "PyPy"
+            executable = "pypy-c"
+            branch = "main"
+        elif branch == "py3.9":
+            proj = "PyPy3.9"
+            executable = "pypy3.9-64"
+        elif branch == "py3.10":
+            proj = "PyPy3.10"
+            executable = "pypy3.10-64"
+        elif branch == "py3.11":
+            proj = "PyPy3.11"
+            executable = "pypy3.11-64"
+        else:
+            skipped.append(filename)
+            continue
+        saveresults.save(proj, revision, result['results'], executable,
+                         host, speed_url, branch=branch, changed=True)
+        saveresults.save(proj, revision, result['results'],
+                         executable.replace('-', '-jit-'),
+                         host, speed_url, branch=branch, changed=False)
+    else:
+        savecpython.save('cpython', '100', result['results'], options, 'cpython', host)
+print("\nOK")
+print("skipped", skipped)

tools/pypy/saveresults.py

@@ -3,64 +3,172 @@
 # This script saves result data                       #
 # It expects the format of unladen swallow's perf.py  #
 #######################################################
-import urllib, urllib2
+
+"""
+Upload a json file generated by runner.py.
+
+Revision, name and host are required.
+
+Example usage:
+
+  $ ./saveresults.py result.json -r '45757:fabe4fc0dc08' -n pypy-c-jit \
+    -H benchmarker
+
+  OR
+
+  $ ./saveresults.py result.json -r '45757:fabe4fc0dc08' -n pypy-c-jit-64 \
+    -H benchmarker
+"""
+from __future__ import division, print_function
+
 from datetime import datetime
+import optparse
+import sys
+import time
+try:
+    import urllib2
+    from urllib import urlencode
+except ImportError:
+    import urllib.request as urllib2
+    from urllib.parse import urlencode
+import json
+import pprint
 
-#SPEEDURL = 'http://127.0.0.1:8000/'
-SPEEDURL = 'http://speed.pypy.org/'
+SPEEDURL = 'http://127.0.0.1:8000/'
+# SPEEDURL = 'https://speed.pypy.org/'
 
-def save(project, revision, results, options, executable, environment, testing=False):
+def save(project, revision, results, executable, host, url, testing=False,
+         changed=True, branch='default'):
     testparams = []
     #Parse data
     data = {}
-    current_date = datetime.today()
+    error = 0
+
+    print("saveresults::save sending from %s to %s" %(host, url))
     for b in results:
         bench_name = b[0]
         res_type = b[1]
         results = b[2]
         value = 0
         if res_type == "SimpleComparisonResult":
-            value = results['changed_time']
+            if changed:
+                value = results['changed_time']
+            else:
+                value = results['base_time']
         elif res_type == "ComparisonResult":
-            value = results['avg_changed']
+            if changed:
+                value = results['avg_changed']
+            else:
+                value = results['avg_base']
+        elif res_type == "RawResult":
+            if changed:
+                value = results["changed_times"]
+            else:
+                value = results["base_times"]
+            if value:
+                assert len(value) == 1
+                value = value[0]
         else:
             print("ERROR: result type unknown " + b[1])
             return 1
-        data = {
+        data = [{
             'commitid': revision,
             'project': project,
             'executable': executable,
             'benchmark': bench_name,
-            'environment': environment,
+            'environment': host,
             'result_value': value,
-        }
+            'branch': branch,
+        }]
+        if not value:
+            print("Ignoring skipped result", data)
+            continue
         if res_type == "ComparisonResult":
-            data['std_dev'] = results['std_changed']
-        if testing: testparams.append(data)
-        else: send(data)
-    if testing: return testparams
-    else: return 0
+            if changed:
+                data[0]['std_dev'] = results['std_changed']
+            else:
+                data[0]['std_dev'] = results['std_base']
+        if testing:
+            testparams.append(data)
+        else:
+            error |= send(data, url)
 
-def send(data):
+    if error:
+        raise IOError("Saving failed.  See messages above.")
+    if testing:
+        return testparams
+    else:
+        return 0
+
+
+def send(data, url):
     #save results
-    params = urllib.urlencode(data)
+    params = urlencode({'json': json.dumps(data)}).encode("utf-8")
     f = None
     response = "None"
-    info = str(datetime.today()) + ": Saving result for " + data['executable']
-    info += " revision " + " " + str(data['commitid']) + ", benchmark "
-    info += data['benchmark']
+    info = ("%s: Saving result for %s revision %s, benchmark %s" %
+            (str(datetime.today()), data[0]['executable'],
+             str(data[0]['commitid']), data[0]['benchmark']))
     print(info)
     try:
-        f = urllib2.urlopen(SPEEDURL + 'result/add/', params)
-        response = f.read()
-        f.close()
+        retries = [1, 3, 6, 20]
+        while True:
+            try:
+                print('result/add')
+                f = urllib2.urlopen(url + 'result/add/json/', params)
+                print('urlopen')
+                response = f.read().decode()
+                print('response')
+                f.close()
+                break
+            except urllib2.URLError:
+                if not retries:
+                    raise
+                d = retries.pop(0)
+                print("retrying in %d seconds..." % d)
+                time.sleep(d)
     except urllib2.URLError as e:
         if hasattr(e, 'reason'):
-            response = '\n  We failed to reach a server\n'
+            response = '\n  We failed to save the data\n'
             response += '  Reason: ' + str(e.reason)
         elif hasattr(e, 'code'):
-            response = '\n  The server couldn\'t fulfill the request\n'
-            response += '  Error code: ' + str(e)
-        print("Server (%s) response: %s\n" % (SPEEDURL, response))
+            response = '\n  The server couldn\'t fulfill the request'
+        if hasattr(e, 'readlines'):
+            response = "".join([response] + [s.decode() for s in e.readlines()])
+        print(response)
+        print('when sending')
+        pprint.pprint(data)
+        raise
+        with open('error.html', 'w') as error_file:
+            error_file.write(response)
+        print("Server (%s) response written to error.html" % (url,))
+        print('  Error code: %s\n' % (e,))
         return 1
+    print("saved correctly!", end='\n\n')
     return 0
+
+if __name__ == '__main__':
+    usage = "Usage: %prog [options] <results.json>"
+    parser = optparse.OptionParser(usage=usage)
+    parser.add_option('-b', '--base', action='store_true',
+                      help='take base values instead of modified')
+    parser.add_option('--host', type=str, default=SPEEDURL,
+                      help="codespeed instance, defaults to '%s'" % SPEEDURL)
+    parser.add_option('--revision',
+         help=('for cpython, revision number (100 for cpython 2.6.2, 101 for 2.7.2, '
+               '110 for cpython3.7.6, 120 for cpython3.11, '
+               'edit admin interface "Revisions" to add more)\n\n'
+               'for pypy, use the hash from the json file'),
+        )
+    parser.add_option('--branch', help="branch name (main, py3.11)", default='main')
+    options, args = parser.parse_args(sys.argv)
+    if options.host[-1] != '/':
+        raise ValueError("host must end with '/'")
+    if not options.host.startswith("http"):
+        raise ValueError("host must start with 'http'")
+    if len(args) != 2:
+        print(parser.usage)
+        sys.exit(1)
+    results = json.load(open(args[1]))['results']
+    save('cpython', options, results, 'cpython', 'benchmarker',
+         testing=False)