merge tobami/master into branch

Author: mattip

README.md

@@ -224,6 +224,18 @@ COMP_EXECUTABLES = [
     ('myexe', 'L'),
 ]
 ```
+* `COMPARISON_COMMIT_TAGS: Defines a list of tags to display on the comparison page. This comes
+  handy when there are a lot of tags. It defaults to ``None`` which means display all the available
+  tags.
+
+### VCS Provider Specific Settings
+
+#### Github
+
+* ``GITHUB_OAUTH_TOKEN`` - Github oAuth token to use for authenticating against
+  the Github API. If not provided, it will default to unauthenticated API requests
+  which have low rate limits so an exception may be thrown when retrieving info
+  from the Github API due to the rate limit being reached.
 
 ## Getting help
 

codespeed/admin.py

@@ -37,7 +37,7 @@ class ProjectAdmin(admin.ModelAdmin):
 
 @admin.register(Branch)
 class BranchAdmin(admin.ModelAdmin):
-    list_display = ('name', 'project')
+    list_display = ('name', 'project', 'display_on_comparison_page')
     list_filter = ('project',)
 
 

codespeed/commits/git.py

@@ -1,37 +1,39 @@
 import datetime
 import logging
 import os
-from string import strip
-from subprocess import Popen, PIPE
 
+from subprocess import Popen, PIPE
 from django.conf import settings
-
 from .exceptions import CommitLogError
 
 logger = logging.getLogger(__name__)
 
 
+def execute_command(cmd, cwd):
+    p = Popen(cmd, stdout=PIPE, stderr=PIPE, cwd=cwd)
+    stdout, stderr = p.communicate()
+    stdout = stdout.decode('utf8') if stdout is not None else stdout
+    stderr = stderr.decode('utf8') if stderr is not None else stderr
+    return (p, stdout, stderr)
+
+
 def updaterepo(project, update=True):
     if os.path.exists(project.working_copy):
         if not update:
             return
 
-        p = Popen(['git', 'pull'], stdout=PIPE, stderr=PIPE,
-                  cwd=project.working_copy)
+        p, _, stderr = execute_command(['git', 'pull'], cwd=project.working_copy)
 
-        stdout, stderr = p.communicate()
         if p.returncode != 0:
             raise CommitLogError("git pull returned %s: %s" % (p.returncode,
                                                                stderr))
         else:
             return [{'error': False}]
     else:
         cmd = ['git', 'clone', project.repo_path, project.repo_name]
-        p = Popen(cmd, stdout=PIPE, stderr=PIPE,
-                  cwd=settings.REPOSITORY_BASE_PATH)
+        p, stdout, stderr = execute_command(cmd, settings.REPOSITORY_BASE_PATH)
         logger.debug('Cloning Git repo {0} for project {1}'.format(
             project.repo_path, project))
-        stdout, stderr = p.communicate()
 
         if p.returncode != 0:
             raise CommitLogError("%s returned %s: %s" % (
@@ -59,32 +61,24 @@ def getlogs(endrev, startrev):
         cmd.append(endrev.commitid)
 
     working_copy = endrev.branch.project.working_copy
-    p = Popen(cmd, stdout=PIPE, stderr=PIPE, cwd=working_copy)
-
-    stdout, stderr = p.communicate()
+    p, stdout, stderr = execute_command(cmd, working_copy)
 
     if p.returncode != 0:
         raise CommitLogError("%s returned %s: %s" % (
                              " ".join(cmd), p.returncode, stderr))
     logs = []
-    for log in filter(None, stdout.split(b'\x1e')):
+    for log in filter(None, stdout.split('\x1e')):
         (short_commit_id, commit_id, date_t, author_name, author_email,
-         subject, body) = map(strip, log.split(b'\x00', 7))
-
-        tag = ""
+            subject, body) = map(lambda s: s.strip(), log.split('\x00', 7))
 
         cmd = ["git", "tag", "--points-at", commit_id]
-        proc = Popen(cmd, stdout=PIPE, stderr=PIPE, cwd=working_copy)
 
         try:
-            stdout, stderr = proc.communicate()
-        except ValueError:
-            stdout = b''
-            stderr = b''
-
-        if proc.returncode == 0:
-            tag = stdout.strip()
+            p, stdout, stderr = execute_command(cmd, working_copy)
+        except Exception:
+            logger.debug('Failed to get tag', exc_info=True)
 
+        tag = stdout.strip() if p.returncode == 0 else ""
         date = datetime.datetime.fromtimestamp(
             int(date_t)).strftime("%Y-%m-%d %H:%M:%S")
 

codespeed/commits/github.py

@@ -11,14 +11,17 @@
 try:
     # Python 3
     from urllib.request import urlopen
+    from urllib.request import Request
 except ImportError:
     # Python 2
-    from urllib import urlopen
+    from urllib2 import urlopen
+    from urllib2 import Request
 import re
 import json
 
 import isodate
 from django.core.cache import cache
+from django.conf import settings
 
 from .exceptions import CommitLogError
 
@@ -42,8 +45,17 @@ def fetch_json(url):
     json_obj = cache.get(url)
 
     if json_obj is None:
+        github_oauth_token = getattr(settings, 'GITHUB_OAUTH_TOKEN', None)
+
+        if github_oauth_token:
+            headers = {'Authorization': 'token %s' % (github_oauth_token)}
+        else:
+            headers = {}
+
+        request = Request(url=url, headers=headers)
+
         try:
-            json_obj = json.load(urlopen(url))
+            json_obj = json.load(urlopen(request))
         except IOError as e:
             logger.exception("Unable to load %s: %s",
                              url, e, exc_info=True)
@@ -91,11 +103,13 @@ def retrieve_revision(commit_id, username, project, revision=None):
     if revision:
         # Overwrite any existing data we might have for this revision since
         # we never want our records to be out of sync with the actual VCS:
-
-        # We need to convert the timezone-aware date to a naive (i.e.
-        # timezone-less) date in UTC to avoid killing MySQL:
-        revision.date = date.astimezone(
-            isodate.tzinfo.Utc()).replace(tzinfo=None)
+        if not getattr(settings, 'USE_TZ_AWARE_DATES', False):
+            # We need to convert the timezone-aware date to a naive (i.e.
+            # timezone-less) date in UTC to avoid killing MySQL:
+            logger.debug('USE_TZ_AWARE_DATES setting is set to False, '
+                         'converting datetime object to a naive one')
+            revision.date = date.astimezone(
+                isodate.tzinfo.Utc()).replace(tzinfo=None)
         revision.author = commit_json['author']['name']
         revision.message = commit_json['message']
         revision.full_clean()

codespeed/commits/tests/__init__.py

@@ -0,0 +1,60 @@
+from datetime import datetime
+from django.test import TestCase, override_settings
+from mock import Mock, patch
+
+from codespeed.commits.git import getlogs
+from codespeed.models import Project, Revision, Branch, Environment
+
+
+@override_settings(ALLOW_ANONYMOUS_POST=True)
+class GitTest(TestCase):
+    def setUp(self):
+        self.env = Environment.objects.create(name='env')
+        self.project = Project.objects.create(name='project',
+                                              repo_path='path',
+                                              repo_type=Project.GIT)
+        self.branch = Branch.objects.create(name='default',
+                                            project_id=self.project.id)
+        self.revision = Revision.objects.create(
+            **{
+               'commitid': 'id1',
+               'date': datetime.now(),
+               'project_id': self.project.id,
+               'branch_id': self.branch.id,
+            }
+        )
+
+    @patch("codespeed.commits.git.Popen")
+    def test_git_output_parsing(self, popen):
+        # given
+        outputs = {
+            "log": b"id\x00long_id\x001583489681\x00author\x00email\x00msg\x00\x1e",
+            "tag": b'tag',
+        }
+
+        def side_effect(cmd, *args, **kwargs):
+            ret = Mock()
+            ret.returncode = 0
+            git_command = cmd[1] if len(cmd) > 0 else None
+            output = outputs.get(git_command, b'')
+            ret.communicate.return_value = (output, b'')
+            return ret
+
+        popen.side_effect = side_effect
+
+        # when
+        # revision doesn't matter here, git commands are mocked
+        logs = getlogs(self.revision, self.revision)
+
+        # then
+        expected = {
+            'date': '2020-03-06 04:14:41',
+            'message': 'msg',
+            'commitid': 'long_id',
+            'author': 'author',
+            'author_email': 'email',
+            'body': '',
+            'short_commit_id': 'id',
+            'tag': 'tag',
+        }
+        self.assertEquals([expected], logs)

codespeed/commits/tests/test_git.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.1.15 on 2020-02-24 11:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('codespeed', '0003_project_default_branch'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='branch',
+            name='display_on_comparison_page',
+            field=models.BooleanField(default=True, verbose_name='True to display this branch on the comparison page'),
+        ),
+    ]

codespeed/migrations/0004_branch_display_on_comparison_page.py

@@ -107,6 +107,9 @@ class Branch(models.Model):
     name = models.CharField(max_length=32)
     project = models.ForeignKey(
         Project, on_delete=models.CASCADE, related_name="branches")
+    display_on_comparison_page = models.BooleanField(
+        "True to display this branch on the comparison page",
+        default=True)
 
     def __str__(self):
         return self.project.name + ":" + self.name

codespeed/models.py

@@ -78,8 +78,29 @@
                          #     ('myexe', '21df2423ra'),
                          #     ('myexe', 'L'),]
 
+COMPARISON_COMMIT_TAGS = None  # List of tag names which should be included in the executables list
+                               # on the comparision page.
+                               # This comes handy where project contains a lot of tags, but you only want
+                               # to list subset of them on the comparison page.
+                               # If this value is set to None (default value), all the available tags will
+                               # be included.
+
+TIMELINE_EXECUTABLE_NAME_MAX_LEN = 22  # Maximum length of the executable name used in the
+                                       # Changes and Timeline view. If the name is longer, the name
+                                       # will be truncated and "..." will be added at the end.
+
+COMPARISON_EXECUTABLE_NAME_MAX_LEN = 20  # Maximum length of the executable name  used in the
+                                         # Coomparison view. If the name is longer, the name
+
 USE_MEDIAN_BANDS = True # True to enable median bands on Timeline view
 
 
 ALLOW_ANONYMOUS_POST = True  # Whether anonymous users can post results
 REQUIRE_SECURE_AUTH = True  # Whether auth needs to be over a secure channel
+
+US_TZ_AWARE_DATES = False  # True to use timezone aware datetime objects with Github provider.
+                           # NOTE: Some database backends may not support tz aware dates.
+
+GITHUB_OAUTH_TOKEN = None  # Github oAuth token to use when using Github repo type. If not
+                           # specified, it will utilize unauthenticated requests which have
+                           # low rate limits.

codespeed/settings.py

@@ -6,7 +6,7 @@
 
 {% block extra_head %}
 {{ block.super }}
-  <link rel="stylesheet" type="text/css" href="{% static '/css/timeline.css' %}" />
+  <link rel="stylesheet" type="text/css" href="{% static 'css/timeline.css' %}" />
   <link rel="stylesheet" type="text/css" href="{% static 'js/jqplot/jquery.jqplot.min.css' %}" />
 {% endblock %}