Merge pull request #12 from mattip/use-python3

Use python3

Author: mattip

.travis.yml

@@ -14,7 +14,7 @@ matrix:
     - python: "3.5"
       env: DJANGO_VERSION=2.1
 install:
-  - pip install flake8
+  - pip install flake8 mock
   - pip install -q Django==$DJANGO_VERSION
   - python setup.py install
 before_script:

README.md

@@ -161,7 +161,7 @@ same name.
 
 * The results associated to an executable and a revision which has a non blank
   tag field will be listed as a baseline option in the Timeline view.
-* Additionaly, the Comparison view will show the results of the latest revision
+* Additionally, the Comparison view will show the results of the latest revision
   of projects being tracked as an executable as well.
 
 ### Defaults
@@ -224,6 +224,18 @@ COMP_EXECUTABLES = [
     ('myexe', 'L'),
 ]
 ```
+* `COMPARISON_COMMIT_TAGS: Defines a list of tags to display on the comparison page. This comes
+  handy when there are a lot of tags. It defaults to ``None`` which means display all the available
+  tags.
+
+### VCS Provider Specific Settings
+
+#### Github
+
+* ``GITHUB_OAUTH_TOKEN`` - Github oAuth token to use for authenticating against
+  the Github API. If not provided, it will default to unauthenticated API requests
+  which have low rate limits so an exception may be thrown when retrieving info
+  from the Github API due to the rate limit being reached.
 
 ## Getting help
 

codespeed/admin.py

@@ -37,7 +37,7 @@ class ProjectAdmin(admin.ModelAdmin):
 
 @admin.register(Branch)
 class BranchAdmin(admin.ModelAdmin):
-    list_display = ('name', 'project')
+    list_display = ('name', 'project', 'display_on_comparison_page')
     list_filter = ('project',)
 
 

codespeed/auth.py

@@ -1,4 +1,5 @@
 import logging
+import types
 from functools import wraps
 from django.contrib.auth import authenticate, login
 from django.http import HttpResponse, HttpResponseForbidden
@@ -9,6 +10,20 @@
 logger = logging.getLogger(__name__)
 
 
+def is_authenticated(request):
+    # NOTE: We do type check so we also support newer versions of Django when
+    # is_authenticated and some other methods have been properties
+    if isinstance(request.user.is_authenticated, (types.FunctionType,
+                                                  types.MethodType)):
+        return request.user.is_authenticated()
+    elif isinstance(request.user.is_authenticated, bool):
+        return request.user.is_authenticated
+    else:
+        logger.info('Got unexpected type for request.user.is_authenticated '
+                    'variable')
+        return False
+
+
 def basic_auth_required(realm='default'):
     def _helper(func):
         @wraps(func)
@@ -18,7 +33,7 @@ def _decorator(request, *args, **kwargs):
             if settings.ALLOW_ANONYMOUS_POST:
                 logger.debug('allowing anonymous post')
                 allowed = True
-            elif hasattr(request, 'user') and request.user.is_authenticated():
+            elif hasattr(request, 'user') and is_authenticated(request=request):
                 allowed = True
             elif 'HTTP_AUTHORIZATION' in request.META:
                 logger.debug('checking for http authorization header')

codespeed/commits/git.py

@@ -1,37 +1,39 @@
 import datetime
 import logging
 import os
-from string import strip
-from subprocess import Popen, PIPE
 
+from subprocess import Popen, PIPE
 from django.conf import settings
-
 from .exceptions import CommitLogError
 
 logger = logging.getLogger(__name__)
 
 
+def execute_command(cmd, cwd):
+    p = Popen(cmd, stdout=PIPE, stderr=PIPE, cwd=cwd)
+    stdout, stderr = p.communicate()
+    stdout = stdout.decode('utf8') if stdout is not None else stdout
+    stderr = stderr.decode('utf8') if stderr is not None else stderr
+    return (p, stdout, stderr)
+
+
 def updaterepo(project, update=True):
     if os.path.exists(project.working_copy):
         if not update:
             return
 
-        p = Popen(['git', 'pull'], stdout=PIPE, stderr=PIPE,
-                  cwd=project.working_copy)
+        p, _, stderr = execute_command(['git', 'pull'], cwd=project.working_copy)
 
-        stdout, stderr = p.communicate()
         if p.returncode != 0:
             raise CommitLogError("git pull returned %s: %s" % (p.returncode,
                                                                stderr))
         else:
             return [{'error': False}]
     else:
         cmd = ['git', 'clone', project.repo_path, project.repo_name]
-        p = Popen(cmd, stdout=PIPE, stderr=PIPE,
-                  cwd=settings.REPOSITORY_BASE_PATH)
+        p, stdout, stderr = execute_command(cmd, settings.REPOSITORY_BASE_PATH)
         logger.debug('Cloning Git repo {0} for project {1}'.format(
             project.repo_path, project))
-        stdout, stderr = p.communicate()
 
         if p.returncode != 0:
             raise CommitLogError("%s returned %s: %s" % (
@@ -59,32 +61,24 @@ def getlogs(endrev, startrev):
         cmd.append(endrev.commitid)
 
     working_copy = endrev.branch.project.working_copy
-    p = Popen(cmd, stdout=PIPE, stderr=PIPE, cwd=working_copy)
-
-    stdout, stderr = p.communicate()
+    p, stdout, stderr = execute_command(cmd, working_copy)
 
     if p.returncode != 0:
         raise CommitLogError("%s returned %s: %s" % (
                              " ".join(cmd), p.returncode, stderr))
     logs = []
-    for log in filter(None, stdout.split(b'\x1e')):
+    for log in filter(None, stdout.split('\x1e')):
         (short_commit_id, commit_id, date_t, author_name, author_email,
-         subject, body) = map(strip, log.split(b'\x00', 7))
-
-        tag = ""
+            subject, body) = map(lambda s: s.strip(), log.split('\x00', 7))
 
         cmd = ["git", "tag", "--points-at", commit_id]
-        proc = Popen(cmd, stdout=PIPE, stderr=PIPE, cwd=working_copy)
 
         try:
-            stdout, stderr = proc.communicate()
-        except ValueError:
-            stdout = b''
-            stderr = b''
-
-        if proc.returncode == 0:
-            tag = stdout.strip()
+            p, stdout, stderr = execute_command(cmd, working_copy)
+        except Exception:
+            logger.debug('Failed to get tag', exc_info=True)
 
+        tag = stdout.strip() if p.returncode == 0 else ""
         date = datetime.datetime.fromtimestamp(
             int(date_t)).strftime("%Y-%m-%d %H:%M:%S")
 

codespeed/commits/github.py

@@ -11,14 +11,17 @@
 try:
     # Python 3
     from urllib.request import urlopen
+    from urllib.request import Request
 except ImportError:
     # Python 2
-    from urllib import urlopen
+    from urllib2 import urlopen
+    from urllib2 import Request
 import re
 import json
 
 import isodate
 from django.core.cache import cache
+from django.conf import settings
 
 from .exceptions import CommitLogError
 
@@ -42,8 +45,17 @@ def fetch_json(url):
     json_obj = cache.get(url)
 
     if json_obj is None:
+        github_oauth_token = getattr(settings, 'GITHUB_OAUTH_TOKEN', None)
+
+        if github_oauth_token:
+            headers = {'Authorization': 'token %s' % (github_oauth_token)}
+        else:
+            headers = {}
+
+        request = Request(url=url, headers=headers)
+
         try:
-            json_obj = json.load(urlopen(url))
+            json_obj = json.load(urlopen(request))
         except IOError as e:
             logger.exception("Unable to load %s: %s",
                              url, e, exc_info=True)
@@ -91,11 +103,13 @@ def retrieve_revision(commit_id, username, project, revision=None):
     if revision:
         # Overwrite any existing data we might have for this revision since
         # we never want our records to be out of sync with the actual VCS:
-
-        # We need to convert the timezone-aware date to a naive (i.e.
-        # timezone-less) date in UTC to avoid killing MySQL:
-        revision.date = date.astimezone(
-            isodate.tzinfo.Utc()).replace(tzinfo=None)
+        if not getattr(settings, 'USE_TZ_AWARE_DATES', False):
+            # We need to convert the timezone-aware date to a naive (i.e.
+            # timezone-less) date in UTC to avoid killing MySQL:
+            logger.debug('USE_TZ_AWARE_DATES setting is set to False, '
+                         'converting datetime object to a naive one')
+            revision.date = date.astimezone(
+                isodate.tzinfo.Utc()).replace(tzinfo=None)
         revision.author = commit_json['author']['name']
         revision.message = commit_json['message']
         revision.full_clean()

codespeed/commits/tests/__init__.py

@@ -0,0 +1,60 @@
+from datetime import datetime
+from django.test import TestCase, override_settings
+from mock import Mock, patch
+
+from codespeed.commits.git import getlogs
+from codespeed.models import Project, Revision, Branch, Environment
+
+
+@override_settings(ALLOW_ANONYMOUS_POST=True)
+class GitTest(TestCase):
+    def setUp(self):
+        self.env = Environment.objects.create(name='env')
+        self.project = Project.objects.create(name='project',
+                                              repo_path='path',
+                                              repo_type=Project.GIT)
+        self.branch = Branch.objects.create(name='default',
+                                            project_id=self.project.id)
+        self.revision = Revision.objects.create(
+            **{
+               'commitid': 'id1',
+               'date': datetime.now(),
+               'project_id': self.project.id,
+               'branch_id': self.branch.id,
+            }
+        )
+
+    @patch("codespeed.commits.git.Popen")
+    def test_git_output_parsing(self, popen):
+        # given
+        outputs = {
+            "log": b"id\x00long_id\x001583489681\x00author\x00email\x00msg\x00\x1e",
+            "tag": b'tag',
+        }
+
+        def side_effect(cmd, *args, **kwargs):
+            ret = Mock()
+            ret.returncode = 0
+            git_command = cmd[1] if len(cmd) > 0 else None
+            output = outputs.get(git_command, b'')
+            ret.communicate.return_value = (output, b'')
+            return ret
+
+        popen.side_effect = side_effect
+
+        # when
+        # revision doesn't matter here, git commands are mocked
+        logs = getlogs(self.revision, self.revision)
+
+        # then
+        expected = {
+            'date': '2020-03-06 04:14:41',
+            'message': 'msg',
+            'commitid': 'long_id',
+            'author': 'author',
+            'author_email': 'email',
+            'body': '',
+            'short_commit_id': 'id',
+            'tag': 'tag',
+        }
+        self.assertEquals([expected], logs)

codespeed/commits/tests/test_git.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.1.15 on 2020-02-24 11:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('codespeed', '0003_project_default_branch'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='branch',
+            name='display_on_comparison_page',
+            field=models.BooleanField(default=True, verbose_name='True to display this branch on the comparison page'),
+        ),
+    ]

codespeed/migrations/0004_branch_display_on_comparison_page.py

@@ -9,7 +9,11 @@
 from django.urls import reverse
 from django.conf import settings
 from django.db import models
-from django.utils.encoding import python_2_unicode_compatible
+try:
+    from django.utils.encoding import python_2_unicode_compatible
+except ImportError:
+    def python_2_unicode_compatible(arg):
+        return arg
 
 from .commits.github import GITHUB_URL_RE
 
@@ -107,6 +111,9 @@ class Branch(models.Model):
     name = models.CharField(max_length=32)
     project = models.ForeignKey(
         Project, on_delete=models.CASCADE, related_name="branches")
+    display_on_comparison_page = models.BooleanField(
+        "True to display this branch on the comparison page",
+        default=True)
 
     def __str__(self):
         return self.project.name + ":" + self.name