Enable catching non pull request commits in repositories

config/repo-config.csv

@@ -1,10 +1,2 @@
-Repository's Location,Branch,File formats,Ignore Glob List,Ignore standalone config,Ignore Commits List,Ignore Authors List,Shallow Cloning
-https://github.com/reposense/testrepo-Alpha.git,master,,,,2fb6b9b2dd9fa40bf0f9815da2cb0ae8731436c7;c5a6dc774e22099cd9ddeb0faff1e75f9cf4f151;cd7f610e0becbdf331d5231887d8010a689f87c7;768015345e70f06add2a8b7d1f901dc07bf70582,,
-https://github.com/reposense/testrepo-Beta.git,master,fxml,docs**,yes,,,
-https://github.com/reposense/testrepo-Beta.git,add-config-json,fxml,docs**,yes,,,
-https://github.com/reposense/testrepo-Delta.git,master,override:java;md,,,,,
-https://github.com/reposense/testrepo-Delta.git,nonExistentBranch,,,,,,
-https://github.com/reposense/testrepo-Delta.git,add-binary-file,,,,,,
-https://github.com/reposense/RepoSense.git,master,,,,,,
-https://github.com/reposense/testrepo-Empty.git,master,,,,,,
-ftp://github.com/reposense/RepoSense.git,master,,,,,,
+Repository's Location,Branch,File formats,Ignore Glob List,Ignore standalone config,Ignore Commits List,Ignore Authors List,Shallow Cloning,Catch Non PR Commits
+https://github.com/FH-30/Test-Repo.git,master,,,,,,,yes

src/main/java/reposense/RepoSense.java

@@ -74,6 +74,9 @@ public static void main(String[] args) {
                     cliArguments.isLastModifiedDateIncluded());
             RepoConfiguration.setIsShallowCloningPerformedToRepoConfigs(configs,
                     cliArguments.isShallowCloningPerformed());
+            RepoConfiguration.setIsCatchingNonPrCommitsPerformedToRepoConfigs(configs,
+                    cliArguments.isCatchingNonPrCommitsPerformed());
+
             List<Path> reportFoldersAndFiles = ReportGenerator.generateReposReport(configs,
                     cliArguments.getOutputFilePath().toAbsolutePath().toString(),
                     cliArguments.getAssetsFilePath().toAbsolutePath().toString(), reportConfig,

src/main/java/reposense/commits/CommitInfoAnalyzer.java

@@ -2,15 +2,19 @@
 
 import static reposense.util.StringsUtil.removeQuote;
 
+import java.net.URL;
+import java.net.URLConnection;
 import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Scanner;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
@@ -46,6 +50,7 @@ public class CommitInfoAnalyzer {
     private static final String REF_SPLITTER = ",\\s";
     private static final String NEW_LINE_SPLITTER = "\\n";
     private static final String TAG_PREFIX = "tag:";
+    private static final String PR_COMMIT_INDICATOR = "class=\"pull-request\"";
 
     private static final int COMMIT_HASH_INDEX = 0;
     private static final int AUTHOR_INDEX = 1;
@@ -76,6 +81,41 @@ public static List<CommitResult> analyzeCommits(List<CommitInfo> commitInfos, Re
                 .collect(Collectors.toList());
     }
 
+    /**
+     * Get HTML from a website.
+     * Source: https://stackoverflow.com/questions/31462/how-to-fetch-html-in-java.
+     * @param site Site to query html from.
+     * @return the HTML from the given website.
+     */
+    public static String getHtml(String site) {
+        String content = null;
+        URLConnection connection = null;
+        try {
+            connection =  new URL(site).openConnection();
+            Scanner scanner = new Scanner(connection.getInputStream());
+            scanner.useDelimiter("\\Z");
+            content = scanner.next();
+            scanner.close();
+        }catch ( Exception ex ) {
+            ex.printStackTrace();
+        }
+
+        return content;
+    }
+
+    /**
+     * Get the URL which returns the pull request url that a commit hash belongs to (if any).
+     * @param config Config of the repository that the commit belongs to.
+     * @param hash Hash of the relevant commit.
+     * @return URL which returns HTML containing pull request url of given commit hash.
+     */
+    private static String getGithubQueryUrl(RepoConfiguration config, String hash) {
+        String githubCloneUrl = config.getLocation().toString();
+        String githubRepoUrl = githubCloneUrl.substring(0, githubCloneUrl.length() - 4);
+        String commitHashUrl = "/branch_commits/" + hash;
+        return githubRepoUrl + commitHashUrl;
+    }
+
     /**
      * Extracts the relevant data from {@code commitInfo} into a {@code CommitResult}.
      */
@@ -98,6 +138,12 @@ public static CommitResult analyzeCommit(CommitInfo commitInfo, RepoConfiguratio
         String messageBody = (elements.length > MESSAGE_BODY_INDEX)
                 ? getCommitMessageBody(elements[MESSAGE_BODY_INDEX]) : "";
 
+
+        String githubCommitUrl = getGithubQueryUrl(config, hash);
+        String htmlContainingPrUrl = getHtml(githubCommitUrl);
+
+        boolean isNonPrCommit = !htmlContainingPrUrl.contains(PR_COMMIT_INDICATOR);
+
         String[] refs = (elements.length > REF_NAME_INDEX)
                 ? elements[REF_NAME_INDEX].split(REF_SPLITTER)
                 : new String[]{""};
@@ -109,22 +155,23 @@ public static CommitResult analyzeCommit(CommitInfo commitInfo, RepoConfiguratio
         }
 
         if (statLine.isEmpty()) { // empty commit, no files changed
-            return new CommitResult(author, hash, date, messageTitle, messageBody, tags);
+            return new CommitResult(author, hash, date, messageTitle, messageBody, isNonPrCommit, tags);
         }
 
         String[] statInfos = statLine.split(NEW_LINE_SPLITTER);
         String[] fileTypeContributions = Arrays.copyOfRange(statInfos, 0, statInfos.length - 1);
         Map<FileType, ContributionPair> fileTypeAndContributionMap =
                 getFileTypesAndContribution(fileTypeContributions, config);
 
-        return new CommitResult(author, hash, date, messageTitle, messageBody, tags, fileTypeAndContributionMap);
+        return new CommitResult(author, hash, date, messageTitle, messageBody,  isNonPrCommit,
+                tags, fileTypeAndContributionMap);
     }
 
     /**
      * Returns the number of lines added and deleted for the specified file types in {@code config}.
      */
     private static Map<FileType, ContributionPair> getFileTypesAndContribution(String[] filePathContributions,
-            RepoConfiguration config) {
+                                                                               RepoConfiguration config) {
         Map<FileType, ContributionPair> fileTypesAndContributionMap = new HashMap<>();
         for (String filePathContribution : filePathContributions) {
             String[] infos = filePathContribution.split(TAB_SPLITTER);

src/main/java/reposense/commits/CommitResultAggregator.java

@@ -40,6 +40,9 @@ public static CommitContributionSummary aggregateCommitResults(
                 getAuthorDailyContributionsMap(config.getAuthorDisplayNameMap().keySet(), commitResults,
                         config.getZoneId());
 
+        Map<Author, Integer> authorNonPrCommitsMap = getAuthorNonPrCommitsMap(
+                config.getAuthorDisplayNameMap().keySet(), commitResults);
+
         Date lastDate = commitResults.size() == 0
                 ? null
                 : getStartOfDate(commitResults.get(commitResults.size() - 1).getTime(), config.getZoneId());
@@ -50,7 +53,8 @@ public static CommitContributionSummary aggregateCommitResults(
         return new CommitContributionSummary(
                 config.getAuthorDisplayNameMap(),
                 authorDailyContributionsMap,
-                authorContributionVariance);
+                authorContributionVariance,
+                authorNonPrCommitsMap);
     }
 
     /**
@@ -121,6 +125,26 @@ private static Map<Author, List<AuthorDailyContribution>> getAuthorDailyContribu
         return authorDailyContributionsMap;
     }
 
+    private static Map<Author, Integer> getAuthorNonPrCommitsMap (Set<Author> authorSet,
+                                                                  List<CommitResult> commitResults) {
+        Map<Author, Integer> authorNonPrCommitsMap = new HashMap<>();
+        authorSet.forEach(author -> authorNonPrCommitsMap.put(author, 0));
+
+        for (CommitResult commitResult : commitResults) {
+            Author commitAuthor = commitResult.getAuthor();
+
+            Integer authorNonPrCommits = authorNonPrCommitsMap.get(commitAuthor);
+
+            if (commitResult.isNonPrCommit()) {
+                authorNonPrCommits++;
+            }
+
+            authorNonPrCommitsMap.put(commitAuthor, authorNonPrCommits);
+        }
+
+        return authorNonPrCommitsMap;
+    }
+
     private static void addDailyContributionForNewDate(
             List<AuthorDailyContribution> authorDailyContributions, Date date) {
         authorDailyContributions.add(new AuthorDailyContribution(date));

src/main/java/reposense/commits/model/CommitContributionSummary.java

@@ -12,14 +12,17 @@ public class CommitContributionSummary {
     private final Map<Author, List<AuthorDailyContribution>> authorDailyContributionsMap;
     private final Map<Author, Float> authorContributionVariance;
     private final Map<Author, String> authorDisplayNameMap;
+    private final Map<Author, Integer> authorNonPrCommitsMap;
 
     public CommitContributionSummary(
             Map<Author, String> authorDisplayNameMap,
             Map<Author, List<AuthorDailyContribution>> authorDailyContributionsMap,
-            Map<Author, Float> authorContributionVariance) {
+            Map<Author, Float> authorContributionVariance,
+            Map<Author, Integer> authorNonPrCommitsMap) {
         this.authorDisplayNameMap = authorDisplayNameMap;
         this.authorDailyContributionsMap = authorDailyContributionsMap;
         this.authorContributionVariance = authorContributionVariance;
+        this.authorNonPrCommitsMap = authorNonPrCommitsMap;
     }
 
     public Map<Author, String> getAuthorDisplayNameMap() {
@@ -33,4 +36,6 @@ public Map<Author, List<AuthorDailyContribution>> getAuthorDailyContributionsMap
     public Map<Author, Float> getAuthorContributionVariance() {
         return authorContributionVariance;
     }
+
+    public Map<Author, Integer> getAuthorNonPrCommitsMap() { return authorNonPrCommitsMap; }
 }

src/main/java/reposense/commits/model/CommitResult.java

@@ -15,29 +15,33 @@ public class CommitResult {
     private final String hash;
     private final String messageTitle;
     private final String messageBody;
+    private final boolean isNonPrCommit;
     private final String[] tags;
     private final Map<FileType, ContributionPair> fileTypesAndContributionMap;
 
     private final transient Author author;
     private final transient Date time;
 
-    public CommitResult(Author author, String hash, Date time, String messageTitle, String messageBody, String[] tags,
-            Map<FileType, ContributionPair> fileTypesAndContributionMap) {
+    public CommitResult(Author author, String hash, Date time, String messageTitle, String messageBody,
+                        boolean isNonPrCommit, String[] tags, Map<FileType, ContributionPair> fileTypesAndContributionMap) {
         this.author = author;
         this.hash = hash;
         this.time = time;
         this.messageTitle = messageTitle;
         this.messageBody = messageBody;
+        this.isNonPrCommit = isNonPrCommit;
         this.tags = tags;
         this.fileTypesAndContributionMap = fileTypesAndContributionMap;
     }
 
-    public CommitResult(Author author, String hash, Date time, String messageTitle, String messageBody, String[] tags) {
+    public CommitResult(Author author, String hash, Date time, String messageTitle, String messageBody,
+                        boolean isNonPrCommit, String[] tags) {
         this.author = author;
         this.hash = hash;
         this.time = time;
         this.messageTitle = messageTitle;
         this.messageBody = messageBody;
+        this.isNonPrCommit = isNonPrCommit;
         this.tags = tags;
         this.fileTypesAndContributionMap = Collections.emptyMap();
     }
@@ -66,6 +70,8 @@ public Date getTime() {
         return time;
     }
 
+    public boolean isNonPrCommit() { return isNonPrCommit; }
+
     public int getInsertions() {
         int insertions = 0;
         for (ContributionPair contributionPair : fileTypesAndContributionMap.values()) {

src/main/java/reposense/model/CliArguments.java

@@ -20,6 +20,7 @@ public abstract class CliArguments {
     protected boolean isShallowCloningPerformed;
     protected boolean isAutomaticallyLaunching;
     protected boolean isStandaloneConfigIgnored;
+    protected boolean isCatchingNonPrCommitsPerformed;
     protected int numCloningThreads;
     protected int numAnalysisThreads;
     protected ZoneId zoneId;
@@ -60,6 +61,8 @@ public boolean isShallowCloningPerformed() {
         return isShallowCloningPerformed;
     }
 
+    public boolean isCatchingNonPrCommitsPerformed() { return isCatchingNonPrCommitsPerformed; }
+
     public List<FileType> getFormats() {
         return formats;
     }

src/main/java/reposense/model/ConfigCliArguments.java

@@ -27,7 +27,7 @@ public class ConfigCliArguments extends CliArguments {
     public ConfigCliArguments(Path configFolderPath, Path outputFilePath, Path assetsFilePath, Date sinceDate,
             Date untilDate, boolean isSinceDateProvided, boolean isUntilDateProvided, int numCloningThreads,
             int numAnalysisThreads, List<FileType> formats, boolean isLastModifiedDateIncluded,
-            boolean isShallowCloningPerformed, boolean isAutomaticallyLaunching,
+            boolean isShallowCloningPerformed, boolean isCatchingNonPrCommitsPerformed, boolean isAutomaticallyLaunching,
             boolean isStandaloneConfigIgnored, ZoneId zoneId, ReportConfiguration reportConfiguration) {
         this.configFolderPath = configFolderPath.equals(EMPTY_PATH)
                 ? configFolderPath.toAbsolutePath()
@@ -45,6 +45,7 @@ public ConfigCliArguments(Path configFolderPath, Path outputFilePath, Path asset
         this.formats = formats;
         this.isLastModifiedDateIncluded = isLastModifiedDateIncluded;
         this.isShallowCloningPerformed = isShallowCloningPerformed;
+        this.isCatchingNonPrCommitsPerformed = isCatchingNonPrCommitsPerformed;
         this.isAutomaticallyLaunching = isAutomaticallyLaunching;
         this.isStandaloneConfigIgnored = isStandaloneConfigIgnored;
         this.numCloningThreads = numCloningThreads;

src/main/java/reposense/model/LocationsCliArguments.java

@@ -14,7 +14,7 @@ public class LocationsCliArguments extends CliArguments {
     public LocationsCliArguments(List<String> locations, Path outputFilePath, Path assetsFilePath, Date sinceDate,
             Date untilDate, boolean isSinceDateProvided, boolean isUntilDateProvided, int numCloningThreads,
             int numAnalysisThreads, List<FileType> formats, boolean isLastModifiedDateIncluded,
-            boolean isShallowCloningPerformed, boolean isAutomaticallyLaunching,
+            boolean isShallowCloningPerformed, boolean isCatchingNonPrCommitsPerformed, boolean isAutomaticallyLaunching,
             boolean isStandaloneConfigIgnored, ZoneId zoneId) {
         this.locations = locations;
         this.outputFilePath = outputFilePath;
@@ -25,6 +25,7 @@ public LocationsCliArguments(List<String> locations, Path outputFilePath, Path a
         this.isUntilDateProvided = isUntilDateProvided;
         this.isLastModifiedDateIncluded = isLastModifiedDateIncluded;
         this.isShallowCloningPerformed = isShallowCloningPerformed;
+        this.isCatchingNonPrCommitsPerformed = isCatchingNonPrCommitsPerformed;
         this.formats = formats;
         this.isAutomaticallyLaunching = isAutomaticallyLaunching;
         this.isStandaloneConfigIgnored = isStandaloneConfigIgnored;

src/main/java/reposense/model/RepoConfiguration.java

@@ -38,6 +38,7 @@ public class RepoConfiguration {
     private transient List<CommitHash> ignoreCommitList;
     private transient boolean isLastModifiedDateIncluded;
     private transient boolean isShallowCloningPerformed;
+    private transient boolean isCatchingNonPrCommitsPerformed;
     private transient boolean isFormatsOverriding;
     private transient boolean isIgnoreGlobListOverriding;
     private transient boolean isIgnoreCommitListOverriding;
@@ -49,13 +50,13 @@ public RepoConfiguration(RepoLocation location) {
 
     public RepoConfiguration(RepoLocation location, String branch) {
         this(location, branch, Collections.emptyList(), Collections.emptyList(), false, Collections.emptyList(),
-                false, false, false, false);
+                false, false, false, false, false);
     }
 
     public RepoConfiguration(RepoLocation location, String branch, List<FileType> formats, List<String> ignoreGlobList,
             boolean isStandaloneConfigIgnored, List<CommitHash> ignoreCommitList, boolean isFormatsOverriding,
             boolean isIgnoreGlobListOverriding, boolean isIgnoreCommitListOverriding,
-            boolean isShallowCloningPerformed) {
+            boolean isShallowCloningPerformed, boolean isCatchingNonPrCommitsPerformed) {
         this.authorConfig = new AuthorConfiguration(location, branch);
         this.location = location;
         this.branch = location.isEmpty() ? DEFAULT_BRANCH : branch;
@@ -67,6 +68,7 @@ public RepoConfiguration(RepoLocation location, String branch, List<FileType> fo
         this.isIgnoreGlobListOverriding = isIgnoreGlobListOverriding;
         this.isIgnoreCommitListOverriding = isIgnoreCommitListOverriding;
         this.isShallowCloningPerformed = isShallowCloningPerformed;
+        this.isCatchingNonPrCommitsPerformed = isCatchingNonPrCommitsPerformed;
 
         String organization = location.getOrganization();
         String repoName = location.getRepoName();
@@ -108,6 +110,13 @@ public static void setIsShallowCloningPerformedToRepoConfigs(List<RepoConfigurat
         }
     }
 
+    public static void setIsCatchingNonPrCommitsPerformedToRepoConfigs(List<RepoConfiguration> configs,
+                                                                       boolean isCatchingNonPrCommitsPerformed) {
+        if (isCatchingNonPrCommitsPerformed) {
+            configs.stream().forEach(config -> config.setIsCatchingNonPrCommitsPerformed(true));
+        }
+    }
+
     /**
      * Merges a {@code RepoConfiguration} from {@code repoConfigs} with an {@code AuthorConfiguration} from
      * {@code authorConfigs} if their {@code RepoLocation} and branch matches
@@ -395,6 +404,10 @@ public void setIsShallowCloningPerformed(boolean isShallowCloningPerformed) {
         this.isShallowCloningPerformed = isShallowCloningPerformed;
     }
 
+    public void setIsCatchingNonPrCommitsPerformed(boolean isCatchingNonPrCommitsPerformed) {
+        this.isCatchingNonPrCommitsPerformed = isCatchingNonPrCommitsPerformed;
+    }
+
     public boolean isLastModifiedDateIncluded() {
         return this.isLastModifiedDateIncluded;
     }