Merge pull request #5 from DeepCodeAI/dev

Major release with markers

Author: Arvi3d

README.md

@@ -3,7 +3,7 @@ Deepcode Public API package in Java
 
 [![deepcode](https://www.deepcode.ai/api/gh/badge?key=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwbGF0Zm9ybTEiOiJnaCIsIm93bmVyMSI6IkRlZXBDb2RlQUkiLCJyZXBvMSI6ImphdmEtY2xpZW50IiwiaW5jbHVkZUxpbnQiOmZhbHNlLCJhdXRob3JJZCI6MTI0NjksImlhdCI6MTU5NjA5NzIzMn0.a8lZClW69fj53juqAW0NJ6uWh-1iOXiR-mn5pN3eATc)](https://www.deepcode.ai/app/gh/DeepCodeAI/java-client/_/dashboard?utm_content=gh%2FDeepCodeAI%2Fjava-client)
 
-For low level APIs look for `DeepCodeRestApi` public methods. For descriptions look [Rest APIs and CLI](https://deepcode.freshdesk.com/support/solutions/folders/60000321393)
+For low level APIs look for `DeepCodeRestApi` public methods. For descriptions - look [Rest APIs and CLI](https://deepcode.freshdesk.com/support/solutions/folders/60000321393)
 
 For high-level APIs look inside `ai.deepcode.javaclient.core` package.
 Here common logic for any Java made IDE presented as `abstract` classes that need to be instantiated and finalised with platform specific code.
@@ -19,4 +19,10 @@ To make a standalone jar file with all dependencies use `shadowJar` gradle task.
 
 ## Run tests
 
+- 2 environment variables with __already logged__ Tokens need to be declared:
+
+`DEEPCODE_API_KEY` - logged at https://www.deepcode.ai Token 
+
+`DEEPCODE_API_KEY_STAGING` - logged at https://www.deepcoded.com Token
+
 - Run gradle test task: `source gradlew test --stacktrace --scan`

build.gradle

@@ -18,7 +18,7 @@ repositories {
     jcenter()
 }
 
-version '2.0.0'
+version '2.0.14'
 sourceCompatibility = 1.8
 
 dependencies {

src/main/java/ai/deepcode/javaclient/core/AnalysisDataBase.java

@@ -17,20 +17,30 @@ protected DeepCodeIgnoreInfoHolderBase(
   }
 
   private static final Map<Object, Set<String>> map_dcignore2Regexps = new ConcurrentHashMap<>();
+  private static final Map<Object, Set<String>> map_gitignore2Regexps = new ConcurrentHashMap<>();
 
-  public boolean isIgnoredFile(@NotNull Object file) {
+  public boolean isDcIgnoredFile(@NotNull Object file) {
     return map_dcignore2Regexps.entrySet().stream()
         .filter(e -> inScope(e.getKey(), file))
         .flatMap(e -> e.getValue().stream())
         .anyMatch(getFilePath(file)::matches);
   }
 
+  public boolean isGitIgnoredFile(@NotNull Object file) {
+    return map_gitignore2Regexps.entrySet().stream()
+        .filter(e -> inScope(e.getKey(), file))
+        .flatMap(e -> e.getValue().stream())
+        .anyMatch(getFilePath(file)::matches);
+  }
+
   protected abstract String getFilePath(@NotNull Object file);
 
-  protected abstract boolean inScope(@NotNull Object dcignoreFile, @NotNull Object fileToCheck);
+  private boolean inScope(@NotNull Object ignoreFile, @NotNull Object fileToCheck) {
+    return getFilePath(fileToCheck).startsWith(getDirPath(ignoreFile));
+  };
 
   public boolean is_ignoreFile(@NotNull Object file) {
-    return is_dcignoreFile(file) || getFileName(file).equals(".gitignore");
+    return is_dcignoreFile(file) || is_gitignoreFile(file);
   }
 
   protected abstract String getFileName(@NotNull Object file);
@@ -39,18 +49,29 @@ public boolean is_dcignoreFile(@NotNull Object file) {
     return getFileName(file).equals(".dcignore");
   }
 
+  public boolean is_gitignoreFile(@NotNull Object file) {
+    return getFileName(file).equals(".gitignore");
+  }
+
   public void remove_dcignoreFileContent(@NotNull Object file) {
     map_dcignore2Regexps.remove(file);
   }
 
+  public void remove_gitignoreFileContent(@NotNull Object file) {
+    map_gitignore2Regexps.remove(file);
+  }
+
   public void update_dcignoreFileContent(@NotNull Object file) {
-    // map_dcignore2Regexps.remove(file);
-    map_dcignore2Regexps.put(file, parse_dcignoreFile2Regexps(file));
+    map_dcignore2Regexps.put(file, parse_ignoreFile2Regexps(file));
+  }
+
+  public void update_gitignoreFileContent(@NotNull Object file) {
+    map_gitignore2Regexps.put(file, parse_ignoreFile2Regexps(file));
   }
 
   protected abstract String getDirPath(@NotNull Object file);
 
-  private Set<String> parse_dcignoreFile2Regexps(@NotNull Object file) {
+  private Set<String> parse_ignoreFile2Regexps(@NotNull Object file) {
     Set<String> result = new HashSet<>();
     String basePath = getDirPath(file);
     String lineSeparator = "[\n\r]";

src/main/java/ai/deepcode/javaclient/core/DeepCodeIgnoreInfoHolderBase.java

@@ -38,6 +38,11 @@ public List<Object> getAllSupportedFilesInProject(@NotNull Object project) {
     allProjectFiles.stream()
         .filter(ignoreInfoHolder::is_dcignoreFile)
         .forEach(ignoreInfoHolder::update_dcignoreFileContent);
+    // Initial scan for .gitignore files
+    allProjectFiles.stream()
+        .filter(ignoreInfoHolder::is_gitignoreFile)
+        .forEach(ignoreInfoHolder::update_gitignoreFileContent);
+
     final List<Object> result =
         allProjectFiles.stream().filter(this::isSupportedFileFormat).collect(Collectors.toList());
     if (result.isEmpty()) dcLogger.logWarn("Empty supported files list for project: " + project);
@@ -46,11 +51,11 @@ public List<Object> getAllSupportedFilesInProject(@NotNull Object project) {
 
   protected abstract Collection<Object> allProjectFiles(@NotNull Object project);
 
-  private static final long MAX_FILE_SIZE = 5242880; // 5MB in bytes
+  private static final long MAX_FILE_SIZE = 4000000; // ~ 4MB in bytes
 
   public boolean isSupportedFileFormat(@NotNull Object file) {
     // DCLogger.getInstance().info("isSupportedFileFormat started for " + psiFile.getName());
-    if (ignoreInfoHolder.isIgnoredFile(file) || isGitIgnored(file)) return false;
+    if (ignoreInfoHolder.isDcIgnoredFile(file) || isGitIgnored(file)) return false;
     final boolean result =
         getFileLength(file) < MAX_FILE_SIZE
             && (supportedExtensions.contains(getFileExtention(file))

src/main/java/ai/deepcode/javaclient/core/DeepCodeUtilsBase.java

@@ -80,12 +80,18 @@ private String doGetHash(@NotNull String fileText) {
     return bytesToHex(encodedHash);
   }
 
+  /**
+   * Look for cached content first, require manual cache invalidation if file been changed
+   */
   @NotNull
-  String getFileContent(@NotNull Object file) {
+  public String getFileContent(@NotNull Object file) {
     // potential OutOfMemoryException for too large projects
     return mapFile2Content.computeIfAbsent(file, this::doGetFileContent);
   }
 
+  /**
+   * Make direct read of File content. NO cache check.
+   */
   @NotNull
   public abstract String doGetFileContent(@NotNull Object file);
 

src/main/java/ai/deepcode/javaclient/core/HashContentUtilsBase.java

@@ -6,6 +6,9 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import java.util.HashSet;
+import java.util.Set;
+
 public abstract class LoginUtilsBase {
 
   private final PlatformDependentUtilsBase pdUtils;
@@ -26,12 +29,21 @@ protected LoginUtilsBase(
 
   protected abstract String getUserAgent();
 
-  private static boolean isLoginCheckLoopStarted = false;
+  private static final Set<Object> PROJECTS_WITH_LOGIN_CHECK_LOOP_STARTED = new HashSet<>();
 
-  /** network request! */
+  /** inner network request! */
   public boolean isLogged(@Nullable Object project, boolean userActionNeeded) {
+    boolean isLogged = checkLogin(project, userActionNeeded);
+    if (isLogged && project != null) {
+      isLogged = checkConsent(project, userActionNeeded);
+    }
+    return isLogged;
+  }
+
+  /** network request! */
+  public boolean checkLogin(@Nullable Object project, boolean userActionNeeded) {
     final String sessionToken = deepCodeParams.getSessionToken();
-    pdUtils.progressCheckCanceled();
+    // pdUtils.progressCheckCanceled();
     final EmptyResponse response = DeepCodeRestApi.checkSession(sessionToken);
     boolean isLogged = response.getStatusCode() == 200;
     String message = response.getStatusDescription();
@@ -41,24 +53,35 @@ public boolean isLogged(@Nullable Object project, boolean userActionNeeded) {
       dcLogger.logWarn("Login check fails: " + message + " Token: " + sessionToken);
     }
     if (!isLogged && userActionNeeded) {
-      if (sessionToken.isEmpty() && response.getStatusCode() == 401) {
-        message = "Authenticate using your GitHub, Bitbucket or GitLab account";
+      if (response.getStatusCode() != 401) {
+        pdUtils.showError(message, project);
       }
-      pdUtils.showLoginLink(project, message);
-    } else if (isLogged && project != null) {
-      if (deepCodeParams.consentGiven(project)) {
-        dcLogger.logInfo("Consent check succeed for: " + pdUtils.getProjectName(project));
+      if (sessionToken.isEmpty()) {
+        pdUtils.showLoginLink(
+            project, "Authenticate using your GitHub, Bitbucket or GitLab account");
       } else {
-        dcLogger.logWarn("Consent check fail! Project: " + pdUtils.getProjectName(project));
-        isLogged = false;
-        pdUtils.showConsentRequest(project, userActionNeeded);
+        pdUtils.showLoginLink(
+            project, "Invalid Token. Correct it or create a new one, please.");
       }
     }
     return isLogged;
   }
 
+  public boolean checkConsent(@NotNull Object project, boolean userActionNeeded) {
+    final boolean consentGiven = deepCodeParams.consentGiven(project);
+    if (consentGiven) {
+      dcLogger.logInfo("Consent check succeed for: " + pdUtils.getProjectName(project));
+    } else {
+      dcLogger.logWarn("Consent check fail! Project: " + pdUtils.getProjectName(project));
+      if (userActionNeeded) {
+        pdUtils.showConsentRequest(project, userActionNeeded);
+      }
+    }
+    return consentGiven;
+  }
+
   /** network request! */
-  public void requestNewLogin(@NotNull Object project, boolean openBrowser) {
+  public void requestNewLogin(@Nullable Object project, boolean openBrowser) {
     dcLogger.logInfo("New Login requested.");
     deepCodeParams.clearLoginParams();
     LoginResponse response = DeepCodeRestApi.newLogin(getUserAgent());
@@ -70,31 +93,36 @@ public void requestNewLogin(@NotNull Object project, boolean openBrowser) {
         pdUtils.showInBrowser(deepCodeParams.getLoginUrl());
         // BrowserUtil.open(DeepCodeParams.getInstance().getLoginUrl());
       }
-      if (!isLoginCheckLoopStarted) {
-        pdUtils.runInBackground(project, () -> startLoginCheckLoop(project));
-        //        ReadAction.nonBlocking(() -> startLoginCheckLoop(project))
-        //            .submit(NonUrgentExecutor.getInstance());
-        dcLogger.logInfo("LoginCheckLoop started");
+      // all projects should be re-scanned
+      for (Object prj : pdUtils.getOpenProjects()) {
+        if (PROJECTS_WITH_LOGIN_CHECK_LOOP_STARTED.add(prj)) {
+          pdUtils.runInBackground(
+              prj,
+              "Waiting Login for " + pdUtils.getProjectName(prj),
+              (progress) -> startLoginCheckLoop(prj, progress));
+          dcLogger.logInfo("LoginCheckLoop started for " + pdUtils.getProjectName(prj));
+        }
       }
     } else {
       dcLogger.logWarn("New Login request fail: " + response.getStatusDescription());
       pdUtils.showError(response.getStatusDescription(), project);
     }
   }
 
-  private void startLoginCheckLoop(@NotNull Object project) {
-    isLoginCheckLoopStarted = true;
+  private void startLoginCheckLoop(@NotNull Object project, @NotNull Object progress) {
     try {
       do {
-        pdUtils.delay(pdUtils.DEFAULT_DELAY);
-      } while (!isLogged(project, false));
+        pdUtils.delay(pdUtils.DEFAULT_DELAY, progress);
+      } while (!checkLogin(project, false));
     } finally {
-      isLoginCheckLoopStarted = false;
+      PROJECTS_WITH_LOGIN_CHECK_LOOP_STARTED.remove(project);
       dcLogger.logInfo("LoginCheckLoop finished for project: " + pdUtils.getProjectName(project));
     }
-    pdUtils.showInfo("Login succeed", project);
+    // pdUtils.showInfo("Login succeed", project);
     analysisData.resetCachesAndTasks(project); // do we need it??
-    pdUtils.doFullRescan(project);
+    if (checkConsent(project, true)) {
+      pdUtils.doFullRescan(project);
+    }
     //    AnalysisData.getInstance().resetCachesAndTasks(project);
     //    RunUtils.asyncAnalyseProjectAndUpdatePanel(project);
   }

src/main/java/ai/deepcode/javaclient/core/LoginUtilsBase.java

@@ -1,13 +1,39 @@
 package ai.deepcode.javaclient.core;
 
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
 public class MyTextRange {
   private final int start;
   private final int end;
+  private final int startRow;
+  private final int endRow;
+  private final int startCol;
+  private final int endCol;
+  //                msg range         poses in source file
+  private final Map<MyTextRange, List<MyTextRange>> markers;
 
-  public MyTextRange(int start, int end) {
+  MyTextRange(
+      int start,
+      int end,
+      int startRow,
+      int endRow,
+      int startCol,
+      int endCol,
+      Map<MyTextRange, List<MyTextRange>> markers) {
 
     this.start = start;
     this.end = end;
+    this.startRow = startRow;
+    this.endRow = endRow;
+    this.startCol = startCol;
+    this.endCol = endCol;
+    this.markers = markers;
+  }
+
+  MyTextRange(int start, int end) {
+    this(start, end, -1, -1, -1, -1, Collections.emptyMap());
   }
 
   public int getStart() {
@@ -17,4 +43,24 @@ public int getStart() {
   public int getEnd() {
     return end;
   }
+
+  public int getStartRow() {
+    return startRow;
+  }
+
+  public int getEndRow() {
+    return endRow;
+  }
+
+  public int getStartCol() {
+    return startCol;
+  }
+
+  public int getEndCol() {
+    return endCol;
+  }
+
+  public Map<MyTextRange, List<MyTextRange>> getMarkers() {
+    return markers;
+  }
 }