ADDED: first functional version with clone functionality

Author: AleixMT

.run/main.run.xml

@@ -14,7 +14,7 @@
     <option name="ADD_CONTENT_ROOTS" value="true" />
     <option name="ADD_SOURCE_ROOTS" value="true" />
     <option name="SCRIPT_NAME" value="$PROJECT_DIR$/src/main.py" />
-    <option name="PARAMETERS" value="AleixMT" />
+    <option name="PARAMETERS" value="AleixMT --exclude-gitlab" />
     <option name="SHOW_COMMAND_LINE" value="false" />
     <option name="EMULATE_TERMINAL" value="false" />
     <option name="MODULE_MODE" value="false" />

requirements.txt

@@ -1,2 +1,3 @@
 PyGithub
-argparse
+argparse
+gitpython

src/defines/ProviderType.py

@@ -5,4 +5,4 @@
 
 class ProviderType(Enum):
     GITHUB = 1
-    GITLAB = 2
+    GITLAB = 2

src/demos.py

@@ -3,31 +3,31 @@
 
 from src.service.GitHubService import GitHubService
 from src.service.ProviderService import ProviderService
-from src.service.TokenService import get_github_token
+from src.service.TokenService import get_github_official_token
 
 
 def get_my_gh_organization_names():
-    gh: ProviderService = GitHubService(get_github_token())
+    gh: ProviderService = GitHubService(get_github_official_token())
     print(gh.get_user_organization_names("AleixMT"))
 
 
 def get_my_gh_owned_repo_names():
-    gh: ProviderService = GitHubService(get_github_token())
+    gh: ProviderService = GitHubService(get_github_official_token())
     print(gh.get_user_owned_repo_names("AleixMT"))
 
 
 def get_my_gh_collaboration_repo_names():
-    gh: ProviderService = GitHubService(get_github_token())
+    gh: ProviderService = GitHubService(get_github_official_token())
     print(gh.get_user_collaboration_repo_names("AleixMT"))
 
 
 def get_my_gh_organization_repo_names():
-    gh: ProviderService = GitHubService(get_github_token())
+    gh: ProviderService = GitHubService(get_github_official_token())
     print(gh.get_organization_repo_names("AleixMT"))
 
 
 def get_my_gh_organizations_repo_names():
-    gh: ProviderService = GitHubService(get_github_token())
+    gh: ProviderService = GitHubService(get_github_official_token())
     for org_name in gh.get_user_organization_names("AleixMT"):
         print("GH organization name: " + org_name)
         print("Repositories:")

src/main.py

@@ -1,14 +1,13 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-from pathlib import Path
 
 from src.defines.FlattenLevel import FlattenLevel
 from src.defines.RenameStrategy import RenameStrategy
 from src.model.Repository import Repository
 from src.service.GitHubService import GitHubService, build_github_official_provider
-from src.service.ProviderService import ProviderService, build_provider
+from src.service.ProviderService import ProviderService, build_provider, build_custom_provider
 from src.service.RepositoryService import compute_path
-from src.service.TokenService import get_github_token
+from src.service.TokenService import get_github_official_token, get_custom_provider_token, get_gitlab_official_token
 from src.defines.ProviderType import ProviderType
 from src.service.GitLabService import build_gitlab_official_provider, GitLabService
 from src.service.ArgumentParserService import build_argument_parser, parse_arguments
@@ -36,64 +35,65 @@ def build_model(args):
     if args.custom_providers:
         for custom_provider in args.custom_providers:
             if args.exclude_github:
-                providers.append(build_provider(custom_provider, ProviderType.GITLAB))
+                providers.append(build_custom_provider(ProviderType.GITLAB, custom_provider))
             if args.exclude_gitlab:
-                providers.append(build_provider(custom_provider, ProviderType.GITHUB))
+                providers.append(build_custom_provider(ProviderType.GITHUB, custom_provider))
             if not args.exclude_gitlab and not args.exclude_github:
-                providers.append(build_provider(custom_provider, ProviderType.GITLAB))
-                providers.append(build_provider(custom_provider, ProviderType.GITHUB))
+                providers.append(build_custom_provider(ProviderType.GITLAB, custom_provider))
+                providers.append(build_custom_provider(ProviderType.GITHUB, custom_provider))
 
     model = {}
     for username in args.usernames:
         for provider in providers:
             provider_service = None
-            if provider.provider == ProviderType.GITLAB:
+            if provider.provider is ProviderType.GITLAB:
                 provider_service = GitLabService(provider.token, provider.url)
-            elif provider.provider == ProviderType.GITHUB:
+            elif provider.provider is ProviderType.GITHUB:
                 provider_service = GitHubService(provider.token, provider.url)
 
             organizations = [username]
-            organizations.extend(provider_service.get_user_organization_names(username))
+            names = provider_service.get_user_organization_names(username)
+            organizations.extend(names)
             for organization in organizations:
                 repos = provider_service.get_organization_repo_names(organization)
                 for repo in repos:
                     new_repo = Repository(args.backup_name, username, provider, organization, repo,
                                provider.url + "/" + organization + "/" + repo)
-                    computed_path = compute_path(repo, FlattenLevel.ROOT.name in args.flatten_directories,
+                    compute_path(new_repo, FlattenLevel.ROOT.name in args.flatten_directories,
                                                  FlattenLevel.USER.name in args.flatten_directories,
                                                  FlattenLevel.PROVIDER.name in args.flatten_directories,
                                                  FlattenLevel.ORGANIZATION.name in args.flatten_directories)
-                    new_repo.path = computed_path
+                    computed_path = new_repo.path
                     if computed_path not in model:
-                        model[computed_path] = [new_repo]
+                        model[computed_path] = new_repo
                     else:
                         if args.rename_strategy == RenameStrategy.IGNORE:
                             pass
                         elif args.rename_strategy == RenameStrategy.SHORTEST:
                             new_repo = Repository(args.backup_name, username, provider, organization, repo,
-                                       provider.url + "/" + organization + "/" + repo)
+                                       provider.url + "/" + organization + "/" + repo, computed_path)
                             repo_name_level = FlattenLevel.USER
                             while computed_path in model:
                                 repo_name_level = FlattenLevel(repo_name_level.value - 1)
-                                computed_path = compute_path(new_repo, FlattenLevel.ROOT.name in args.flatten_directories,
+                                compute_path(new_repo, FlattenLevel.ROOT.name in args.flatten_directories,
                                                               FlattenLevel.USER.name in args.flatten_directories,
                                                               FlattenLevel.PROVIDER.name in args.flatten_directories,
                                                               FlattenLevel.ORGANIZATION.name in args.flatten_directories,
                                                               FlattenLevel(repo_name_level.value - 1))
-                            new_repo.path = computed_path
-                            model[computed_path] = [new_repo]
+                            computed_path = new_repo.path
+                            model[computed_path] = new_repo
                         elif args.rename_strategy == RenameStrategy.SYSTEMATIC:
                             repo1 = model.pop(computed_path)
-                            repo1.path = compute_path(repo1, FlattenLevel.ROOT.name in args.flatten_directories,
+                            compute_path(repo1, FlattenLevel.ROOT.name in args.flatten_directories,
                                                               FlattenLevel.USER.name in args.flatten_directories,
                                                               FlattenLevel.PROVIDER.name in args.flatten_directories,
                                                               FlattenLevel.ORGANIZATION.name in args.flatten_directories,
                                                               FlattenLevel.ROOT)
-                            model[repo1.path] = [repo1]
+                            model[repo1.path] = repo1
 
                             new_repo = Repository(args.backup_name, username, provider, organization, repo,
                                        provider.url + "/" + organization + "/" + repo)
-                            new_repo.path = computed_path(repo, FlattenLevel.ROOT.name in args.flatten_directories,
+                            compute_path(new_repo, FlattenLevel.ROOT.name in args.flatten_directories,
                                           FlattenLevel.USER.name in args.flatten_directories,
                                           FlattenLevel.PROVIDER.name in args.flatten_directories,
                                           FlattenLevel.ORGANIZATION.name in args.flatten_directories,
@@ -103,20 +103,36 @@ def build_model(args):
                             pass
                             # TODO
 
-    print(model)
+    return model
+
+
+def clone_repos(model, backup_folder):
+    for key, value in model.items():
+        provider_service = None
+        if value.provider.provider is ProviderType.GITLAB:
+            provider_service = GitLabService(value.provider.token, value.provider.url)
+        elif value.provider.provider is ProviderType.GITHUB:
+            provider_service = GitHubService(value.provider.token, value.provider.url)
+        print(value.link + "   " + backup_folder + "/" + key.__str__())
+        provider_service.clone_repo(value.link, backup_folder / key)
+
 
 def main():
     parser = build_argument_parser()
     args = parse_arguments(parser)
     if args.is_verbose:
         print_summary(args)
     model = build_model(args)
+    clone_repos(model, args.backup_folder)
 
 
 if __name__ == "__main__":
     # Generate same date string for all entities
+    import sys
+    import os
+    sys.path.append(os.path.abspath(os.path.dirname(__file__)))
 
-    gh: ProviderService = GitHubService(get_github_token())
-    print(gh.get_user_collaboration_repo_names("AleixMT"))
+    #gh: ProviderService = GitHubService(get_github_official_token())
+    #print(gh.get_user_organization_names("AleixMT"))
 
     main()

src/model/Provider.py

@@ -6,9 +6,9 @@
 
 
 class Provider:
-    def __init__(self, url: str, provider: ProviderType, token: str):
-        self.url = url
+    def __init__(self, provider: ProviderType, url: str, token: str):
         self.provider = provider
+        self.url = url
         self.token = token
 
 

src/service/ArgumentParserService.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 import os
+import re
 from datetime import datetime
 
-from src.service.FileService import is_file_directory_writable, is_file_writable
+from src.service.IOService import is_file_directory_writable, is_file_writable
 import argparse
 
 from src.defines.CollisionAction import CollisionAction
@@ -180,7 +181,7 @@ def parse_arguments(parser: argparse.ArgumentParser):
 
     # Supply default backup directory
     if not args.backup_folder:
-        args.backup_folder = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "backup")
+        args.backup_folder = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "backup")
 
     # Check existence and access of backup directory
     if not os.path.exists(args.backup_folder):
@@ -215,6 +216,10 @@ def parse_arguments(parser: argparse.ArgumentParser):
             and not args.rename_strategy:
         args.rename_strategy = RenameStrategy.SHORTEST_SYSTEMATIC
 
+    # Default to empty list
+    if not args.flatten_directories:
+        args.flatten_directories = []
+
     if not args.rename_strategy:
         args.rename_strategy = RenameStrategy.SHORTEST_SYSTEMATIC
 
@@ -263,3 +268,23 @@ def parse_arguments(parser: argparse.ArgumentParser):
             if args.exclude_gitlab:
                 custom_providers.append(build_provider(custom_provider,  ProviderType.GITHUB))
     return args
+
+
+def infer_name(input_string):
+    # Regular expressions
+    ip_regex = r'^((https?://)?((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))$'
+    dns_regex = r'^((https?://)?([a-zA-Z0-9-]+\.)*[a-zA-Z0-9-]+)$'
+
+    # Check if the string is an IP or an IP with "https://"
+    if re.match(ip_regex, input_string):
+        # Extract the IP address without "https://"
+        ip_match = re.search(r'((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)', input_string)
+        return ip_match.group()
+
+    # Check if the string is a DNS name (including single-word hostnames)
+    elif re.match(dns_regex, input_string):
+        # Extract the DNS name without "https://"
+        dns_match = re.search(r'([a-zA-Z0-9-]+\.)*[a-zA-Z0-9-]+$', input_string)
+        return dns_match.group()
+
+    raise ValueError("Invalid Input: " + input_string)

src/service/FileService.py

@@ -1,18 +1,25 @@
 from typing import Optional, List
 
+from src.service.ArgumentParserService import infer_name
 from src.service.ProviderService import ProviderService, build_provider
 from github import Github
 from github import Auth
+from git import Repo  # TODO separate in its own file
+
 
 from src.defines.ProviderType import ProviderType
+from src.service.TokenService import get_github_official_token
 
 
 class GitHubService(ProviderService):
     """Service for interacting with GitHub."""
 
     def __init__(self, access_token, url: Optional[str] = None):
         if url:
-            self.g = Github(base_url=url, auth=Auth.Token(access_token))
+            if infer_name(url).__eq__("github.com"):
+                self.g = Github(auth=Auth.Token(access_token))
+            else:
+                self.g = Github(base_url=url, auth=Auth.Token(access_token))
         else:
             self.g = Github(auth=Auth.Token(access_token))
 
@@ -56,6 +63,9 @@ def get_organization_repos(self, organization):
     def get_organization_repo_names(self, organization) -> List[str]:
         return [repo.name for repo in self.get_user_owned_repos(organization)]
 
+    def clone_repo(self, url, path):
+        Repo.clone_from(url, path)
+
 
 def build_github_official_provider():
-    return build_provider('https://github.com/', ProviderType.GITHUB)
+    return build_provider(ProviderType.GITHUB, 'https://github.com', get_github_official_token())

src/service/GitHubService.py

@@ -2,6 +2,7 @@
 
 from src.service.ProviderService import ProviderService, build_provider
 from src.defines.ProviderType import ProviderType
+from src.service.TokenService import get_gitlab_official_token
 
 
 class GitLabService(ProviderService):
@@ -30,4 +31,4 @@ def get_organization_repo_names(self, organization) -> List[str]:
 
 
 def build_gitlab_official_provider():
-    return build_provider('https://gitlab.com/', ProviderType.GITLAB)
+    return build_provider(ProviderType.GITLAB, 'https://gitlab.com/', get_gitlab_official_token())