feat: add cloud storage support for session start (#3629)

Author: Panaetius

.pre-commit-config.yaml

@@ -43,6 +43,7 @@ repos:
           - --ignore=D105,D107,D202,D203,D212,D213,D401,D406,D407,D410,D411,D413
         additional_dependencies:
           - toml
+        exclude: ^renku/ui/service/views/
   - repo: https://github.com/koalaman/shellcheck-precommit
     rev: v0.8.0
     hooks:

docs/_static/cheatsheet/cheatsheet.json

@@ -441,7 +441,7 @@
           ]
         },
         {
-          "command": "$ renku storage pull <path>...",
+          "command": "$ renku lfs pull <path>...",
           "description": "Pull <path>'s from external storage (LFS).",
           "target": [
             "rp"

docs/_static/cheatsheet/cheatsheet.pdf

@@ -1,2 +1,2 @@
-5316163d742bdb6792ed8bcb35031f6c  cheatsheet.tex
+b8a4fc75c7ba023773b0ccc2e98ebc02  cheatsheet.tex
 c70c179e07f04186ec05497564165f11  sdsc_cheatsheet.cls

docs/cheatsheet_hash

@@ -1 +1 @@
-1856fb451165d013777c7c4cdd56e575  cheatsheet.json
+171f230e9ec6372e52129df1bfcf485a  cheatsheet.json

docs/cheatsheet_json_hash

@@ -3,4 +3,4 @@
 renku storage
 *************
 
-.. automodule:: renku.ui.cli.storage
+.. automodule:: renku.ui.cli.lfs

docs/reference/commands/storage.rst

@@ -241,7 +241,7 @@ pattern = """(?x)                                          (?# ignore whitespace
 """
 
 [tool.pytest.ini_options]
-addopts = "--doctest-glob=\"*.rst\" --doctest-modules --cov --cov-report=term-missing --ignore=docs/cheatsheet/ --tb=line"
+addopts = "--doctest-glob=\"*.rst\" --doctest-modules --cov --cov-report=term-missing --ignore=docs/cheatsheet/ -ra"
 doctest_optionflags = "ALLOW_UNICODE"
 flake8-ignore = ["*.py", "E121", "E126", "E203", "E226", "E231", "W503", "W504", "docs/conf.py", "docs/cheatsheet/conf.py", "ALL"]
 flake8-max-line-length = 120

pyproject.toml

@@ -16,7 +16,7 @@
 """Check for large files in Git history."""
 
 from renku.command.util import WARNING
-from renku.core.storage import check_external_storage, check_lfs_migrate_info
+from renku.core.lfs import check_external_storage, check_lfs_migrate_info
 
 
 def check_lfs_info(**_):

renku/command/checks/storage.py

@@ -17,6 +17,7 @@
 
 import contextlib
 import functools
+import shutil
 import threading
 from collections import defaultdict
 from pathlib import Path
@@ -455,6 +456,13 @@ def require_clean(self) -> "Command":
 
         return RequireClean(self)
 
+    @check_finalized
+    def require_login(self) -> "Command":
+        """Check that the user is logged in."""
+        from renku.command.command_builder.repo import RequireLogin
+
+        return RequireLogin(self)
+
     @check_finalized
     def with_communicator(self, communicator: CommunicationCallback) -> "Command":
         """Create a communicator.
@@ -479,6 +487,20 @@ def with_database(self, write: bool = False, path: Optional[str] = None, create:
 
         return DatabaseCommand(self, write, path, create)
 
+    @check_finalized
+    def with_gitlab_api(self) -> "Command":
+        """Inject gitlab api client."""
+        from renku.command.command_builder.gitlab import GitlabApiCommand
+
+        return GitlabApiCommand(self)
+
+    @check_finalized
+    def with_storage_api(self) -> "Command":
+        """Inject storage api client."""
+        from renku.command.command_builder.storage import StorageApiCommand
+
+        return StorageApiCommand(self)
+
 
 class CommandResult:
     """The result of a command.
@@ -496,3 +518,26 @@ def __init__(self, output, error, status) -> None:
         self.output = output
         self.error = error
         self.status = status
+
+
+class RequireExecutable(Command):
+    """Builder to check if an executable is installed."""
+
+    HOOK_ORDER = 4
+
+    def __init__(self, builder: Command, executable: str) -> None:
+        """__init__ of RequireExecutable."""
+        self._builder = builder
+        self._executable = executable
+
+    def _pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
+        """Check if an executable exists on the system.
+
+        Args:
+            builder(Command): Current ``CommandBuilder``.
+            context(dict): Current context.
+        """
+        if not shutil.which(self._executable):
+            raise errors.ExecutableNotFound(
+                f"Couldn't find the executable '{self._executable}' on this system. Please make sure it's installed"
+            )

renku/command/command_builder/command.py

@@ -0,0 +1,54 @@
+# Copyright Swiss Data Science Center (SDSC). A partnership between
+# École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Command builder for gitlab api."""
+
+
+from renku.command.command_builder.command import Command, check_finalized
+from renku.core.interface.git_api_provider import IGitAPIProvider
+from renku.domain_model.project_context import project_context
+from renku.infrastructure.gitlab_api_provider import GitlabAPIProvider
+
+
+class GitlabApiCommand(Command):
+    """Builder to get a gitlab api client."""
+
+    PRE_ORDER = 4
+
+    def __init__(self, builder: Command) -> None:
+        self._builder = builder
+
+    def _injection_pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
+        """Create a gitlab api provider."""
+
+        if not project_context.has_context():
+            raise ValueError("Gitlab API builder needs a ProjectContext to be set.")
+
+        def _get_provider():
+            from renku.core.login import read_renku_token
+
+            token = read_renku_token(None, True)
+            if not token:
+                return None
+            return GitlabAPIProvider(token=token)
+
+        context["constructor_bindings"][IGitAPIProvider] = _get_provider
+
+    @check_finalized
+    def build(self) -> Command:
+        """Build the command."""
+        self._builder.add_injection_pre_hook(self.PRE_ORDER, self._injection_pre_hook)
+
+        return self._builder.build()