Skip to content

feat: apps init speed improvements #4678

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
MarioCadenas wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: apps init speed improvements #4678

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions cmd/apps/deploy_bundle.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/databricks/cli/bundle/run"
"github.com/databricks/cli/cmd/bundle/utils"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/apps/prompt"
"github.com/databricks/cli/libs/apps/validation"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/log"
Expand Down Expand Up @@ -153,11 +154,11 @@ func runBundleDeploy(cmd *cobra.Command, force, skipValidation, skipTests bool)

log.Infof(ctx, "Running app: %s", appKey)
if err := runBundleApp(ctx, b, appKey); err != nil {
cmdio.LogString(ctx, "✔ Deployment succeeded, but failed to start app")
prompt.PrintDone(ctx, "Deployment succeeded, but failed to start app")
return fmt.Errorf("failed to run app: %w. Run `databricks apps logs` to view logs", err)
}

cmdio.LogString(ctx, "✔ Deployment complete!")
prompt.PrintDone(ctx, "Deployment complete")
return nil
}

Expand Down
6 changes: 4 additions & 2 deletions cmd/apps/import.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/databricks/cli/bundle/run"
bundleutils "github.com/databricks/cli/cmd/bundle/utils"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/apps/prompt"
"github.com/databricks/cli/libs/cmdctx"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/dyn"
Expand Down Expand Up @@ -202,9 +203,10 @@ Examples:
}

if !quiet {
cmdio.LogString(ctx, fmt.Sprintf("\n✓ App '%s' has been successfully imported to %s", name, outputDir))
cmdio.LogString(ctx, "")
prompt.PrintDone(ctx, fmt.Sprintf("App '%s' imported to %s", name, outputDir))
if cleanup && oldSourceCodePath != "" {
cmdio.LogString(ctx, "✓ Previous app folder has been cleaned up")
prompt.PrintDone(ctx, "Previous app folder cleaned up")
}
cmdio.LogString(ctx, "\nYou can now deploy changes with: databricks bundle deploy")
}
Expand Down
248 changes: 203 additions & 45 deletions cmd/apps/init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,14 @@ func promptForPluginsAndDeps(ctx context.Context, m *manifest.Manifest, preSelec
}
theme := prompt.AppkitTheme()

// Eagerly start fetching resources for ALL plugins in the background.
// This runs while the user is selecting plugins, so by the time resource
// pickers appear the data is likely already cached.
allPluginNames := m.GetPluginNames()
allPossibleResources := m.CollectResources(allPluginNames)
allPossibleResources = append(allPossibleResources, m.CollectOptionalResources(allPluginNames)...)
ctx = prompt.PrefetchResources(ctx, allPossibleResources)

// Step 1: Plugin selection (skip if plugins already provided via flag)
selectablePlugins := m.GetSelectablePlugins()
if len(config.Features) == 0 && len(selectablePlugins) > 0 {
Expand Down Expand Up @@ -361,8 +369,11 @@ func promptForPluginsAndDeps(ctx context.Context, m *manifest.Manifest, preSelec
// Always include mandatory plugins.
config.Features = appendUnique(config.Features, m.GetMandatoryPluginNames()...)

// Step 2: Prompt for required plugin resource dependencies
// Collect resources for the user's actual selection.
resources := m.CollectResources(config.Features)
optionalResources := m.CollectOptionalResources(config.Features)

// Step 2: Prompt for required plugin resource dependencies
for _, r := range resources {
values, err := promptForResource(ctx, r, theme, true)
if err != nil {
Expand All @@ -374,7 +385,6 @@ func promptForPluginsAndDeps(ctx context.Context, m *manifest.Manifest, preSelec
}

// Step 3: Prompt for optional plugin resource dependencies
optionalResources := m.CollectOptionalResources(config.Features)
for _, r := range optionalResources {
values, err := promptForResource(ctx, r, theme, false)
if err != nil {
Expand Down Expand Up @@ -490,43 +500,178 @@ func cloneRepo(ctx context.Context, repoURL, branch string) (string, error) {
return tempDir, nil
}

// resolveTemplate resolves a template path, handling both local paths and GitHub URLs.
// branch is used for cloning (can contain "/" for feature branches).
// subdir is an optional subdirectory within the repo to use (for default appkit template).
// Returns the local path to use, a cleanup function (for temp dirs), and any error.
func resolveTemplate(ctx context.Context, templatePath, branch, subdir string) (localPath string, cleanup func(), err error) {
// Case 1: Local path - return as-is
// resolveTemplate resolves a template synchronously with a spinner.
// Used by commands that don't benefit from background cloning (e.g., manifest).
func resolveTemplate(ctx context.Context, templatePath, branch, subdir string) (string, func(), error) {
ch := resolveTemplateAsync(ctx, templatePath, branch, subdir)
return awaitTemplate(ctx, ch)
}

// templateResult holds the outcome of a background template resolution.
type templateResult struct {
path string
cleanup func()
err error
}

// resolveTemplateAsync starts resolving the template in a background goroutine.
// For local paths this completes immediately; for GitHub URLs it clones the repo.
// The caller reads the result from the returned channel, optionally showing a
// spinner if the clone hasn't finished by the time it's needed.
func resolveTemplateAsync(ctx context.Context, templatePath, branch, subdir string) <-chan templateResult {
ch := make(chan templateResult, 1)

// Local path — instant.
if !strings.HasPrefix(templatePath, "https://") {
return templatePath, nil, nil
ch <- templateResult{path: templatePath}
return ch
}

// Case 2: GitHub URL - parse and clone
repoURL, urlSubdir, urlBranch := git.ParseGitHubURL(templatePath)
if branch == "" {
branch = urlBranch // Use branch from URL if not overridden by flag
branch = urlBranch
}
if subdir == "" {
subdir = urlSubdir // Use subdir from URL if not overridden
subdir = urlSubdir
}

// Clone to temp dir with spinner
var tempDir string
err = prompt.RunWithSpinnerCtx(ctx, "Cloning template...", func() error {
var cloneErr error
tempDir, cloneErr = cloneRepo(ctx, repoURL, branch)
return cloneErr
})
go func() {
tempDir, err := cloneRepo(ctx, repoURL, branch)
if err != nil {
ch <- templateResult{err: err}
return
}
cleanup := func() { os.RemoveAll(tempDir) }
localPath := tempDir
if subdir != "" {
localPath = filepath.Join(tempDir, subdir)
}
ch <- templateResult{path: localPath, cleanup: cleanup}
}()

return ch
}

// awaitTemplate waits for the background clone to finish.
// If the result is already available it returns immediately with a
// checkmark; otherwise it shows a spinner while waiting.
func awaitTemplate(ctx context.Context, ch <-chan templateResult) (string, func(), error) {
select {
case res := <-ch:
// Clone finished while the user was typing — print completion.
if res.err == nil && res.cleanup != nil {
prompt.PrintDone(ctx, "Template cloned")
}
return res.path, res.cleanup, res.err
default:
// Still cloning — show a spinner for the remaining wait.
var res templateResult
err := prompt.RunWithSpinnerCtx(ctx, "Cloning template...", func() error {
res = <-ch
return res.err
})
return res.path, res.cleanup, err
}
}

// findProjectSrcDir locates the actual source directory inside a template.
// Templates may nest their content inside a {{.project_name}} directory.
func findProjectSrcDir(templateDir string) string {
entries, err := os.ReadDir(templateDir)
if err != nil {
return "", nil, err
return templateDir
}
for _, e := range entries {
if e.IsDir() && strings.Contains(e.Name(), "{{.project_name}}") {
return filepath.Join(templateDir, e.Name())
}
}
return templateDir
}

cleanup = func() { os.RemoveAll(tempDir) }
// startBackgroundNpmInstall copies the package files from the template into
// destDir and launches `npm ci` in the background. The caller should read the
// returned channel after copyTemplate to get the result. Returns nil if the
// template is not a Node.js project or npm is not available.
func startBackgroundNpmInstall(ctx context.Context, srcProjectDir, destDir, projectName string) <-chan error {
// Check that the template has a package-lock.json (needed by npm ci).
lockFile := filepath.Join(srcProjectDir, "package-lock.json")
if _, err := os.Stat(lockFile); err != nil {
return nil
}

// Return path to subdirectory if specified
if subdir != "" {
return filepath.Join(tempDir, subdir), cleanup, nil
if _, err := exec.LookPath("npm"); err != nil {
return nil
}

if err := os.MkdirAll(destDir, 0o755); err != nil {
return nil
}

// Copy package.json (apply template substitution so the file is valid JSON)
// and package-lock.json (no template vars — copy raw).
for _, name := range []string{"package.json", "package.json.tmpl"} {
src := filepath.Join(srcProjectDir, name)
content, err := os.ReadFile(src)
if err != nil {
continue
}
// Minimal template vars so package.json renders to valid JSON.
minVars := templateData(templateVars{
ProjectName: projectName,
AppDescription: prompt.DefaultAppDescription,
Plugins: make(map[string]*pluginVar),
})
tmpl, err := template.New(name).Option("missingkey=zero").Parse(string(content))
if err != nil {
// Not a Go template — copy raw.
_ = os.WriteFile(filepath.Join(destDir, "package.json"), content, 0o644)
break
}
var buf bytes.Buffer
if err := tmpl.Execute(&buf, minVars); err != nil {
_ = os.WriteFile(filepath.Join(destDir, "package.json"), content, 0o644)
break
}
_ = os.WriteFile(filepath.Join(destDir, "package.json"), buf.Bytes(), 0o644)
break
}

// Copy package-lock.json raw (never has template vars).
if data, err := os.ReadFile(lockFile); err == nil {
_ = os.WriteFile(filepath.Join(destDir, "package-lock.json"), data, 0o644)
}

ch := make(chan error, 1)
go func() {
cmd := exec.CommandContext(ctx, "npm", "ci", "--no-audit", "--no-fund", "--prefer-offline")
cmd.Dir = destDir
cmd.Stdout = nil
cmd.Stderr = nil
ch <- cmd.Run()
}()

log.Debugf(ctx, "Started background npm install in %s", destDir)
return ch
}

// awaitBackgroundNpmInstall waits for the background npm install to complete.
// Shows an instant checkmark if already done, or a spinner for the remainder.
func awaitBackgroundNpmInstall(ctx context.Context, ch <-chan error) error {
select {
case err := <-ch:
if err == nil {
prompt.PrintDone(ctx, "Dependencies installed")
}
return err
default:
var installErr error
err := prompt.RunWithSpinnerCtx(ctx, "Installing dependencies...", func() error {
installErr = <-ch
return installErr
})
return err
}
return tempDir, cleanup, nil
}

func runCreate(ctx context.Context, opts createOptions) error {
Expand Down Expand Up @@ -564,8 +709,16 @@ func runCreate(ctx context.Context, opts createOptions) error {
templateSrc = appkitRepoURL
}

// Step 1: Get project name first (needed before we can check destination)
// Determine output directory for validation
// Start cloning in the background so it runs while the user types the name.
branchForClone := opts.branch
subdirForClone := ""
if usingDefaultTemplate {
branchForClone = gitRef
subdirForClone = appkitTemplateDir
}
templateCh := resolveTemplateAsync(ctx, templateSrc, branchForClone, subdirForClone)

// Step 1: Get project name (clone runs in parallel for remote templates)
destDir := opts.name
if opts.outputDir != "" {
destDir = filepath.Join(opts.outputDir, opts.name)
Expand All @@ -575,19 +728,16 @@ func runCreate(ctx context.Context, opts createOptions) error {
if !isInteractive {
return errors.New("--name is required in non-interactive mode")
}
// Prompt includes validation for name format AND directory existence
name, err := prompt.PromptForProjectName(ctx, opts.outputDir)
if err != nil {
return err
}
opts.name = name
// Update destDir with the actual name
destDir = opts.name
if opts.outputDir != "" {
destDir = filepath.Join(opts.outputDir, opts.name)
}
} else {
// Non-interactive mode: validate name and directory existence
if err := prompt.ValidateProjectName(opts.name); err != nil {
return err
}
Expand All @@ -596,16 +746,8 @@ func runCreate(ctx context.Context, opts createOptions) error {
}
}

// Step 2: Resolve template (handles GitHub URLs by cloning)
// For custom templates, --branch can override the URL's branch
// For default appkit template, pass gitRef directly (supports branches with "/" in name)
branchForClone := opts.branch
subdirForClone := ""
if usingDefaultTemplate {
branchForClone = gitRef
subdirForClone = appkitTemplateDir
}
resolvedPath, cleanup, err := resolveTemplate(ctx, templateSrc, branchForClone, subdirForClone)
// Step 2: Wait for template (may already be done if the user took time typing the name)
resolvedPath, cleanup, err := awaitTemplate(ctx, templateCh)
if err != nil {
return err
}
Expand All @@ -623,6 +765,11 @@ func runCreate(ctx context.Context, opts createOptions) error {
}
}

// Start npm install in the background so it runs while the user answers prompts.
// This is a Node.js-only optimisation — non-Node templates skip this.
srcProjectDir := findProjectSrcDir(templateDir)
npmInstallCh := startBackgroundNpmInstall(ctx, srcProjectDir, destDir, opts.name)

// Step 3: Load manifest from template (optional — templates without it skip plugin/resource logic)
var m *manifest.Manifest
if manifest.HasManifest(templateDir) {
Expand Down Expand Up @@ -737,12 +884,12 @@ func runCreate(ctx context.Context, opts createOptions) error {
}
}

// Track whether we started creating the project for cleanup on failure
// Track whether we started creating the project for cleanup on failure.
// The background npm install may have created destDir early.
var projectCreated bool
var runErr error
defer func() {
if runErr != nil && projectCreated {
// Clean up partially created project on failure
if runErr != nil && (projectCreated || npmInstallCh != nil) {
os.RemoveAll(destDir)
}
}()
Expand Down Expand Up @@ -826,7 +973,17 @@ func runCreate(ctx context.Context, opts createOptions) error {
absOutputDir = destDir
}

// Initialize project based on type (Node.js, Python, etc.)
// Await background npm install (started before prompts to overlap with user interaction).
// If it finishes before this point, the checkmark appears instantly.
if npmInstallCh != nil {
if err := awaitBackgroundNpmInstall(ctx, npmInstallCh); err != nil {
log.Warnf(ctx, "Background npm install failed: %v, will retry during project initialization", err)
}
}

// Initialize project based on type (Node.js, Python, etc.).
// For Node.js, if the background install succeeded node_modules exists
// and the initializer skips the redundant install step.
var nextStepsCmd string
projectInitializer := initializer.GetProjectInitializer(absOutputDir)
if projectInitializer != nil {
Expand Down Expand Up @@ -892,10 +1049,11 @@ func runCreate(ctx context.Context, opts createOptions) error {

if shouldDeploy {
cmdio.LogString(ctx, "")
cmdio.LogString(ctx, "Deploying app...")
if err := runPostCreateDeploy(ctx, profile); err != nil {
cmdio.LogString(ctx, fmt.Sprintf("⚠ Deploy failed: %v", err))
cmdio.LogString(ctx, " You can deploy manually with: databricks apps deploy")
} else {
prompt.PrintDone(ctx, "Deploy complete")
}
}

Expand Down
Loading
Loading