feat: add custom walker support

cmd/format/format.go

@@ -107,13 +107,13 @@ func Run(v *viper.Viper, statz *stats.Stats, cmd *cobra.Command, paths []string)
 		cancel()
 	}()
 
-	// parse the walk type
-	walkType, err := walk.TypeString(cfg.Walk)
+	// parse the walk selector
+	walkSelector, err := newWalkSelector(cfg)
 	if err != nil {
 		return fmt.Errorf("invalid walk type: %w", err)
 	}
 
-	if walkType == walk.Stdin && len(paths) != 1 {
+	if walkSelector.IsBuiltin(walk.Stdin) && len(paths) != 1 {
 		// check we have only received one path arg which we use for the file extension / matching to formatters
 		return errors.New("exactly one path should be specified when using the --stdin flag")
 	}
@@ -125,7 +125,7 @@ func Run(v *viper.Viper, statz *stats.Stats, cmd *cobra.Command, paths []string)
 	}
 
 	// create a new walker for traversing the paths
-	walker, err := walk.NewCompositeReader(walkType, cfg.TreeRoot, paths, db, statz)
+	walker, err := walk.NewCompositeReader(walkSelector, cfg.TreeRoot, paths, db, statz)
 	if err != nil {
 		return fmt.Errorf("failed to create walker: %w", err)
 	}
@@ -205,3 +205,21 @@ func Run(v *viper.Viper, statz *stats.Stats, cmd *cobra.Command, paths []string)
 
 	return nil
 }
+
+func newWalkSelector(cfg *config.Config) (walk.Selector, error) {
+	walkType, err := walk.TypeString(cfg.Walk)
+	if err == nil {
+		return walk.BuiltinSelector(walkType), nil
+	}
+
+	walkerCfg, ok := cfg.WalkerConfigs[cfg.Walk]
+	if !ok {
+		return walk.Selector{}, fmt.Errorf("walker %v not found in config", cfg.Walk)
+	}
+
+	return walk.CustomSelector(walk.CustomConfig{
+		Name:    cfg.Walk,
+		Command: walkerCfg.Command,
+		Options: walkerCfg.Options,
+	}), nil
+}

cmd/init/init.toml

@@ -48,9 +48,17 @@
 # verbose = 2
 
 # The method used to traverse the files within the tree root
-# Currently, we support 'auto', 'git', 'jujutsu', or 'filesystem'
+# Built-in values are 'auto', 'git', 'jujutsu', and 'filesystem'
+# You can also set this to the name of a configured custom walker
 # Env $TREEFMT_WALK
 # walk = "filesystem"
+# walk = "mywalker"
+
+# Custom walkers are configured with [walker.<name>]
+# They receive requested file and directory paths as positional args
+# [walker.mywalker]
+# command = "command-to-run"
+# options = []
 
 [formatter.mylanguage]
 # Command to execute

cmd/root_test.go

@@ -1814,6 +1814,65 @@ func TestJujutsu(t *testing.T) {
 	)
 }
 
+func TestCustomWalker(t *testing.T) {
+	tempDir := test.TempExamples(t)
+	configPath := filepath.Join(tempDir, "/treefmt.toml")
+
+	test.ChangeWorkDir(t, tempDir)
+
+	cfg := &config.Config{
+		Walk: "mywalker",
+		WalkerConfigs: map[string]*config.Walker{
+			"mywalker": {
+				Command: "bash",
+				Options: []string{
+					"-c",
+					`if [ "$#" -eq 0 ]; then set -- go haskell; fi
+for path in "$@"; do
+	case "$path" in
+	go) printf '%s\n' go/main.go go/go.mod ;;
+	haskell) printf '%s\n' haskell/Foo.hs ;;
+	*) printf '%s\n' "$path" ;;
+	esac
+done`,
+					"custom-walker",
+				},
+			},
+		},
+		FormatterConfigs: map[string]*config.Formatter{
+			"echo": {
+				Command:  "echo", // will not generate any underlying changes in the file
+				Includes: []string{"*"},
+			},
+		},
+	}
+
+	test.WriteConfig(t, configPath, cfg)
+
+	treefmt(t,
+		withConfig(configPath, cfg),
+		withNoError(t),
+		withStats(t, map[stats.Type]int{
+			stats.Traversed: 3,
+			stats.Matched:   3,
+			stats.Formatted: 3,
+			stats.Changed:   0,
+		}),
+	)
+
+	treefmt(t,
+		withArgs("--clear-cache", "go"),
+		withConfig(configPath, cfg),
+		withNoError(t),
+		withStats(t, map[stats.Type]int{
+			stats.Traversed: 2,
+			stats.Matched:   2,
+			stats.Formatted: 2,
+			stats.Changed:   0,
+		}),
+	)
+}
+
 func TestTreeRootCmd(t *testing.T) {
 	as := require.New(t)
 

config/config.go

@@ -45,6 +45,7 @@ type Config struct {
 	Stdin                 bool     `mapstructure:"stdin"                   toml:"-"` // not allowed in config
 
 	FormatterConfigs map[string]*Formatter `mapstructure:"formatter" toml:"formatter,omitempty"`
+	WalkerConfigs    map[string]*Walker    `mapstructure:"walker"    toml:"walker,omitempty"`
 
 	Global struct {
 		// Deprecated: Use Excludes
@@ -68,6 +69,13 @@ type Formatter struct {
 	NoPositionalArgSupport *bool `mapstructure:"no-positional-arg-support" toml:"no-positional-arg-support"`
 }
 
+type Walker struct {
+	// Command is the command to invoke when walking the tree.
+	Command string `mapstructure:"command" toml:"command"`
+	// Options are an optional list of args to be passed to Command.
+	Options []string `mapstructure:"options,omitempty" toml:"options,omitempty"`
+}
+
 // SetFlags appends our flags to the provided flag set.
 // We have a flag matching most entries in Config, taking care to ensure the name matches the field name defined in the
 // mapstructure tag.
@@ -165,6 +173,7 @@ func NewViper() (*viper.Viper, error) {
 	v.SetEnvPrefix("treefmt")
 	v.AutomaticEnv()
 	v.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_"))
+	v.SetDefault("walk", walk.Auto.String())
 
 	// unset some env variables that we don't want automatically applied
 	if err := os.Unsetenv("TREEFMT_STDIN"); err != nil {
@@ -233,6 +242,36 @@ func FromViper(v *viper.Viper) (*Config, error) {
 		}
 	}
 
+	for name, walkerCfg := range cfg.WalkerConfigs {
+		if !nameRegex.MatchString(name) {
+			return nil, fmt.Errorf(
+				"walker name %q is invalid, must be of the form %s",
+				name, nameRegex.String(),
+			)
+		}
+
+		if _, err := walk.TypeString(name); err == nil {
+			return nil, fmt.Errorf("walker name %q is reserved for a built-in walk type", name)
+		}
+
+		if walkerCfg.Command == "" {
+			return nil, fmt.Errorf("walker %v has no command", name)
+		}
+	}
+
+	if _, err := walk.TypeString(cfg.Walk); err != nil {
+		if !nameRegex.MatchString(cfg.Walk) {
+			return nil, fmt.Errorf(
+				"walk value %q is invalid, must be a built-in walk type or a walker name of the form %s",
+				cfg.Walk, nameRegex.String(),
+			)
+		}
+
+		if _, ok := cfg.WalkerConfigs[cfg.Walk]; !ok {
+			return nil, fmt.Errorf("walker %v not found in config", cfg.Walk)
+		}
+	}
+
 	// filter formatters based on provided names
 	if len(cfg.Formatters) > 0 {
 		filtered := make(map[string]*Formatter)

config/config_test.go

@@ -591,6 +591,117 @@ func TestWalk(t *testing.T) {
 	checkValue("auto")
 }
 
+func TestWalkers(t *testing.T) {
+	t.Run("configured walker", func(t *testing.T) {
+		as := require.New(t)
+
+		cfg := &config.Config{
+			Walk: "mywalker",
+			WalkerConfigs: map[string]*config.Walker{
+				"mywalker": {
+					Command: "command-to-run",
+					Options: []string{
+						"--foo",
+						"bar",
+					},
+				},
+			},
+		}
+
+		v, _ := newViper(t)
+
+		readValue(t, v, cfg, func(cfg *config.Config) {
+			walker, ok := cfg.WalkerConfigs["mywalker"]
+			as.True(ok, "walker not found")
+			as.Equal("mywalker", cfg.Walk)
+			as.Equal("command-to-run", walker.Command)
+			as.Equal([]string{"--foo", "bar"}, walker.Options)
+		})
+	})
+
+	t.Run("missing walker", func(t *testing.T) {
+		as := require.New(t)
+
+		cfg := &config.Config{
+			Walk: "mywalker",
+		}
+
+		v, _ := newViper(t)
+
+		readError(t, v, cfg, func(err error) {
+			as.ErrorContains(err, "walker mywalker not found in config")
+		})
+	})
+
+	t.Run("empty command", func(t *testing.T) {
+		as := require.New(t)
+
+		cfg := &config.Config{
+			Walk: "mywalker",
+			WalkerConfigs: map[string]*config.Walker{
+				"mywalker": {},
+			},
+		}
+
+		v, _ := newViper(t)
+
+		readError(t, v, cfg, func(err error) {
+			as.ErrorContains(err, "walker mywalker has no command")
+		})
+	})
+
+	t.Run("invalid walker name", func(t *testing.T) {
+		as := require.New(t)
+
+		cfg := &config.Config{
+			Walk: "mywalker",
+			WalkerConfigs: map[string]*config.Walker{
+				"my/walker": {
+					Command: "command-to-run",
+				},
+			},
+		}
+
+		v, _ := newViper(t)
+
+		readError(t, v, cfg, func(err error) {
+			as.ErrorContains(err, "walker name \"my/walker\" is invalid")
+		})
+	})
+
+	t.Run("reserved walker name", func(t *testing.T) {
+		as := require.New(t)
+
+		cfg := &config.Config{
+			WalkerConfigs: map[string]*config.Walker{
+				"git": {
+					Command: "command-to-run",
+				},
+			},
+		}
+
+		v, _ := newViper(t)
+
+		readError(t, v, cfg, func(err error) {
+			as.ErrorContains(err, "walker name \"git\" is reserved for a built-in walk type")
+		})
+	})
+
+	t.Run("invalid walk value", func(t *testing.T) {
+		as := require.New(t)
+
+		cfg := &config.Config{
+			Walk: "my.walker",
+		}
+
+		v, _ := newViper(t)
+
+		readError(t, v, cfg, func(err error) {
+			as.ErrorContains(err, "walk value \"my.walker\" is invalid")
+		})
+	})
+}
+
 func TestWorkingDirectory(t *testing.T) {
 	as := require.New(t)
 
@@ -662,7 +773,9 @@ func TestStdin(t *testing.T) {
 func TestSampleConfigFile(t *testing.T) {
 	as := require.New(t)
 
-	v := viper.New()
+	v, err := config.NewViper()
+	as.NoError(err, "failed to create viper")
+
 	v.SetConfigFile("../test/examples/treefmt.toml")
 	as.NoError(v.ReadInConfig(), "failed to read config file")
 

docs/site/getting-started/configure.md

@@ -396,7 +396,8 @@ Set the verbosity level of logs:
 ### `walk`
 
 The method used to traverse the files within the tree root.
-Currently, we support 'auto', 'git', 'jujutsu' or 'filesystem'
+Built-in values are `auto`, `git`, `jujutsu`, and `filesystem`.
+You can also set this to the name of a configured [custom walker](#walker-options).
 
 === "Flag"
 
@@ -416,6 +417,14 @@ Currently, we support 'auto', 'git', 'jujutsu' or 'filesystem'
     walk = "filesystem"
     ```
 
+    ```toml
+    walk = "mywalker"
+
+    [walker.mywalker]
+    command = "command-to-run"
+    options = []
+    ```
+
 ### `working-dir`
 
 Run as if `treefmt` was started in the specified working directory instead of the current working directory.
@@ -433,6 +442,42 @@ Run as if `treefmt` was started in the specified working directory instead of th
     TREEFMT_WORKING_DIR=/tmp/foo treefmt
     ```
 
+## Walker Options
+
+Custom walkers are configured using a [table](https://toml.io/en/v1.0.0#table) entry in `treefmt.toml` of the form
+`[walker.<name>]`.
+To use a custom walker, set the global [`walk`](#walk) option to the same name:
+
+```toml
+walk = "mywalker"
+
+[walker.mywalker]
+command = "command-to-run"
+options = []
+```
+
+### `command`
+
+The command to invoke when walking the tree.
+`treefmt` runs the command from the tree root.
+
+When you pass file or directory paths to `treefmt`, `treefmt` passes those paths to the walker command as positional
+arguments relative to the tree root.
+A custom walker should use those arguments to reduce the paths it emits.
+This improves performance when you format a subtree in a large repository.
+
+`treefmt` still filters the command output to the requested paths.
+
+The command must write path records to `stdout`.
+Records may be separated by newlines or NUL bytes.
+Each path must be relative to the tree root or an absolute path inside the tree root.
+Paths containing newlines are unsupported.
+
+### `options`
+
+An optional list of args to be passed to `command`.
+`treefmt` passes these args before any positional path filters.
+
 ## Formatter Options
 
 Formatters are configured using a [table](https://toml.io/en/v1.0.0#table) entry in `treefmt.toml` of the form