diff --git a/go.mod b/go.mod index 231b838e..524e07ec 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,8 @@ require ( github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea go.starlark.net v0.0.0-20200901195727-6e684ef5eeee go.uber.org/zap v1.10.0 + golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 + golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect golang.org/x/sys v0.0.0-20210324051608-47abb6519492 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e diff --git a/notes/06-sandboxing.md b/notes/06-sandboxing.md index 7ad837b9..ec3b56e9 100644 --- a/notes/06-sandboxing.md +++ b/notes/06-sandboxing.md @@ -31,3 +31,7 @@ Support docker as an option, would work only for linux or just for running thing - Default mounts: https://github.com/opencontainers/runtime-tools/blob/a7974a4078764ec41acf5feaa05f07854af44aa6/generate/generate.go#L174-L211 - Create dev/null https://www.commandlinefu.com/commands/view/24199/create-devnull-if-accidentally-deleted-or-for-a-chroot - container linux with mknod examples https://github.com/cloudify-incubator/cloudify-rest-go-client/blob/f8139d8e38b0909fae3e4212eb05497483c0e5b8/container/container_linux.go + +## Darwin + +- https://github.com/LnL7/nix-darwin/blob/1464d9efd3930dafecb45668e6c58349041ea830/modules/security/sandbox/default.nix diff --git a/pkg/sandbox/chrootenv.go b/pkg/sandbox/chrootenv_linux.go similarity index 90% rename from pkg/sandbox/chrootenv.go rename to pkg/sandbox/chrootenv_linux.go index d65b3272..ade4f896 100644 --- a/pkg/sandbox/chrootenv.go +++ b/pkg/sandbox/chrootenv_linux.go @@ -1,8 +1,11 @@ +// +build linux + package sandbox import ( "fmt" "os" + "os/exec" "path/filepath" "strings" "syscall" @@ -144,3 +147,22 @@ func (chr *chroot) Cleanup() (err error) { } return nil } + +func (s Sandbox) runExecStep() { + cmd := exec.Cmd{ + Path: s.Path, + Dir: s.Dir, + Args: append([]string{s.Path}, s.Args...), + Env: os.Environ(), + + // We don't use the passed sandbox stdio because + // it's been passed to the very first run command + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + } + if err := cmd.Run(); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } +} diff --git a/pkg/sandbox/sandbox.go b/pkg/sandbox/sandbox.go index 119d7537..8178fe58 100644 --- a/pkg/sandbox/sandbox.go +++ b/pkg/sandbox/sandbox.go @@ -6,35 +6,17 @@ import ( "encoding/json" "fmt" "io" - "log" "os" - "os/exec" "os/signal" - "path/filepath" "syscall" - "github.com/creack/pty" - "github.com/docker/docker/pkg/term" - "github.com/maxmcd/bramble/pkg/logger" "github.com/pkg/errors" - "golang.org/x/sys/unix" ) const ( - newNamespaceStepArg = "newNamespace" - setupStepArg = "setup" - execStepArg = "exec" - setUIDExecName = "bramble-setuid" + execStepArg = "exec" ) -func firstArgMatchesStep() bool { - switch os.Args[0] { - case newNamespaceStepArg, setupStepArg, execStepArg: - return true - } - return false -} - // Entrypoint must be run at the beginning of your executable. When the sandbox // runs it re-runs the same binary with various arguments to indicate that we // want the process to be run as a sandbox. If this function detects that it @@ -51,27 +33,6 @@ func Entrypoint() { os.Exit(0) } -func entrypoint() (err error) { - if len(os.Args) <= 1 { - return errors.New("unexpected argument count for sandbox step") - } - s, err := parseSerializedArg(os.Args[1]) - if err != nil { - return err - } - switch os.Args[0] { - case newNamespaceStepArg: - return s.newNamespaceStep() - case setupStepArg: - return s.setupStep() - case execStepArg: - s.runExecStep() - return nil - default: - return errors.New("first argument didn't match any known sandbox steps") - } -} - // Sandbox defines a command or function that you want to run in a sandbox type Sandbox struct { Stdin io.Reader `json:"-"` @@ -108,30 +69,11 @@ func parseSerializedArg(arg string) (s Sandbox, err error) { // Run runs the sandbox until execution has been completed func (s Sandbox) Run(ctx context.Context) (err error) { - if term.IsTerminal(os.Stdin.Fd()) { - logger.Debug("is terminal") - } - - serialized, err := s.serializeArg() + cmd, err := s.runCommand() if err != nil { return err } - // TODO: allow reference to self - path, err := exec.LookPath(setUIDExecName) - if err != nil { - return err - } - logger.Debugw("newSanbox", "execpath", path) - // interrupt will be caught be the child process and the process - // will exiting, causing this process to exit ignoreInterrupt() - cmd := &exec.Cmd{ - Path: path, - Args: []string{newNamespaceStepArg, serialized}, - Stdin: s.Stdin, - Stdout: s.Stdout, - Stderr: s.Stderr, - } errChan := make(chan error) go func() { if err := cmd.Run(); err != nil { @@ -156,141 +98,6 @@ func (s Sandbox) Run(ctx context.Context) (err error) { return nil // Start the command with a pty. } -func (s Sandbox) newNamespaceStep() (err error) { - selfExe, err := os.Readlink("/proc/self/exe") - if err != nil { - return err - } - defer func() { - logger.Debugw("clean up chrootDir", "path", s.ChrootPath) - if er := os.RemoveAll(s.ChrootPath); er != nil { - logger.Debugw("error cleaning up", "err", er) - if err == nil { - err = errors.Wrap(er, "error removing all files in "+s.ChrootPath) - } - } - }() - serialized, err := s.serializeArg() - if err != nil { - return err - } - - var cloneFlags uintptr = syscall.CLONE_NEWUTS | - syscall.CLONE_NEWNS | - syscall.CLONE_NEWPID - - if s.DisableNetwork { - cloneFlags |= syscall.CLONE_NEWNET - } - - // interrupt will be caught be the child process and the process - // will exiting, causing this process to exit - ignoreInterrupt() - - cmd := &exec.Cmd{ - Path: selfExe, - Args: []string{setupStepArg, serialized}, - SysProcAttr: &syscall.SysProcAttr{ - // maybe sigint will allow the child more time to clean up its mounts???? - Pdeathsig: unix.SIGINT, - Cloneflags: cloneFlags, - }, - } - - // We must use a pty here to enable interactive input. If we naively pass - // os.Stdin to an exec.Cmd then we run into issues with the parent and - // child terminals getting confused about who is supposed to process various - // control signals. - // We can then just set to raw and copy the bytes across. We could remove - // the pty entirely for jobs that don't pass a terminal as a stdin. - ptmx, err := pty.Start(cmd) - if err != nil { - return errors.Wrap(err, "error starting pty") - } - defer func() { _ = ptmx.Close() }() - - // only handle stdin and set raw if it's an interactive terminal - if os.Stdin != nil && term.IsTerminal(os.Stdin.Fd()) { - // Handle pty resize - ch := make(chan os.Signal, 1) - signal.Notify(ch, syscall.SIGWINCH) - go func() { - for range ch { - if err := pty.InheritSize(os.Stdin, ptmx); err != nil { - log.Printf("error resizing pty: %s", err) - } - } - }() - ch <- syscall.SIGWINCH // Initial resize. - oldState, err := term.MakeRaw(os.Stdin.Fd()) - if err != nil { - return err - } - // restore when complete - defer func() { _ = term.RestoreTerminal(os.Stdin.Fd(), oldState) }() - go func() { _, _ = io.Copy(ptmx, os.Stdin) }() - } - _, _ = io.Copy(os.Stdout, ptmx) - return errors.Wrap(cmd.Wait(), "error running newNamespace") -} - -func (s Sandbox) setupStep() (err error) { - logger.Debugw("setup chroot", "dir", s.ChrootPath) - creds := &syscall.Credential{ - Gid: uint32(s.GroupID), - Uid: uint32(s.UserID), - } - if err := os.Chown(s.ChrootPath, int(creds.Uid), int(creds.Gid)); err != nil { - return err - } - - chr := newChroot(s.ChrootPath, s.Mounts) - defer func() { - if er := chr.Cleanup(); er != nil { - if err == nil { - err = er - } else { - logger.Debugw("error during cleanup", "err", er) - } - } - }() - var selfExe string - { - // hardlink in executable - selfExe, err = os.Readlink("/proc/self/exe") - if err != nil { - return err - } - if err := os.MkdirAll(filepath.Join(s.ChrootPath, filepath.Dir(selfExe)), 0777); err != nil { - return err - } - if err = os.Link(selfExe, filepath.Join(s.ChrootPath, selfExe)); err != nil { - return err - } - } - - if err := chr.Init(); err != nil { - return err - } - - serialized, err := s.serializeArg() - if err != nil { - return err - } - - cmd := exec.CommandContext(interruptContext(), selfExe) - cmd.Path = selfExe - cmd.Args = []string{execStepArg, serialized} - cmd.Env = append([]string{"USER=bramblebuild0", "HOME=/homeless"}, s.Env...) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.SysProcAttr = &syscall.SysProcAttr{ - Credential: creds, - } - return cmd.Run() -} - func ignoreInterrupt() { c := make(chan os.Signal) signal.Notify(c, os.Interrupt, syscall.SIGTERM) @@ -311,22 +118,3 @@ func interruptContext() context.Context { }() return ctx } - -func (s Sandbox) runExecStep() { - cmd := exec.Cmd{ - Path: s.Path, - Dir: s.Dir, - Args: append([]string{s.Path}, s.Args...), - Env: os.Environ(), - - // We don't use the passed sandbox stdio because - // it's been passed to the very first run command - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, - } - if err := cmd.Run(); err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } -} diff --git a/pkg/sandbox/sandbox_darwin.go b/pkg/sandbox/sandbox_darwin.go new file mode 100644 index 00000000..1db6a3f7 --- /dev/null +++ b/pkg/sandbox/sandbox_darwin.go @@ -0,0 +1,32 @@ +// +build darwin + +package sandbox + +import ( + "os/exec" +) + +func (s Sandbox) runCommand() (*exec.Cmd, error) { + profile := `(version 1) + (deny default) + (allow process-exec*) + (import "/System/Library/Sandbox/Profiles/bsd.sb")` + if !s.DisableNetwork { + profile += `(allow network*)` + } + cmd := exec.Command("sandbox-exec", "-p", profile, s.Path) + cmd.Args = append(cmd.Args, s.Args...) + cmd.Dir = s.Dir + cmd.Env = s.Env + cmd.Stderr = s.Stderr + cmd.Stdout = s.Stdout + cmd.Stdin = s.Stdin + return cmd, nil +} + +func firstArgMatchesStep() bool { + return false +} +func entrypoint() error { + return nil +} diff --git a/pkg/sandbox/sandbox_darwin_test.go b/pkg/sandbox/sandbox_darwin_test.go new file mode 100644 index 00000000..877a8fa6 --- /dev/null +++ b/pkg/sandbox/sandbox_darwin_test.go @@ -0,0 +1,55 @@ +// +build darwin + +package sandbox + +import ( + "bytes" + "context" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSandbox_runCommand(t *testing.T) { + t.Run("no network", func(t *testing.T) { + var buf bytes.Buffer + sbx := Sandbox{ + Stdout: os.Stdout, + Stderr: &buf, + DisableNetwork: true, + Path: "/sbin/ping", + Args: []string{"-c1", "google.com"}, + Dir: "/Users/maxm/go/src/github.com/maxmcd/bramble", //TODO + } + require.Error(t, sbx.Run(context.Background())) + assert.Contains(t, buf.String(), "Unknown host") + }) + t.Run("yes network", func(t *testing.T) { + var buf bytes.Buffer + sbx := Sandbox{ + Stdout: &buf, + Stderr: os.Stderr, + DisableNetwork: false, + Path: "/sbin/ping", + Args: []string{"-c1", "google.com"}, + Dir: "/Users/maxm/go/src/github.com/maxmcd/bramble", //TODO + } + require.NoError(t, sbx.Run(context.Background())) + assert.Contains(t, buf.String(), "0.0% packet loss") + }) + t.Run("hllo wrld", func(t *testing.T) { + var buf bytes.Buffer + sbx := Sandbox{ + Stdout: &buf, + Stderr: os.Stderr, + DisableNetwork: true, + Path: "/bin/echo", + Args: []string{"hi"}, + Dir: "/Users/maxm/go/src/github.com/maxmcd/bramble", //TODO + } + require.NoError(t, sbx.Run(context.Background())) + assert.Equal(t, buf.String(), "hi\n") + }) +} diff --git a/pkg/sandbox/sandbox_linux.go b/pkg/sandbox/sandbox_linux.go new file mode 100644 index 00000000..3b953b3c --- /dev/null +++ b/pkg/sandbox/sandbox_linux.go @@ -0,0 +1,212 @@ +// +build linux + +package sandbox + +import ( + "io" + "log" + "os" + "os/exec" + "os/signal" + "path/filepath" + "syscall" + + "github.com/creack/pty" + "github.com/maxmcd/bramble/pkg/logger" + "github.com/pkg/errors" + "golang.org/x/crypto/ssh/terminal" + "golang.org/x/sys/unix" +) + +const ( + newNamespaceStepArg = "newNamespace" + setupStepArg = "setup" + setUIDExecName = "bramble-setuid" +) + +func firstArgMatchesStep() bool { + switch os.Args[0] { + case newNamespaceStepArg, setupStepArg, execStepArg: + return true + } + return false +} + +func entrypoint() (err error) { + if len(os.Args) <= 1 { + return errors.New("unexpected argument count for sandbox step") + } + s, err := parseSerializedArg(os.Args[1]) + if err != nil { + return err + } + switch os.Args[0] { + case newNamespaceStepArg: + return s.newNamespaceStep() + case setupStepArg: + return s.setupStep() + case execStepArg: + s.runExecStep() + return nil + default: + return errors.New("first argument didn't match any known sandbox steps") + } +} + +func (s Sandbox) runCommand() (*exec.Cmd, error) { + serialized, err := s.serializeArg() + if err != nil { + return nil, err + } + // TODO: allow reference to self + // TODO: figure out what ^ means + path, err := exec.LookPath(setUIDExecName) + if err != nil { + return nil, err + } + logger.Debugw("newSanbox", "execpath", path) + // interrupt will be caught be the child process and the process + // will exiting, causing this process to exit + return &exec.Cmd{ + Path: path, + Args: []string{newNamespaceStepArg, serialized}, + Stdin: s.Stdin, + Stdout: s.Stdout, + Stderr: s.Stderr, + }, nil +} + +func (s Sandbox) newNamespaceStep() (err error) { + selfExe, err := os.Readlink("/proc/self/exe") + if err != nil { + return err + } + defer func() { + logger.Debugw("clean up chrootDir", "path", s.ChrootPath) + if er := os.RemoveAll(s.ChrootPath); er != nil { + logger.Debugw("error cleaning up", "err", er) + if err == nil { + err = errors.Wrap(er, "error removing all files in "+s.ChrootPath) + } + } + }() + serialized, err := s.serializeArg() + if err != nil { + return err + } + + var cloneFlags uintptr = syscall.CLONE_NEWUTS | + syscall.CLONE_NEWNS | + syscall.CLONE_NEWPID + + if s.DisableNetwork { + cloneFlags |= syscall.CLONE_NEWNET + } + + // interrupt will be caught be the child process and the process + // will exiting, causing this process to exit + ignoreInterrupt() + + cmd := &exec.Cmd{ + Path: selfExe, + Args: []string{setupStepArg, serialized}, + SysProcAttr: &syscall.SysProcAttr{ + // maybe sigint will allow the child more time to clean up its mounts???? + Pdeathsig: unix.SIGINT, + Cloneflags: cloneFlags, + }, + } + + // We must use a pty here to enable interactive input. If we naively pass + // os.Stdin to an exec.Cmd then we run into issues with the parent and + // child terminals getting confused about who is supposed to process various + // control signals. + // We can then just set to raw and copy the bytes across. We could remove + // the pty entirely for jobs that don't pass a terminal as a stdin. + ptmx, err := pty.Start(cmd) + if err != nil { + return errors.Wrap(err, "error starting pty") + } + defer func() { _ = ptmx.Close() }() + + // only handle stdin and set raw if it's an interactive terminal + if os.Stdin != nil && terminal.IsTerminal(int(os.Stdin.Fd())) { + // Handle pty resize + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGWINCH) + go func() { + for range ch { + if err := pty.InheritSize(os.Stdin, ptmx); err != nil { + log.Printf("error resizing pty: %s", err) + } + } + }() + ch <- syscall.SIGWINCH // Initial resize. + oldState, err := terminal.MakeRaw(int(os.Stdin.Fd())) + if err != nil { + return err + } + // restore when complete + defer func() { _ = terminal.Restore(int(os.Stdin.Fd()), oldState) }() + go func() { _, _ = io.Copy(ptmx, os.Stdin) }() + } + _, _ = io.Copy(os.Stdout, ptmx) + return errors.Wrap(cmd.Wait(), "error running newNamespace") +} + +func (s Sandbox) setupStep() (err error) { + logger.Debugw("setup chroot", "dir", s.ChrootPath) + creds := &syscall.Credential{ + Gid: uint32(s.GroupID), + Uid: uint32(s.UserID), + } + if err := os.Chown(s.ChrootPath, int(creds.Uid), int(creds.Gid)); err != nil { + return err + } + + chr := newChroot(s.ChrootPath, s.Mounts) + defer func() { + if er := chr.Cleanup(); er != nil { + if err == nil { + err = er + } else { + logger.Debugw("error during cleanup", "err", er) + } + } + }() + var selfExe string + { + // hardlink in executable + selfExe, err = os.Readlink("/proc/self/exe") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(s.ChrootPath, filepath.Dir(selfExe)), 0777); err != nil { + return err + } + if err = os.Link(selfExe, filepath.Join(s.ChrootPath, selfExe)); err != nil { + return err + } + } + + if err := chr.Init(); err != nil { + return err + } + + serialized, err := s.serializeArg() + if err != nil { + return err + } + + cmd := exec.CommandContext(interruptContext(), selfExe) + cmd.Path = selfExe + cmd.Args = []string{execStepArg, serialized} + cmd.Env = append([]string{"USER=bramblebuild0", "HOME=/homeless"}, s.Env...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: creds, + } + return cmd.Run() +}