From 871cdfc71165a4fcfdc8d9934c6e6a645bd5b5e6 Mon Sep 17 00:00:00 2001 From: RA341 Date: Fri, 21 Mar 2025 21:51:35 -0400 Subject: [PATCH 1/7] removed fixed tests --- src/service/docker/docker_service_test.go | 16 ---------------- src/service/jobs/job_service_test.go | 9 +++++---- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/src/service/docker/docker_service_test.go b/src/service/docker/docker_service_test.go index e8f6cfd..08c5989 100644 --- a/src/service/docker/docker_service_test.go +++ b/src/service/docker/docker_service_test.go @@ -5,7 +5,6 @@ import ( "github.com/docker/docker/api/types/container" "github.com/google/uuid" "github.com/makeopensource/leviathan/common" - v1 "github.com/makeopensource/leviathan/generated/types/v1" "sync" "testing" ) @@ -49,21 +48,6 @@ func TestCopyToContainer(t *testing.T) { t.Fatalf("%v", err) } - dir, err := common.CreateTmpJobDir(ifg.String(), "", &v1.FileUpload{ - Filename: "test.txt", - Content: []byte("tests test"), - }) - if err != nil { - t.Fatalf("%v", err) - return - } - - // just copy and check if it succeeds - err = machine.CopyToContainer(contId, dir) - if err != nil { - t.Fatalf("%v", err) - } - err = machine.RemoveContainer(contId, true, true) if err != nil { t.Fatalf("%v", err) diff --git a/src/service/jobs/job_service_test.go b/src/service/jobs/job_service_test.go index bfb24d6..e068c18 100644 --- a/src/service/jobs/job_service_test.go +++ b/src/service/jobs/job_service_test.go @@ -9,6 +9,7 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/stretchr/testify/assert" + "os" "path/filepath" "strings" "sync" @@ -161,22 +162,22 @@ func testJobProcessor(t *testing.T, studentCodePath string, correctOutput string } func setupJobProcess(studentCodePath string, timeout time.Duration) string { - graderBytes, err := common.ReadFileBytes(graderFilePath) + graderBytes, err := os.ReadFile(graderFilePath) if err != nil { log.Fatal().Err(err).Msg("Error reading grader.py") } - makefileBytes, err := common.ReadFileBytes(makeFilePath) + makefileBytes, err := os.ReadFile(makeFilePath) if err != nil { log.Fatal().Err(err).Msg("Error reading grader.py") } - studentBytes, err := common.ReadFileBytes(studentCodePath) + studentBytes, err := os.ReadFile(studentCodePath) if err != nil { log.Fatal().Err(err).Msg("Error reading student") } - dockerBytes, err := common.ReadFileBytes(dockerFilePath) + dockerBytes, err := os.ReadFile(dockerFilePath) if err != nil { log.Fatal().Err(err).Msg("Error reading docker file") } From 93eaaa9d4a60cc395edbe4da948cdac80a126e8f Mon Sep 17 00:00:00 2001 From: RA341 Date: Sat, 22 Mar 2025 16:22:39 -0400 Subject: [PATCH 2/7] updated ignore --- .dockerignore | 3 ++- .gitignore | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 1912629..a00da78 100644 --- a/.dockerignore +++ b/.dockerignore @@ -24,4 +24,5 @@ appdata .env /spec/generated/ /**/node_modules -/**/.parcel-cache \ No newline at end of file +/**/.parcel-cache +/**/appdata* diff --git a/.gitignore b/.gitignore index bd391c9..98e03cd 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ appdata /spec/generated/ /**/student.py .idea/* +/**/appdata* \ No newline at end of file From c4624548b96497d906db80025caac4e5567190d6 Mon Sep 17 00:00:00 2001 From: RA341 Date: Sat, 22 Mar 2025 16:22:45 -0400 Subject: [PATCH 3/7] updated commands --- Justfile | 13 +++++++++---- docker-compose.yml | 6 ++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Justfile b/Justfile index be2de7d..755d67a 100644 --- a/Justfile +++ b/Justfile @@ -18,6 +18,15 @@ krn: dk: docker build . -t {{imageName}} +lrn: + just dk + docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock {{imageName}} + +# docker compose up +lrc: + docker compose --profile lev up --build + + # build leviathan with version and other metadata dkv: docker build \ @@ -40,10 +49,6 @@ down: dev: docker compose --profile dev up -bdrn: - just dk - docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock {{imageName}} - alias dc := dclean dclean: docker rm -f $(docker ps -aq) diff --git a/docker-compose.yml b/docker-compose.yml index 43bdc3a..5fc06c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,14 +10,12 @@ services: - .env volumes: - /var/run/docker.sock:/var/run/docker.sock - - ./appdata/:/app/appdata + - ./appdata2/:/app/appdata # warning mounting ssh will not work in windows hosts https://nickjanetakis.com/blog/docker-tip-56-volume-mounting-ssh-keys-into-a-docker-container - ~/.ssh:/root/.ssh:ro # Read-only for security - restart: "no" - depends_on: - - db profiles: + - lev - '' # so it starts with normal docker compose db: From 1223438e15243e0958d0707cdcc3ba00c04f2c46 Mon Sep 17 00:00:00 2001 From: RA341 Date: Sat, 22 Mar 2025 16:22:57 -0400 Subject: [PATCH 4/7] changed ssh dir location --- src/common/config.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/common/config.go b/src/common/config.go index bfebcb0..3abde5d 100644 --- a/src/common/config.go +++ b/src/common/config.go @@ -62,7 +62,7 @@ func InitConfig() { sshFolderPath := setIfEnvPresentOrDefault( sshDirKey, "SSH_CONFIG_DIR", - fmt.Sprintf("%s/%s", baseDir, "ssh_config"), + fmt.Sprintf("%s/%s", configDir, "ssh_config"), ) err = makeDirectories([]string{submissionFolderPath, outputFolderPath, sshFolderPath}) @@ -162,11 +162,13 @@ func setupDefaultOptions(configDir string) { viper.SetDefault(concurrentJobsKey, 50) viper.SetDefault(clientSSHKey, map[string]models.MachineOptions{ "example": { - Name: "example", - Enable: false, - Host: "http://localhost:8080", - User: "test", - Port: 22, + Enable: false, + Host: "192.168.1.69", + Port: 22, + User: "test", + Password: "", + RemotePublickey: "", + UsePublicKeyAuth: false, }, }) } From 4b9bce0ccb9182f95d79ca61e3adb0d8e7963e45 Mon Sep 17 00:00:00 2001 From: RA341 Date: Sat, 22 Mar 2025 16:23:15 -0400 Subject: [PATCH 5/7] added methods --- src/models/machine.go | 73 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/src/models/machine.go b/src/models/machine.go index e8bb5e5..9f5484c 100644 --- a/src/models/machine.go +++ b/src/models/machine.go @@ -1,11 +1,70 @@ package models +import ( + "fmt" + "reflect" + "strings" +) + type MachineOptions struct { - Enable bool `mapstructure:"enable"` - Name string `mapstructure:"name"` - Host string `mapstructure:"host"` - Port int `mapstructure:"port"` - User string `mapstructure:"user"` - Password string `mapstructure:"password"` - Publickey string `mapstructure:"host_publickey"` + name string // use get/set so that this field is not written to the config file + Enable bool `mapstructure:"enable"` + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + User string `mapstructure:"user"` + Password string `mapstructure:"password"` + RemotePublickey string `json:"remote_public_key"` + UsePublicKeyAuth bool `json:"use_public_key_auth"` +} + +func (opts *MachineOptions) Log() string { + var result strings.Builder + v := reflect.ValueOf(*opts) + typeOfS := v.Type() + + result.WriteString("options:\n") + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + fieldName := typeOfS.Field(i).Name + tag := typeOfS.Field(i).Tag + + // Skip field + if fieldName == "RemotePublickey" || fieldName == "Password" { + continue + } + + // Check for mapstructure or json tags + mapstructureTag := tag.Get("mapstructure") + jsonTag := tag.Get("json") + + var outputName string + if mapstructureTag != "" { + outputName = mapstructureTag + } else if jsonTag != "" { + outputName = jsonTag + } else { + outputName = fieldName + } + + // Format the output based on the field type + switch field.Kind() { + case reflect.String: + result.WriteString(fmt.Sprintf(" %s: %q\n", outputName, field.String())) + case reflect.Bool: + result.WriteString(fmt.Sprintf(" %s: %t\n", outputName, field.Bool())) + case reflect.Int: + result.WriteString(fmt.Sprintf(" %s: %d\n", outputName, field.Int())) + default: + result.WriteString(fmt.Sprintf(" %s: %v\n", outputName, field.Interface())) // Handle other types if needed + } + } + return strings.TrimSpace(result.String()) +} + +func (opts *MachineOptions) Name() string { + return opts.name +} + +func (opts *MachineOptions) SetName(name string) { + opts.name = name } From 4bbc26335f64f65267505081a8722db3867dd180 Mon Sep 17 00:00:00 2001 From: RA341 Date: Sat, 22 Mar 2025 16:23:38 -0400 Subject: [PATCH 6/7] small refactor --- src/service/docker/docker_utils_ssh.go | 51 ++++++++++++++++++-------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/src/service/docker/docker_utils_ssh.go b/src/service/docker/docker_utils_ssh.go index 02b58c5..68dbce5 100644 --- a/src/service/docker/docker_utils_ssh.go +++ b/src/service/docker/docker_utils_ssh.go @@ -55,26 +55,26 @@ func sshDialer(sshClient *ssh.Client) func(ctx context.Context, network string, func saveHostKey(machine models.MachineOptions) func(hostname string, remote net.Addr, key ssh.PublicKey) error { return func(hostname string, remote net.Addr, key ssh.PublicKey) error { - log.Debug().Msgf("Empty public key for %s, public key will be saved on connect", machine.Name) + log.Debug().Msgf("Empty public key for %s, public key will be saved on connect", machine.Name()) - comment := fmt.Sprintf("added by leviathan for machine %s on %s", machine.Name, time.Now().String()) + comment := fmt.Sprintf("added by leviathan for machine %s on %s", machine.Name(), time.Now().String()) stringKey, err := publicKeyToString(key, comment) if err != nil { - return fmt.Errorf("unable to convert public key for machine %s ", machine.Name) + return fmt.Errorf("unable to convert public key for machine %s ", machine.Name()) } - machine.Publickey = stringKey + machine.RemotePublickey = stringKey writeMachineToConfigFile(machine) return nil } } func writeMachineToConfigFile(machine models.MachineOptions) { - machineKey := fmt.Sprintf("%s.%s", common.ClientsSSH.ConfigKey, machine.Name) + machineKey := fmt.Sprintf("%s.%s", common.ClientsSSH.ConfigKey, machine.Name()) viper.Set(machineKey, machine) err := viper.WriteConfig() if err != nil { - log.Warn().Err(err).Msgf("failed to update machine %s public key to config", machine.Name) + log.Warn().Err(err).Msgf("failed to update machine %s public key to config", machine.Name()) } } @@ -104,16 +104,32 @@ func GenerateKeyPair() (privateKey []byte, publicKey []byte, err error) { return privatePEM, pubKeyBytes, nil } -func InitKeyPairFile() (priv string, pub string) { +// initKeyPairFile creates RSA key-pair files, +// if they do not exist, otherwise skips generation +// +// the generated keys can be found in common.SSHConfigFolder +func initKeyPairFile() { + basePath := common.SSHConfigFolder.GetStr() + privateKeyPath := fmt.Sprintf("%s/%s", basePath, "id_rsa") + publicKeyPath := fmt.Sprintf("%s/%s", basePath, "id_rsa.pub") + + defer log.Info(). + Msgf("to add the public key to other hosts use\nssh-copy-id -i %s @\n", publicKeyPath) + + logF := log.Info(). + Str("private_key_file", privateKeyPath). + Str("public_key_file", publicKeyPath) + + if fileExists(privateKeyPath) && fileExists(publicKeyPath) { + logF.Msg("found existing keys... skipping generation") + return + } + privateKey, publicKey, err := GenerateKeyPair() if err != nil { log.Fatal().Err(err).Msg("Failed to generate key pair") } - basePath := common.SSHConfigFolder.GetStr() - privateKeyPath := fmt.Sprintf("%s/%s", basePath, "id_rsa") - publicKeyPath := fmt.Sprintf("%s/%s", basePath, "id_rsa.pub") - if err := os.WriteFile(privateKeyPath, privateKey, 0600); err != nil { log.Fatal().Err(err).Msg("Failed to save private key") } @@ -121,12 +137,15 @@ func InitKeyPairFile() (priv string, pub string) { log.Fatal().Err(err).Msg("Failed to save public key") } - log.Info(). - Str("private_key_file", privateKeyPath). - Str("public_key_file", publicKeyPath). - Msg("Generated new SSH key pair") + logF.Msg("Generated new SSH key pair") +} - return privateKeyPath, publicKeyPath +func fileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() } func LoadPrivateKey() ([]byte, error) { From 23ae0600c035f133bba5e3795a4011c72d3dbbc6 Mon Sep 17 00:00:00 2001 From: RA341 Date: Sat, 22 Mar 2025 16:23:50 -0400 Subject: [PATCH 7/7] added public auth support --- src/service/docker/docker_manager.go | 88 ++++++++++++++++------- src/service/docker/docker_manager_test.go | 32 ++++++++- 2 files changed, 93 insertions(+), 27 deletions(-) diff --git a/src/service/docker/docker_manager.go b/src/service/docker/docker_manager.go index a870574..45c7402 100644 --- a/src/service/docker/docker_manager.go +++ b/src/service/docker/docker_manager.go @@ -6,13 +6,14 @@ import ( "github.com/docker/cli/cli/connhelper" "github.com/docker/docker/api/types/system" "github.com/docker/docker/client" - "github.com/go-viper/mapstructure/v2" "github.com/makeopensource/leviathan/common" "github.com/makeopensource/leviathan/models" "github.com/rs/zerolog/log" + "github.com/spf13/viper" "golang.org/x/crypto/ssh" "net/http" "sync" + "time" ) type Machine struct { @@ -28,6 +29,8 @@ type RemoteClientManager struct { } func NewRemoteClientManager() *RemoteClientManager { + initKeyPairFile() + untestedClientList := getClientList() clientList := MachineMap{} // contains final connected list @@ -35,19 +38,21 @@ func NewRemoteClientManager() *RemoteClientManager { var remoteClient *DkClient var err error - if machine.Password != "" { + if machine.UsePublicKeyAuth { + remoteClient, err = NewSSHClientWithPublicKeyAuth(machine) + } else if machine.Password != "" { remoteClient, err = NewSSHClientWithPasswordAuth(machine) } else { - remoteClient, err = NewSSHClient(machine) + remoteClient, err = NewHostSSHClient(machine) } if err != nil { - log.Error().Err(err).Msgf("Failed to setup remote docker client: %s", machine.Name) + log.Error().Err(err).Msgf("Failed to setup remote docker client: %s", machine.Name()) continue } info, err := testClientConn(remoteClient.Client) if err != nil { - log.Warn().Err(err).Msgf("Remote docker client failed to connect: %s", machine.Name) + log.Warn().Err(err).Msgf("Remote docker client failed to connect: %s", machine.Name()) continue } @@ -84,11 +89,16 @@ func NewRemoteClientManager() *RemoteClientManager { return &RemoteClientManager{Clients: clientList, mu: sync.RWMutex{}} } -// NewSSHClient connects to a remote docker host using a public private auth. +// NewHostSSHClient connects to a remote docker host using public/private key authentication. +// It uses the local machine's SSH configuration (typically ~/.ssh) for connection. +// +// Prerequisites: +// - The remote host must have an SSH server (sshd) running and accessible. +// - The local machine must have the necessary SSH keys and configuration to connect. // -// It is assumed host running leviathan has done the necessary SSH setup, via sshd or other ssh programs -// and the remote host loaded is accessible via SSH -func NewSSHClient(machine models.MachineOptions) (*DkClient, error) { +// This function assumes the user has already configured SSH access to the remote host. +// It does not handle key generation or SSH configuration. +func NewHostSSHClient(machine models.MachineOptions) (*DkClient, error) { connectionString := fmt.Sprintf("%s@%s:%d", machine.User, machine.Host, machine.Port) helper, err := connhelper.GetConnectionHelper(fmt.Sprintf("ssh://%s", connectionString)) if err != nil { @@ -117,26 +127,51 @@ func NewSSHClient(machine models.MachineOptions) (*DkClient, error) { return NewDkClient(newClient), nil } +// NewSSHClientWithPublicKeyAuth connects to a remote docker host using public/private key authentication. +// +// Unlike NewHostSSHClient, this function uses the SSH keys generated by leviathan. +// Removes the dependency from configuring the host machine, useful for deploying in docker +// +// This function assumes the user has already transferred the public key generated by initKeyPairFile +// and configured SSH access to the remote host. +func NewSSHClientWithPublicKeyAuth(machine models.MachineOptions) (*DkClient, error) { + privateKey, err := LoadPrivateKey() + if err != nil { + return nil, fmt.Errorf("failed to load private key: %s", err.Error()) + } + signer, err := ssh.ParsePrivateKey(privateKey) + if err != nil { + return nil, fmt.Errorf("failed to parse private key: %s", err.Error()) + } + + auth := ssh.PublicKeys(signer) + return createSshDockerConnection(machine, auth) +} + // NewSSHClientWithPasswordAuth connects to a remote docker host using a password. // // It is assumed machine models.MachineOptions has the correct password set. -// -// if the public key is empty then leviathan will save the public key on connection -// else it will verify the set public key func NewSSHClientWithPasswordAuth(machine models.MachineOptions) (*DkClient, error) { + auth := ssh.Password(machine.Password) + return createSshDockerConnection(machine, auth) +} + +// createSshDockerConnection establishes an SSH connection to a Docker host based on the provided authentication method. +// +// If models.MachineOptions contains an empty public key, the key is saved on connect; +// otherwise, the provided key is verified. +func createSshDockerConnection(machine models.MachineOptions, auth ...ssh.AuthMethod) (*DkClient, error) { sshHost := fmt.Sprintf("%s:%d", machine.Host, machine.Port) - // Create an SSH client configuration. config := &ssh.ClientConfig{ - User: machine.User, - Auth: []ssh.AuthMethod{ - ssh.Password(machine.Password), - }, + User: machine.User, + Auth: auth, HostKeyCallback: saveHostKey(machine), + Timeout: 10 * time.Second, } - if machine.Publickey != "" { - log.Debug().Msgf("Verifying public key for %s", machine.Name) - pubkey, err := stringToPublicKey(machine.Publickey) + if machine.RemotePublickey != "" { + log.Debug().Msgf("Verifying public key for %s", machine.Name()) + pubkey, err := stringToPublicKey(machine.RemotePublickey) if err != nil { return nil, err } @@ -259,20 +294,21 @@ func getClientList() map[string]models.MachineOptions { return machineMap } - for name, info := range clients { + for name := range clients { var options models.MachineOptions - err := mapstructure.Decode(info, &options) - if err != nil { + key := fmt.Sprintf("clients.ssh.%s", name) + + if err := viper.UnmarshalKey(key, &options); err != nil { log.Warn().Err(err).Msgf("Error decoding configuration structure for %s", name) continue } - options.Name = name // Set the name manually since it's not part of the nested structure + options.SetName(name) // Set the name manually since it's not part of the nested structure if options.Enable { machineMap[name] = options - log.Info().Any("options", options).Msgf("found machine config: %s", name) + log.Info().Msgf("found %s", options.Log()) } else { - log.Debug().Any("options", options).Msgf("found machine config: %s, but it was disabled", name) + log.Debug().Msgf("found disabled %s", options.Log()) } } diff --git a/src/service/docker/docker_manager_test.go b/src/service/docker/docker_manager_test.go index af59029..47c290f 100644 --- a/src/service/docker/docker_manager_test.go +++ b/src/service/docker/docker_manager_test.go @@ -99,7 +99,7 @@ func TestRemoteClientManager_GetLeastJobCountMachineId(t *testing.T) { func TestNewSSHClientWithPasswordAuth(t *testing.T) { common.InitConfig() - InitKeyPairFile() + initKeyPairFile() // when running this test update the config.yaml with the test machine info cli := getClientList() @@ -126,3 +126,33 @@ func TestNewSSHClientWithPasswordAuth(t *testing.T) { t.Log(image) } } + +func TestNewSSHClientWithPublicKeyAuth(t *testing.T) { + common.InitConfig() + initKeyPairFile() + + // when running this test update the config.yaml with the test machine info + cli := getClientList() + mName := "test" + machine, ok := cli[mName] + if !ok { + t.Fatalf("machine %s not configured", mName) + return + } + + client, err := NewSSHClientWithPublicKeyAuth(machine) + if err != nil { + t.Fatalf("failed create remote docker client %v", err) + return + } + + images, err := client.ListImages() + if err != nil { + t.Fatalf("failed list images %v", err) + return + } + + for _, image := range images { + t.Log(image) + } +}