Dev

.dockerignore

@@ -24,4 +24,5 @@ appdata
 .env
 /spec/generated/
 /**/node_modules
-/**/.parcel-cache
+/**/.parcel-cache
+/**/appdata*

.gitignore

@@ -27,3 +27,4 @@ appdata
 /spec/generated/
 /**/student.py
 .idea/*
+/**/appdata*

Justfile

@@ -18,6 +18,15 @@ krn:
 dk:
 	docker build . -t {{imageName}}
 
+lrn:
+    just dk
+    docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock {{imageName}}
+
+# docker compose up
+lrc:
+    docker compose --profile lev up --build
+
+
 # build leviathan with version and other metadata
 dkv:
     docker build \
@@ -40,10 +49,6 @@ down:
 dev:
     docker compose --profile dev up
 
-bdrn:
-    just dk
-    docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock {{imageName}}
-
 alias dc := dclean
 dclean:
     docker rm -f $(docker ps -aq)

docker-compose.yml

@@ -10,14 +10,12 @@ services:
       - .env
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
-      - ./appdata/:/app/appdata
+      - ./appdata2/:/app/appdata
       # warning mounting ssh will not work in windows hosts https://nickjanetakis.com/blog/docker-tip-56-volume-mounting-ssh-keys-into-a-docker-container
       - ~/.ssh:/root/.ssh:ro  # Read-only for security
-
     restart: "no"
-    depends_on:
-      - db
     profiles:
+      - lev
       - '' # so it starts with normal docker compose
 
   db:

src/common/config.go

@@ -62,7 +62,7 @@ func InitConfig() {
 	sshFolderPath := setIfEnvPresentOrDefault(
 		sshDirKey,
 		"SSH_CONFIG_DIR",
-		fmt.Sprintf("%s/%s", baseDir, "ssh_config"),
+		fmt.Sprintf("%s/%s", configDir, "ssh_config"),
 	)
 
 	err = makeDirectories([]string{submissionFolderPath, outputFolderPath, sshFolderPath})
@@ -162,11 +162,13 @@ func setupDefaultOptions(configDir string) {
 	viper.SetDefault(concurrentJobsKey, 50)
 	viper.SetDefault(clientSSHKey, map[string]models.MachineOptions{
 		"example": {
-			Name:   "example",
-			Enable: false,
-			Host:   "http://localhost:8080",
-			User:   "test",
-			Port:   22,
+			Enable:           false,
+			Host:             "192.168.1.69",
+			Port:             22,
+			User:             "test",
+			Password:         "",
+			RemotePublickey:  "",
+			UsePublicKeyAuth: false,
 		},
 	})
 }

src/models/machine.go

@@ -1,11 +1,70 @@
 package models
 
+import (
+	"fmt"
+	"reflect"
+	"strings"
+)
+
 type MachineOptions struct {
-	Enable    bool   `mapstructure:"enable"`
-	Name      string `mapstructure:"name"`
-	Host      string `mapstructure:"host"`
-	Port      int    `mapstructure:"port"`
-	User      string `mapstructure:"user"`
-	Password  string `mapstructure:"password"`
-	Publickey string `mapstructure:"host_publickey"`
+	name             string // use get/set so that this field is not written to the config file
+	Enable           bool   `mapstructure:"enable"`
+	Host             string `mapstructure:"host"`
+	Port             int    `mapstructure:"port"`
+	User             string `mapstructure:"user"`
+	Password         string `mapstructure:"password"`
+	RemotePublickey  string `json:"remote_public_key"`
+	UsePublicKeyAuth bool   `json:"use_public_key_auth"`
+}
+
+func (opts *MachineOptions) Log() string {
+	var result strings.Builder
+	v := reflect.ValueOf(*opts)
+	typeOfS := v.Type()
+
+	result.WriteString("options:\n")
+	for i := 0; i < v.NumField(); i++ {
+		field := v.Field(i)
+		fieldName := typeOfS.Field(i).Name
+		tag := typeOfS.Field(i).Tag
+
+		// Skip field
+		if fieldName == "RemotePublickey" || fieldName == "Password" {
+			continue
+		}
+
+		// Check for mapstructure or json tags
+		mapstructureTag := tag.Get("mapstructure")
+		jsonTag := tag.Get("json")
+
+		var outputName string
+		if mapstructureTag != "" {
+			outputName = mapstructureTag
+		} else if jsonTag != "" {
+			outputName = jsonTag
+		} else {
+			outputName = fieldName
+		}
+
+		// Format the output based on the field type
+		switch field.Kind() {
+		case reflect.String:
+			result.WriteString(fmt.Sprintf("  %s: %q\n", outputName, field.String()))
+		case reflect.Bool:
+			result.WriteString(fmt.Sprintf("  %s: %t\n", outputName, field.Bool()))
+		case reflect.Int:
+			result.WriteString(fmt.Sprintf("  %s: %d\n", outputName, field.Int()))
+		default:
+			result.WriteString(fmt.Sprintf("  %s: %v\n", outputName, field.Interface())) // Handle other types if needed
+		}
+	}
+	return strings.TrimSpace(result.String())
+}
+
+func (opts *MachineOptions) Name() string {
+	return opts.name
+}
+
+func (opts *MachineOptions) SetName(name string) {
+	opts.name = name
 }

src/service/docker/docker_manager.go

@@ -6,13 +6,14 @@ import (
 	"github.com/docker/cli/cli/connhelper"
 	"github.com/docker/docker/api/types/system"
 	"github.com/docker/docker/client"
-	"github.com/go-viper/mapstructure/v2"
 	"github.com/makeopensource/leviathan/common"
 	"github.com/makeopensource/leviathan/models"
 	"github.com/rs/zerolog/log"
+	"github.com/spf13/viper"
 	"golang.org/x/crypto/ssh"
 	"net/http"
 	"sync"
+	"time"
 )
 
 type Machine struct {
@@ -28,26 +29,30 @@ type RemoteClientManager struct {
 }
 
 func NewRemoteClientManager() *RemoteClientManager {
+	initKeyPairFile()
+
 	untestedClientList := getClientList()
 	clientList := MachineMap{} // contains final connected list
 
 	for _, machine := range untestedClientList {
 		var remoteClient *DkClient
 		var err error
 
-		if machine.Password != "" {
+		if machine.UsePublicKeyAuth {
+			remoteClient, err = NewSSHClientWithPublicKeyAuth(machine)
+		} else if machine.Password != "" {
 			remoteClient, err = NewSSHClientWithPasswordAuth(machine)
 		} else {
-			remoteClient, err = NewSSHClient(machine)
+			remoteClient, err = NewHostSSHClient(machine)
 		}
 		if err != nil {
-			log.Error().Err(err).Msgf("Failed to setup remote docker client: %s", machine.Name)
+			log.Error().Err(err).Msgf("Failed to setup remote docker client: %s", machine.Name())
 			continue
 		}
 
 		info, err := testClientConn(remoteClient.Client)
 		if err != nil {
-			log.Warn().Err(err).Msgf("Remote docker client failed to connect: %s", machine.Name)
+			log.Warn().Err(err).Msgf("Remote docker client failed to connect: %s", machine.Name())
 			continue
 		}
 
@@ -84,11 +89,16 @@ func NewRemoteClientManager() *RemoteClientManager {
 	return &RemoteClientManager{Clients: clientList, mu: sync.RWMutex{}}
 }
 
-// NewSSHClient connects to a remote docker host using a public private auth.
+// NewHostSSHClient connects to a remote docker host using public/private key authentication.
+// It uses the local machine's SSH configuration (typically ~/.ssh) for connection.
+//
+// Prerequisites:
+//   - The remote host must have an SSH server (sshd) running and accessible.
+//   - The local machine must have the necessary SSH keys and configuration to connect.
 //
-// It is assumed host running leviathan has done the necessary SSH setup, via sshd or other ssh programs
-// and the remote host loaded is accessible via SSH
-func NewSSHClient(machine models.MachineOptions) (*DkClient, error) {
+// This function assumes the user has already configured SSH access to the remote host.
+// It does not handle key generation or SSH configuration.
+func NewHostSSHClient(machine models.MachineOptions) (*DkClient, error) {
 	connectionString := fmt.Sprintf("%s@%s:%d", machine.User, machine.Host, machine.Port)
 	helper, err := connhelper.GetConnectionHelper(fmt.Sprintf("ssh://%s", connectionString))
 	if err != nil {
@@ -117,26 +127,51 @@ func NewSSHClient(machine models.MachineOptions) (*DkClient, error) {
 	return NewDkClient(newClient), nil
 }
 
+// NewSSHClientWithPublicKeyAuth connects to a remote docker host using public/private key authentication.
+//
+// Unlike NewHostSSHClient, this function uses the SSH keys generated by leviathan.
+// Removes the dependency from configuring the host machine, useful for deploying in docker
+//
+// This function assumes the user has already transferred the public key generated by initKeyPairFile
+// and configured SSH access to the remote host.
+func NewSSHClientWithPublicKeyAuth(machine models.MachineOptions) (*DkClient, error) {
+	privateKey, err := LoadPrivateKey()
+	if err != nil {
+		return nil, fmt.Errorf("failed to load private key: %s", err.Error())
+	}
+	signer, err := ssh.ParsePrivateKey(privateKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse private key: %s", err.Error())
+	}
+
+	auth := ssh.PublicKeys(signer)
+	return createSshDockerConnection(machine, auth)
+}
+
 // NewSSHClientWithPasswordAuth connects to a remote docker host using a password.
 //
 // It is assumed machine models.MachineOptions has the correct password set.
-//
-// if the public key is empty then leviathan will save the public key on connection
-// else it will verify the set public key
 func NewSSHClientWithPasswordAuth(machine models.MachineOptions) (*DkClient, error) {
+	auth := ssh.Password(machine.Password)
+	return createSshDockerConnection(machine, auth)
+}
+
+// createSshDockerConnection establishes an SSH connection to a Docker host based on the provided authentication method.
+//
+// If models.MachineOptions contains an empty public key, the key is saved on connect;
+// otherwise, the provided key is verified.
+func createSshDockerConnection(machine models.MachineOptions, auth ...ssh.AuthMethod) (*DkClient, error) {
 	sshHost := fmt.Sprintf("%s:%d", machine.Host, machine.Port)
-	// Create an SSH client configuration.
 	config := &ssh.ClientConfig{
-		User: machine.User,
-		Auth: []ssh.AuthMethod{
-			ssh.Password(machine.Password),
-		},
+		User:            machine.User,
+		Auth:            auth,
 		HostKeyCallback: saveHostKey(machine),
+		Timeout:         10 * time.Second,
 	}
 
-	if machine.Publickey != "" {
-		log.Debug().Msgf("Verifying public key for %s", machine.Name)
-		pubkey, err := stringToPublicKey(machine.Publickey)
+	if machine.RemotePublickey != "" {
+		log.Debug().Msgf("Verifying public key for %s", machine.Name())
+		pubkey, err := stringToPublicKey(machine.RemotePublickey)
 		if err != nil {
 			return nil, err
 		}
@@ -259,20 +294,21 @@ func getClientList() map[string]models.MachineOptions {
 		return machineMap
 	}
 
-	for name, info := range clients {
+	for name := range clients {
 		var options models.MachineOptions
-		err := mapstructure.Decode(info, &options)
-		if err != nil {
+		key := fmt.Sprintf("clients.ssh.%s", name)
+
+		if err := viper.UnmarshalKey(key, &options); err != nil {
 			log.Warn().Err(err).Msgf("Error decoding configuration structure for %s", name)
 			continue
 		}
 
-		options.Name = name // Set the name manually since it's not part of the nested structure
+		options.SetName(name) // Set the name manually since it's not part of the nested structure
 		if options.Enable {
 			machineMap[name] = options
-			log.Info().Any("options", options).Msgf("found machine config: %s", name)
+			log.Info().Msgf("found %s", options.Log())
 		} else {
-			log.Debug().Any("options", options).Msgf("found machine config: %s, but it was disabled", name)
+			log.Debug().Msgf("found disabled %s", options.Log())
 		}
 	}
 

src/service/docker/docker_manager_test.go

@@ -99,7 +99,7 @@ func TestRemoteClientManager_GetLeastJobCountMachineId(t *testing.T) {
 
 func TestNewSSHClientWithPasswordAuth(t *testing.T) {
 	common.InitConfig()
-	InitKeyPairFile()
+	initKeyPairFile()
 
 	// when running this test update the config.yaml with the test machine info
 	cli := getClientList()
@@ -126,3 +126,33 @@ func TestNewSSHClientWithPasswordAuth(t *testing.T) {
 		t.Log(image)
 	}
 }
+
+func TestNewSSHClientWithPublicKeyAuth(t *testing.T) {
+	common.InitConfig()
+	initKeyPairFile()
+
+	// when running this test update the config.yaml with the test machine info
+	cli := getClientList()
+	mName := "test"
+	machine, ok := cli[mName]
+	if !ok {
+		t.Fatalf("machine %s not configured", mName)
+		return
+	}
+
+	client, err := NewSSHClientWithPublicKeyAuth(machine)
+	if err != nil {
+		t.Fatalf("failed create remote docker client %v", err)
+		return
+	}
+
+	images, err := client.ListImages()
+	if err != nil {
+		t.Fatalf("failed list images %v", err)
+		return
+	}
+
+	for _, image := range images {
+		t.Log(image)
+	}
+}

src/service/docker/docker_service_test.go

@@ -5,7 +5,6 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/google/uuid"
 	"github.com/makeopensource/leviathan/common"
-	v1 "github.com/makeopensource/leviathan/generated/types/v1"
 	"sync"
 	"testing"
 )
@@ -49,21 +48,6 @@ func TestCopyToContainer(t *testing.T) {
 		t.Fatalf("%v", err)
 	}
 
-	dir, err := common.CreateTmpJobDir(ifg.String(), "", &v1.FileUpload{
-		Filename: "test.txt",
-		Content:  []byte("tests test"),
-	})
-	if err != nil {
-		t.Fatalf("%v", err)
-		return
-	}
-
-	// just copy and check if it succeeds
-	err = machine.CopyToContainer(contId, dir)
-	if err != nil {
-		t.Fatalf("%v", err)
-	}
-
 	err = machine.RemoveContainer(contId, true, true)
 	if err != nil {
 		t.Fatalf("%v", err)