Persistance

README.md

@@ -16,6 +16,7 @@ This is a distributed, in-memory key-value store with write-ahead logging (WAL)
 - [ ] **Persistent Disk Storage**: Working on storing data to disk efficiently.
 - [ ] **Additional Endpoints**: Implementing `GET`, `UPDATE`, `DELETE`, and other operations.
 - [ ] **GraphQL & Live Queries**: Exploring GraphQL or similar solutions for reactivity.
+- [ ] Implement red black tree by myself
 
 ## Future Plans
 - **Replication & Failover**: Implement strategies for high availability.

cmd/default-config.json

@@ -19,6 +19,6 @@
 	 "inMemoryStorageThreshold": 2000,
 	 "metaDataConfig": {
 	 	 "state": 1,
-	 	 "walPath": "/var/lib/db/wal"
+	 	 "walPath": "../runtime-files/wal-storage"
 	 }
 }

cmd/init.go

@@ -9,23 +9,29 @@ import (
 	"github.com/SuperALKALINEdroiD/timelyDB/config"
 	"github.com/SuperALKALINEdroiD/timelyDB/core"
 	"github.com/SuperALKALINEdroiD/timelyDB/handlers"
+	"github.com/SuperALKALINEdroiD/timelyDB/utils/common"
 	"github.com/go-chi/chi/middleware"
 	"github.com/go-chi/chi/v5"
 )
 
 func initEnvironment() (*config.DatabaseConfig, error) {
-	var configPath = os.Getenv("CONFIG_PATH")
-
-	fmt.Println(configPath)
+	var configPath = os.Getenv("LOG_BASE_SETTINGS")
 
 	cfg, err := config.LoadConfig(configPath)
 	if err != nil {
 		log.Printf("Error loading configuration: %v", err)
 		return nil, err
 	}
+
 	return cfg, nil
 }
 
+func GetAppPath() string {
+	path := common.GetAppPath()
+
+	return path
+}
+
 func initRouter(app *core.App) *chi.Mux {
 	router := chi.NewRouter()
 	addMiddlewares(router)
@@ -54,6 +60,8 @@ func initRoutes(router *chi.Mux, app *core.App) {
 
 		r.Post("/insert", handlers.InsertHandler(app))
 
+		r.Get("/", handlers.GetValue(app))
+
 		r.Post("/update", func(w http.ResponseWriter, r *http.Request) {
 			w.WriteHeader(http.StatusOK)
 			fmt.Fprintf(w, "Update Endpoint WIP - Config: %+v", app)

cmd/main.go

@@ -7,14 +7,17 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
-	"runtime/pprof"
+	"path/filepath"
 	"syscall"
 	"time"
 
 	"github.com/SuperALKALINEdroiD/timelyDB/core"
+	"github.com/SuperALKALINEdroiD/timelyDB/utils/common"
 	"github.com/SuperALKALINEdroiD/timelyDB/utils/logs"
 	"github.com/SuperALKALINEdroiD/timelyDB/utils/nodes"
 	"github.com/SuperALKALINEdroiD/timelyDB/utils/storage"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
 func main() {
@@ -24,10 +27,6 @@ func main() {
 	}
 	defer f.Close()
 
-	if err := pprof.StartCPUProfile(f); err != nil {
-		panic(err)
-	}
-	defer pprof.StopCPUProfile()
 	ctx, cancel := context.WithCancel(context.Background())
 
 	signalChannel := make(chan os.Signal, 1)
@@ -47,11 +46,28 @@ func main() {
 
 	grpcNodes, nodeHashInfo := nodes.LoadServers(ctx, config)
 	wal := &storage.LocalWAL{}
-	wal.Connect("wal-storage")
+	appPath := common.GetAppPath()
+	wal.Connect(filepath.Join(appPath, config.MetaDataConfig.WALName))
+
+	nodeByID := make(map[string]*nodes.Node, len(grpcNodes))
+	nodeClients := make(map[string]nodes.NodeServiceClient, len(grpcNodes))
+	for _, n := range grpcNodes {
+		if n == nil {
+			continue
+		}
+		nodeByID[n.ID] = n
+		conn, connErr := grpc.NewClient(n.Address, grpc.WithTransportCredentials(insecure.NewCredentials()))
+		if connErr != nil {
+			log.Fatalf("failed to create gRPC client for node %s: %v", n.ID, connErr)
+		}
+		nodeClients[n.ID] = nodes.NewNodeServiceClient(conn)
+	}
 
 	app := &core.App{
 		Config:       config,
 		Nodes:        grpcNodes,
+		NodeByID:     nodeByID,
+		NodeClients:  nodeClients,
 		NodeHashInfo: nodeHashInfo,
 		WAL:          wal,
 	}
@@ -61,7 +77,7 @@ func main() {
 	router := initRouter(app)
 
 	serverAddress := fmt.Sprintf(":%d", app.Config.Port)
-	log.Printf("Starting server on %s", serverAddress)
+	log.Printf("Starting %s server on %s", config.StoreName, serverAddress)
 	server := &http.Server{Addr: ":7001", Handler: router}
 
 	go func() {

config/main.go

@@ -5,22 +5,28 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path/filepath"
+
+	"github.com/SuperALKALINEdroiD/timelyDB/manifest"
+	"github.com/SuperALKALINEdroiD/timelyDB/utils/common"
+	"github.com/go-playground/validator/v10"
 )
 
 func LoadConfig(filePath string) (*DatabaseConfig, error) {
 	if filePath == "" {
-		config, error := GenerateConfig("default-config.json")
-
-		if error != nil {
-			return nil, fmt.Errorf("failed to create config file: %v", error)
-		}
-
-		return config, nil
+		return GenerateConfig()
 	}
 
+	log.Printf("Loading config at %s", filePath)
+
 	file, err := os.Open(filePath)
 
 	if err != nil {
+		if os.IsNotExist(err) {
+			log.Println("Config file not found. Generating default config.")
+			return GenerateConfig()
+		}
+
 		return nil, fmt.Errorf("failed to open config file: %v", err)
 	}
 
@@ -35,6 +41,11 @@ func LoadConfig(filePath string) (*DatabaseConfig, error) {
 	}
 
 	if config.validateConfig() {
+		manifest, err := manifest.GetManifest()
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse config file: %v", err)
+		}
+		config.Manifest = *manifest
 		return &config, nil
 	}
 
@@ -43,9 +54,23 @@ func LoadConfig(filePath string) (*DatabaseConfig, error) {
 }
 
 func (config *DatabaseConfig) validateConfig() bool {
-	if len(config.Nodes) == 0 {
+	if len(config.Nodes) == 0 && config.NodeCount > 0 {
 		config.Nodes = generateNodeConfig(config.NodeCount, "")
 	}
+	validate := validator.New()
+	err := validate.Struct(config)
+	if err != nil {
+		for _, err := range err.(validator.ValidationErrors) {
+			log.Printf("Validation error: Field '%s', Tag: '%s', Param: '%s'\n", err.Field(), err.Tag(), err.Param())
+		}
+
+		return false
+	}
+
+	if len(config.Nodes) != config.NodeCount {
+		log.Printf("Validation error: Node count mismatch. Expected %d nodes, but got %d nodes.\n", config.NodeCount, len(config.Nodes))
+		return false
+	}
 
 	return true
 }
@@ -68,21 +93,40 @@ func (c *DatabaseConfig) SaveConfig(filePath string) error {
 	return nil
 }
 
-func GenerateConfig(filePath string) (*DatabaseConfig, error) {
-	file, err := os.Create(filePath)
+func GenerateConfig() (*DatabaseConfig, error) {
+	defaultConfigFile := "default-config.json"
+
+	defaultDbPath := common.GetAppPath()
+	configData := GenerateExampleConfig(2, "localhost")
+
+	defaultConfigPath := filepath.Join(defaultDbPath, defaultConfigFile)
+	os.Setenv("DATABASE_SETTINGS", defaultDbPath)
+
+	data, err := json.Marshal(configData)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create config file: %v", err)
+		fmt.Println("Error marshalling example config:", err)
+		return nil, err
+	}
+
+	if _, err := os.Stat(defaultDbPath); os.IsNotExist(err) {
+		err := os.MkdirAll(defaultDbPath, os.ModePerm)
+		if err != nil {
+			return nil, err
+		}
 	}
-	defer file.Close()
 
-	exampleConfig := GenerateExampleConfig(2, "") // if no config, start with 1 node on local machine
+	err = os.WriteFile(defaultConfigPath, data, 0644)
+	if err != nil {
+		fmt.Println("Error writing config to file:", err)
+		return nil, err
+	}
 
-	encoder := json.NewEncoder(file)
-	encoder.SetIndent("", "	 ")
-	if err := encoder.Encode(exampleConfig); err != nil {
-		return nil, fmt.Errorf("failed to save config file, error on encoder: %v", err)
+	manifest, err := manifest.GetManifest()
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse config file: %v", err)
 	}
 
-	log.Println("Configuration file created successfully at:", filePath)
-	return &exampleConfig, nil
+	configData.Manifest = *manifest
+
+	return &configData, nil
 }

config/models.go

@@ -1,24 +1,29 @@
 package config
 
-import "fmt"
+import (
+	"fmt"
+
+	"github.com/SuperALKALINEdroiD/timelyDB/manifest"
+)
 
 type StoreMode string
 
 type DatabaseConfig struct {
-	StoreName                string         `json:"dbName"`
-	Port                     int            `json:"port"`
-	IsLockEnabled            bool           `json:"isLockEnabled"`
-	TimelyConfig             TimelyConfig   `json:"timelyConfig"`
-	Nodes                    []NodeConfig   `json:"nodes"`
-	NodeCount                int            `json:"nodeCount"`
-	Mode                     StoreMode      `json:"mode"`
-	InMemoryStorageThreshold int64          `json:"inMemoryStorageThreshold"`
-	MetaDataConfig           MetaDataConfig `json:"metaDataConfig"`
+	StoreName                string            `json:"dbName" validate:"required"`
+	Port                     int               `json:"port" validate:"required,gt=0"`
+	IsLockEnabled            bool              `json:"isLockEnabled"`
+	TimelyConfig             TimelyConfig      `json:"timelyConfig"`
+	Nodes                    []NodeConfig      `json:"nodes" validate:"required"`
+	NodeCount                int               `json:"nodeCount" validate:"required,gt=0"`
+	Mode                     StoreMode         `json:"mode" validate:"required"`
+	InMemoryStorageThreshold int64             `json:"inMemoryStorageThreshold" validate:"required,gt=0"`
+	MetaDataConfig           MetaDataConfig    `json:"metaDataConfig" validate:"required"`
+	Manifest                 manifest.Manifest `validate:"omitempty"`
 }
 
 type MetaDataConfig struct {
 	State   NodeState `json:"state"`
-	WALPath string    `json:"walPath"`
+	WALName string    `json:"walPath"`
 }
 
 type NodeState int
@@ -62,10 +67,10 @@ func GenerateExampleConfig(nodeCount int, host string) DatabaseConfig {
 		Mode:                     KV,
 		Nodes:                    nodes,
 		NodeCount:                nodeCount,
-		InMemoryStorageThreshold: 2000,
+		InMemoryStorageThreshold: 2024, // 2MB
 		MetaDataConfig: MetaDataConfig{
 			State:   NodeStateReady,
-			WALPath: "/var/lib/db/wal",
+			WALName: "wal-storage",
 		},
 	}
 }

config/restricted.go

@@ -0,0 +1,39 @@
+package config
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+)
+
+var restrictedWords map[string]struct{}
+
+func LoadRestrictedWords(filePath string) error {
+	restrictedWords = make(map[string]struct{})
+
+	file, err := os.Open(filePath)
+	if err != nil {
+		return fmt.Errorf("failed to open restricted words file: %v", err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		word := strings.TrimSpace(scanner.Text())
+		if word != "" {
+			restrictedWords[word] = struct{}{}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("failed to read restricted words file: %v", err)
+	}
+
+	return nil
+}
+
+func IsARestrictedWord(word string) bool {
+	_, exists := restrictedWords[word]
+	return exists
+}

core/app.go

@@ -13,6 +13,8 @@ type App struct {
 	Config       *config.DatabaseConfig
 	Router       *chi.Mux
 	Nodes        []*nodes.Node
+	NodeByID     map[string]*nodes.Node
+	NodeClients  map[string]nodes.NodeServiceClient
 	NodeHashInfo hashing.NodeHash
 	WAL          storage.WAL
 }

go.mod

@@ -11,9 +11,30 @@ require (
 )
 
 require (
+	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
-	golang.org/x/net v0.32.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
+	github.com/go-openapi/analysis v0.23.0 // indirect
+	github.com/go-openapi/errors v0.22.0 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/loads v0.22.0 // indirect
+	github.com/go-openapi/spec v0.21.0 // indirect
+	github.com/go-openapi/strfmt v0.23.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.26.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/oklog/ulid v1.3.1 // indirect
+	go.mongodb.org/mongo-driver v1.14.0 // indirect
+	golang.org/x/crypto v0.33.0 // indirect
+	golang.org/x/net v0.34.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
+	golang.org/x/text v0.22.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )