From 2f84db7441e8ee50d86ffd33071f1880a4ffe555 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Thu, 16 Jul 2020 22:22:23 +0530 Subject: [PATCH 01/56] added go mod, golang ci --- go.mod | 11 +++++++ go.sum | 37 ++++++++++++++++++++++ http_handlers.go | 78 ++++++++++++++++++++++++++++++++-------------- main.go | 36 +++++++++++++-------- server.go | 4 +++ sync.go | 69 +++++++++++++++++++++++++++++++--------- varnam_handlers.go | 46 ++++++++++++++++++++------- varnam_learn.go | 3 ++ 8 files changed, 222 insertions(+), 62 deletions(-) create mode 100644 go.mod create mode 100644 go.sum diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..cd3e04f --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module github.com/varnamproject/varnamd + +go 1.14 + +require ( + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e + github.com/golang/protobuf v1.4.2 // indirect + github.com/gorilla/mux v1.7.4 + github.com/mattn/go-sqlite3 v1.14.0 + github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a9b9603 --- /dev/null +++ b/go.sum @@ -0,0 +1,37 @@ +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932 h1:Z6VhoopafnsFFYf8KwGty1dmJteTCM/mLGRAvc81QPk= +github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932/go.mod h1:p0y0nS7FVUFgPK6dOkEY/NiOI7+aBeYgrl4ACVFuLIo= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= diff --git a/http_handlers.go b/http_handlers.go index 19e448e..38f9bed 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -3,6 +3,7 @@ package main import ( "bytes" "compress/gzip" + "context" "encoding/json" "errors" "fmt" @@ -24,6 +25,7 @@ var errCacheSkipped = errors.New("cache skipped") // Data will be set if the cache returns CacheSkipped type varnamCacheContext struct { Data []byte + context.Context } type standardResponse struct { @@ -34,10 +36,12 @@ type standardResponse struct { func newStandardResponse(err string) standardResponse { s := standardResponse{Success: true, Error: "", At: time.Now().UTC().String()} + if err != "" { s.Error = err s.Success = false } + return s } @@ -67,6 +71,7 @@ type requestParams struct { func parseParams(r *http.Request) *requestParams { params := mux.Vars(r) downloadStart, _ := strconv.Atoi(params["downloadStart"]) + return &requestParams{langCode: params["langCode"], word: params["word"], downloadStart: downloadStart} } @@ -74,6 +79,7 @@ func parseParams(r *http.Request) *requestParams { func corsHandler(h http.Handler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") + if r.Method == "OPTIONS" { w.WriteHeader(http.StatusOK) } else { @@ -94,49 +100,54 @@ func recoverHandler(next http.Handler) http.Handler { }() next.ServeHTTP(w, r) } + return http.HandlerFunc(fn) } func renderError(w http.ResponseWriter, err error) { w.Header().Set("Content-Type", "application/json; charset=utf-8") + if err != nil { w.WriteHeader(http.StatusBadRequest) + errorData := newStandardResponse(err.Error()) - json.NewEncoder(w).Encode(errorData) + _ = json.NewEncoder(w).Encode(errorData) } } func renderGzippedJSON(w http.ResponseWriter, data []byte) { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("Content-Encoding", "gzip") - w.Write(data) + _, _ = w.Write(data) } func renderJSON(w http.ResponseWriter, data interface{}) { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(data) + _ = json.NewEncoder(w).Encode(data) } func getLanguageAndWord(r *http.Request) (langCode string, word string) { params := mux.Vars(r) langCode = params["langCode"] word = params["word"] + return } -func getLangCode(r *http.Request) string { - params := mux.Vars(r) - return params["langCode"] -} +// func getLangCode(r *http.Request) string { +// params := mux.Vars(r) +// return params["langCode"] +// } -func getWord(r *http.Request) string { - params := mux.Vars(r) - return params["word"] -} +// func getWord(r *http.Request) string { +// params := mux.Vars(r) +// return params["word"] +// } func statusHandler(w http.ResponseWriter, r *http.Request) { - uptime := time.Now().Sub(startedAt) + uptime := time.Since(startedAt) + resp := struct { Version string `json:"version"` Uptime string `json:"uptime"` @@ -153,6 +164,7 @@ func statusHandler(w http.ResponseWriter, r *http.Request) { func transliterationHandler(w http.ResponseWriter, r *http.Request) { langCode, word := getLanguageAndWord(r) words, err := transliterate(langCode, word) + if err != nil { renderError(w, err) } else { @@ -164,6 +176,7 @@ func transliterationHandler(w http.ResponseWriter, r *http.Request) { func reverseTransliterationHandler(w http.ResponseWriter, r *http.Request) { langCode, word := getLanguageAndWord(r) result, err := reveseTransliterate(langCode, word) + if err != nil { renderError(w, err) } else { @@ -180,13 +193,14 @@ func reverseTransliterationHandler(w http.ResponseWriter, r *http.Request) { func metadataHandler(w http.ResponseWriter, r *http.Request) { schemeIdentifier, _ := getLanguageAndWord(r) - getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { + _, _ = getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { details, err := handle.GetCorpusDetails() if err != nil { renderError(w, err) return } renderJSON(w, &metaResponse{Result: details, standardResponse: newStandardResponse("")}) + return }) } @@ -194,23 +208,25 @@ func metadataHandler(w http.ResponseWriter, r *http.Request) { func downloadHandler(w http.ResponseWriter, r *http.Request) { params := parseParams(r) if params.downloadStart < 0 { - renderError(w, errors.New("Invalid parameters")) + renderError(w, errors.New("invalid parameters")) return } - fillCache := func(ctx groupcache.Context, key string, dest groupcache.Sink) error { + fillCache := func(ctx context.Context, key string, dest groupcache.Sink) error { // cache miss, fetch from DB // key is in the form + parts := strings.Split(key, "+") - schemeId := parts[0] + schemeID := parts[0] downloadStart, _ := strconv.Atoi(parts[1]) - words, err := getWords(schemeId, downloadStart) + words, err := getWords(schemeID, downloadStart) + if err != nil { return err } response := downloadResponse{Count: len(words), Words: words, standardResponse: newStandardResponse("")} b, err := json.Marshal(response) + if err != nil { return err } @@ -218,17 +234,21 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) { // gzipping the response so that it can be served directly var gb bytes.Buffer gWriter := gzip.NewWriter(&gb) - defer gWriter.Close() - gWriter.Write(b) - gWriter.Flush() + + defer func() { _ = gWriter.Close() }() + + _, _ = gWriter.Write(b) + _ = gWriter.Flush() if len(words) < downloadPageSize { - varnamCtx := ctx.(*varnamCacheContext) + varnamCtx, _ := ctx.(*varnamCacheContext) varnamCtx.Data = gb.Bytes() + return errCacheSkipped } - dest.SetBytes(gb.Bytes()) + _ = dest.SetBytes(gb.Bytes()) + return nil } @@ -247,7 +267,9 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) { cacheGroup := cacheGroups[params.langCode] ctx := varnamCacheContext{} + var data []byte + if err := cacheGroup.Get(&ctx, fmt.Sprintf("%s+%d", params.langCode, params.downloadStart), groupcache.AllocatingByteSliceSink(&data)); err != nil { if err == errCacheSkipped { renderGzippedJSON(w, ctx.Data) @@ -255,6 +277,7 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) { } renderError(w, err) + return } @@ -266,25 +289,32 @@ func languagesHandler(w http.ResponseWriter, r *http.Request) { } func learnHandler(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) var args Args + + decoder := json.NewDecoder(r.Body) + if e := decoder.Decode(&args); e != nil { renderError(w, e) return } ch, ok := learnChannels[args.LangCode] + if !ok { - renderError(w, errors.New("Unable to find language")) + renderError(w, errors.New("unable to find language")) + return } + go func(word string) { ch <- word }(args.Word) + renderJSON(w, "success") } func toggleDownloadEnabledStatus(w http.ResponseWriter, r *http.Request, status bool) { params := parseParams(r) err := varnamdConfig.setDownloadStatus(params.langCode, status) + if err != nil { renderError(w, err) } else { diff --git a/main.go b/main.go index 9318a68..b75144b 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,6 @@ package main import ( - "errors" "flag" "fmt" "log" @@ -34,34 +33,38 @@ var ( // varnamd configurations // this is populated from various command line flags type config struct { - upstream string - schemesToDownload map[string]bool - syncIntervalInSecs time.Duration + upstream string + schemesToDownload map[string]bool + syncInterval time.Duration } func initConfig() *config { toDownload := make(map[string]bool) schemes := strings.Split(downloadEnabledSchemes, ",") + for _, scheme := range schemes { s := strings.TrimSpace(scheme) + if s != "" { if !isValidSchemeIdentifier(s) { panic(fmt.Sprintf("%s is not a valid libvarnam supported scheme", s)) } + toDownload[s] = true } } return &config{upstream: upstreamURL, schemesToDownload: toDownload, - syncIntervalInSecs: time.Duration(syncIntervalInSecs)} + syncInterval: time.Duration(syncIntervalInSecs)} } func (c *config) setDownloadStatus(langCode string, status bool) error { if !isValidSchemeIdentifier(langCode) { - return errors.New(fmt.Sprintf("%s is not a valid libvarnam supported scheme", langCode)) + return fmt.Errorf("%s is not a valid libvarnam supported scheme", langCode) } c.schemesToDownload[langCode] = status + if status { // when varnamd was started without any langcodes to sync, the dispatcher won't be running // in that case, we need to start the dispatcher since we have a new lang code to download now @@ -74,15 +77,16 @@ func (c *config) setDownloadStatus(langCode string, status bool) error { func getConfigDir() string { if runtime.GOOS == "windows" { return path.Join(os.Getenv("localappdata"), ".varnamd") - } else { - return path.Join(os.Getenv("HOME"), ".varnamd") } + + return path.Join(os.Getenv("HOME"), ".varnamd") } func getLogsDir() string { d := getConfigDir() logsDir := path.Join(d, "logs") - err := os.MkdirAll(logsDir, 0777) + err := os.MkdirAll(logsDir, 0750) + if err != nil { panic(err) } @@ -93,10 +97,12 @@ func getLogsDir() string { func redirectLogToFile() { year, month, day := time.Now().Date() logfile := path.Join(getLogsDir(), fmt.Sprintf("%d-%d-%d.log", year, month, day)) - f, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + f, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { panic(err) } + log.SetOutput(f) } @@ -123,22 +129,26 @@ func syncRequired() bool { // Starts the sync process only if it is not running func startSyncDispatcher() { if syncRequired() && !syncDispatcherRunning { - sync := newSyncDispatcher(varnamdConfig.syncIntervalInSecs * time.Second) + sync := newSyncDispatcher(varnamdConfig.syncInterval * time.Second) sync.start() sync.runNow() // run one round of sync immediatly rather than waiting for the next interval to occur + syncDispatcherRunning = true } } func main() { - runtime.GOMAXPROCS(runtime.NumCPU()) - flag.Parse() varnamdConfig = initConfig() startedAt = time.Now() + + runtime.GOMAXPROCS(runtime.NumCPU()) + flag.Parse() + if version { fmt.Println(varnamdVersion) os.Exit(0) } + if logToFile { redirectLogToFile() } diff --git a/server.go b/server.go index 6156b6e..fe7c4f9 100644 --- a/server.go +++ b/server.go @@ -12,7 +12,9 @@ import ( func startDaemon() { initLanguageChannels() initLearnChannels() + r := mux.NewRouter() + r.HandleFunc("/tl/{langCode}/{word}", transliterationHandler).Methods("GET") r.HandleFunc("/rtl/{langCode}/{word}", reverseTransliterationHandler).Methods("GET") r.HandleFunc("/meta/{langCode}", metadataHandler).Methods("GET") @@ -20,6 +22,7 @@ func startDaemon() { r.HandleFunc("/learn", learnHandler).Methods("POST") r.HandleFunc("/languages", languagesHandler).Methods("GET") r.HandleFunc("/status", statusHandler).Methods("GET") + if enableInternalApis { r.HandleFunc("/sync/download/{langCode}/enable", enableDownload).Methods("POST") r.HandleFunc("/sync/download/{langCode}/disable", disableDownload).Methods("POST") @@ -29,6 +32,7 @@ func startDaemon() { address := fmt.Sprintf("%s:%d", host, port) log.Printf("Listening on %s", address) + if enableSSL { if err := http.ListenAndServeTLS(address, certFilePath, keyFilePath, recoverHandler(corsHandler(r))); err != nil { log.Fatalln(err) diff --git a/sync.go b/sync.go index 52f0b53..75729d5 100644 --- a/sync.go +++ b/sync.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "path" + "path/filepath" "strconv" "strings" "time" @@ -21,21 +22,25 @@ type syncDispatcher struct { ticker *time.Ticker } -func newSyncDispatcher(intervalInSeconds time.Duration) *syncDispatcher { - return &syncDispatcher{ticker: time.NewTicker(intervalInSeconds), force: make(chan bool), quit: make(chan struct{})} +func newSyncDispatcher(interval time.Duration) *syncDispatcher { + return &syncDispatcher{ticker: time.NewTicker(interval), force: make(chan bool), quit: make(chan struct{})} } func (s *syncDispatcher) start() { err := createSyncMetadataDir() + if err != nil { fmt.Printf("Failed to create sync metadata directory. Sync will be disabled.\nActual error: %s\n", err.Error()) return } + for s := range varnamdConfig.schemesToDownload { // download cache directory for each of the languages err = createLearnQueueDir(s) + if err != nil { - fmt.Printf("Failed to create learn queue directory for '%s'. Sync will be disabled.\nActual error: %s\n", err.Error()) + fmt.Printf("Failed to create learn queue directory for '%s'. Sync will be disabled.\nActual error: %s\n", s, err.Error()) + return } } @@ -55,9 +60,9 @@ func (s *syncDispatcher) start() { }() } -func (s *syncDispatcher) stop() { - close(s.quit) -} +// func (s *syncDispatcher) stop() { +// close(s.quit) +// } func (s *syncDispatcher) runNow() { s.force <- true @@ -106,12 +111,14 @@ func syncWordsFromUpstreamFor(langCode string) { func addFilesFromLocalLearnQueue(langCode string, files []string, filesToLearn chan string) { if files != nil { log.Printf("Adding %d files to learn from local learn queue\n", len(files)) + for _, f := range files { filesToLearn <- f } } else { log.Printf("Local learn queue for '%s' is empty", langCode) } + close(filesToLearn) } @@ -119,16 +126,22 @@ func downloadAllWords(langCode string, corpusSize int, output chan string) { for { offset := getDownloadOffset(langCode) log.Printf("Offset: %d\n", offset) + if offset >= corpusSize { break } + filePath, err := downloadWordsAndUpdateOffset(langCode, offset) + if err != nil { break } + output <- filePath } + log.Println("Local copy is upto date. No need to download from upstream") + close(output) } @@ -139,9 +152,11 @@ func learnAll(langCode string, filesToLearn chan string) { } func learnFromFile(langCode, fileToLearn string) { - log.Printf("Learning from %s\n", fileToLearn) start := time.Now() - getOrCreateHandler(langCode, func(handle *libvarnam.Varnam) (data interface{}, err error) { + + log.Printf("Learning from %s\n", fileToLearn) + + _, _ = getOrCreateHandler(langCode, func(handle *libvarnam.Varnam) (data interface{}, err error) { learnStatus, err := handle.LearnFromFile(fileToLearn) end := time.Now() if err != nil { @@ -178,25 +193,33 @@ func downloadWordsAndUpdateOffset(langCode string, offset int) (string, error) { func getCorpusDetails(langCode string) (*libvarnam.CorpusDetails, error) { url := fmt.Sprintf("%s/meta/%s", varnamdConfig.upstream, langCode) log.Printf("Fetching corpus details for '%s'\n", langCode) + var m metaResponse err := getJSONResponse(url, &m) + if err != nil { return nil, err } + log.Printf("Corpus size: %d\n", m.Result.WordsCount) + return m.Result, nil } // Downloads words from upstream starting from the specified offset and stores it locally in the learn queue // Returns the number of words downloaded, local file path and error if any func downloadWords(langCode string, offset int) (totalWordsDownloaded int, downloadedFilePath string, err error) { - url := fmt.Sprintf("%s/download/%s/%d", varnamdConfig.upstream, langCode, offset) var response downloadResponse + + url := fmt.Sprintf("%s/download/%s/%d", varnamdConfig.upstream, langCode, offset) err = getJSONResponse(url, &response) + if err != nil { return 0, "", err } + downloadedFilePath, err = transformAndPersistWords(langCode, offset, &response) + if err != nil { log.Printf("Download was successful, but failed to persist to local learn queue. %s\n", err.Error()) return 0, "", err @@ -208,10 +231,12 @@ func downloadWords(langCode string, offset int) (totalWordsDownloaded int, downl func transformAndPersistWords(langCode string, offset int, dresp *downloadResponse) (string, error) { learnQueueDir := getLearnQueueDir(langCode) targetFile, err := os.Create(path.Join(learnQueueDir, fmt.Sprintf("%s.%d", langCode, offset))) + if err != nil { return "", err } - defer targetFile.Close() + + defer func() { _ = targetFile.Close() }() for _, word := range dresp.Words { _, err = targetFile.WriteString(fmt.Sprintf("%s %d\n", word.Word, word.Confidence)) @@ -219,13 +244,16 @@ func transformAndPersistWords(langCode string, offset int, dresp *downloadRespon return "", err } } + return targetFile.Name(), nil } func getFilesFromLearnQueue(langCode string) []string { var files []string + learnQueueDir := getLearnQueueDir(langCode) queueContents, err := ioutil.ReadDir(learnQueueDir) + if err != nil { return nil } @@ -241,22 +269,29 @@ func getFilesFromLearnQueue(langCode string) []string { func getJSONResponse(url string, output interface{}) error { log.Printf("GET: '%s'\n", url) + resp, err := http.Get(url) + if err != nil { return err } - defer resp.Body.Close() + + defer func() { _ = resp.Body.Close() }() + jsonDecoder := json.NewDecoder(resp.Body) err = jsonDecoder.Decode(output) + if err != nil { return err } + return nil } func getDownloadOffset(langCode string) int { filePath := getDownloadOffsetMetadataFile(langCode) - content, err := ioutil.ReadFile(filePath) + content, err := ioutil.ReadFile(filepath.Clean(filePath)) + if err != nil { return 0 } @@ -281,30 +316,36 @@ func getDownloadOffsetMetadataFile(langCode string) string { func createLearnQueueDir(langCode string) error { queueDir := getLearnQueueDir(langCode) - err := os.MkdirAll(queueDir, 0777) + err := os.MkdirAll(queueDir, 0750) + if err != nil { return err } + return nil } func getLearnQueueDir(langCode string) string { syncDir := getSyncMetadataDir() queueDir := path.Join(syncDir, fmt.Sprintf("%s.learn.queue", langCode)) + return queueDir } func createSyncMetadataDir() error { syncDir := getSyncMetadataDir() - err := os.MkdirAll(syncDir, 0777) + err := os.MkdirAll(syncDir, 0750) + if err != nil { return err } + return nil } func getSyncMetadataDir() string { configDir := getConfigDir() syncDir := path.Join(configDir, "sync") + return syncDir } diff --git a/varnam_handlers.go b/varnam_handlers.go index 3bdd173..f6a7188 100644 --- a/varnam_handlers.go +++ b/varnam_handlers.go @@ -13,7 +13,7 @@ import ( ) type word struct { - Id int `json:"id"` + ID int `json:"id"` Confidence int `json:"confidence"` Word string `json:"word"` } @@ -24,8 +24,8 @@ var ( mutex *sync.Mutex once sync.Once schemeDetails = libvarnam.GetAllSchemeDetails() - peers = groupcache.NewHTTPPool("http://localhost") cacheGroups = make(map[string]*groupcache.Group) + // peers = groupcache.NewHTTPPool("http://localhost") ) func isValidSchemeIdentifier(id string) bool { @@ -34,6 +34,7 @@ func isValidSchemeIdentifier(id string) bool { return true } } + return false } @@ -41,9 +42,11 @@ func initLanguageChannels() { languageChannels = make(map[string]chan *libvarnam.Varnam) channelsCount = make(map[string]int) mutex = &sync.Mutex{} + for _, scheme := range schemeDetails { languageChannels[scheme.Identifier] = make(chan *libvarnam.Varnam, maxHandleCount) channelsCount[scheme.Identifier] = maxHandleCount + for i := 0; i < maxHandleCount; i++ { handle, err := libvarnam.Init(scheme.Identifier) if err != nil { @@ -57,22 +60,27 @@ func initLanguageChannels() { func getOrCreateHandler(schemeIdentifier string, f func(handle *libvarnam.Varnam) (data interface{}, err error)) (data interface{}, err error) { ch, ok := languageChannels[schemeIdentifier] if !ok { - return nil, errors.New("Invalid scheme identifier") + return nil, errors.New("invalid scheme identifier") } select { case handle := <-ch: data, err = f(handle) + go func() { ch <- handle }() case <-time.After(800 * time.Millisecond): var handle *libvarnam.Varnam handle, err = libvarnam.Init(schemeIdentifier) + if err != nil { log.Println(err) - return nil, errors.New("Unable to initialize varnam handle") + return nil, errors.New("unable to initialize varnam handle") } + data, err = f(handle) + go sendHandlerToChannel(schemeIdentifier, handle, ch) } + return } @@ -89,10 +97,12 @@ func getWords(schemeIdentifier string, downloadStart int) ([]*word, error) { }) db, err := sql.Open("sqlite3", filepath.(string)) + if err != nil { return nil, err } - defer db.Close() + + defer func() { _ = db.Close() }() // Making an index for all learned words so that download is faster // this needs to be removed when there is more clarity on how learned words needs to be handled @@ -101,19 +111,30 @@ func getWords(schemeIdentifier string, downloadStart int) ([]*word, error) { return nil, err } - q := "select id, word, confidence from words where id in (select distinct(word_id) from patterns_content where learned = 1) order by id asc limit ? offset ?;" + q := `select id, word, confidence from words where id in (select distinct(word_id) from patterns_content where learned = 1) order by id asc limit ? offset ?;` rows, err := db.Query(q, downloadPageSize, downloadStart) + if err != nil { return nil, err } - defer rows.Close() + + if rows.Err() != nil { + return nil, rows.Err() + } + + defer func() { _ = rows.Close() }() var words []*word + for rows.Next() { - var id, confidence int - var _word string - rows.Scan(&id, &_word, &confidence) - words = append(words, &word{Id: id, Confidence: confidence, Word: _word}) + var ( + id, confidence int + _word string + ) + + _ = rows.Scan(&id, &_word, &confidence) + + words = append(words, &word{ID: id, Confidence: confidence, Word: _word}) } return words, nil @@ -130,11 +151,14 @@ func sendHandlerToChannel(schemeIdentifier string, handle *libvarnam.Varnam, ch mutex.Lock() count := channelsCount[schemeIdentifier] mutex.Unlock() + if count == maxHandleCount { log.Printf("Throw away handle") handle.Destroy() + return } + select { case ch <- handle: mutex.Lock() diff --git a/varnam_learn.go b/varnam_learn.go index 6d23eb6..bb8143d 100644 --- a/varnam_learn.go +++ b/varnam_learn.go @@ -6,6 +6,7 @@ import ( "github.com/varnamproject/libvarnam-golang" ) +// Args to read. type Args struct { LangCode string `json:"lang"` Word string `json:"word"` @@ -20,9 +21,11 @@ func initLearnChannels() { for _, scheme := range schemeDetails { learnChannels[scheme.Identifier] = make(chan string, 100) handle, err := libvarnam.Init(scheme.Identifier) + if err != nil { log.Fatal("Unable to initialize varnam for lang", scheme.LangCode) } + go listenForWords(scheme.Identifier, handle) } } From b55980f6c0d8fcbe203fc6a6246e3fc73cc0ecad Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Thu, 16 Jul 2020 23:39:07 +0530 Subject: [PATCH 02/56] migrated to echo --- go.mod | 8 +- go.sum | 58 ++++++++++++- http_handlers.go | 222 ++++++++++++++++++++++------------------------- server.go | 47 +++++----- 4 files changed, 185 insertions(+), 150 deletions(-) diff --git a/go.mod b/go.mod index cd3e04f..a7fbcdd 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,13 @@ go 1.14 require ( github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/protobuf v1.4.2 // indirect - github.com/gorilla/mux v1.7.4 + github.com/labstack/echo/v4 v4.1.16 + github.com/mattn/go-colorable v0.1.7 // indirect github.com/mattn/go-sqlite3 v1.14.0 + github.com/valyala/fasttemplate v1.2.0 // indirect github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932 + golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect + golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect + golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect + golang.org/x/text v0.3.3 // indirect ) diff --git a/go.sum b/go.sum index a9b9603..a234f57 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -13,19 +17,65 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz0o= +github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= +github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.0 h1:y3yXRCoDvC2HTtIHvL2cc7Zd+bqA+zqDO6oQzsJO07E= +github.com/valyala/fasttemplate v1.2.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932 h1:Z6VhoopafnsFFYf8KwGty1dmJteTCM/mLGRAvc81QPk= github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932/go.mod h1:p0y0nS7FVUFgPK6dOkEY/NiOI7+aBeYgrl4ACVFuLIo= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -35,3 +85,7 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/http_handlers.go b/http_handlers.go index 38f9bed..0c26ba0 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -15,7 +15,7 @@ import ( "time" "github.com/golang/groupcache" - "github.com/gorilla/mux" + "github.com/labstack/echo/v4" "github.com/varnamproject/libvarnam-golang" ) @@ -34,15 +34,8 @@ type standardResponse struct { At string `json:"at"` } -func newStandardResponse(err string) standardResponse { - s := standardResponse{Success: true, Error: "", At: time.Now().UTC().String()} - - if err != "" { - s.Error = err - s.Success = false - } - - return s +func newStandardResponse() standardResponse { + return standardResponse{Success: true, At: time.Now().UTC().String()} } type transliterationResponse struct { @@ -62,20 +55,6 @@ type downloadResponse struct { standardResponse } -type requestParams struct { - langCode string - word string - downloadStart int -} - -func parseParams(r *http.Request) *requestParams { - params := mux.Vars(r) - downloadStart, _ := strconv.Atoi(params["downloadStart"]) - - return &requestParams{langCode: params["langCode"], word: params["word"], - downloadStart: downloadStart} -} - func corsHandler(h http.Handler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") @@ -104,37 +83,6 @@ func recoverHandler(next http.Handler) http.Handler { return http.HandlerFunc(fn) } -func renderError(w http.ResponseWriter, err error) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - - if err != nil { - w.WriteHeader(http.StatusBadRequest) - - errorData := newStandardResponse(err.Error()) - _ = json.NewEncoder(w).Encode(errorData) - } -} - -func renderGzippedJSON(w http.ResponseWriter, data []byte) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.Header().Set("Content-Encoding", "gzip") - _, _ = w.Write(data) -} - -func renderJSON(w http.ResponseWriter, data interface{}) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - _ = json.NewEncoder(w).Encode(data) -} - -func getLanguageAndWord(r *http.Request) (langCode string, word string) { - params := mux.Vars(r) - langCode = params["langCode"] - word = params["word"] - - return -} - // func getLangCode(r *http.Request) string { // params := mux.Vars(r) // return params["langCode"] @@ -145,7 +93,7 @@ func getLanguageAndWord(r *http.Request) (langCode string, word string) { // return params["word"] // } -func statusHandler(w http.ResponseWriter, r *http.Request) { +func handleStatus(c echo.Context) error { uptime := time.Since(startedAt) resp := struct { @@ -155,61 +103,80 @@ func statusHandler(w http.ResponseWriter, r *http.Request) { }{ varnamdVersion, uptime.String(), - newStandardResponse(""), + newStandardResponse(), } - renderJSON(w, resp) + return c.JSON(http.StatusOK, resp) } -func transliterationHandler(w http.ResponseWriter, r *http.Request) { - langCode, word := getLanguageAndWord(r) +func handleTransliteration(c echo.Context) error { + var ( + langCode = c.Param("langCode") + word = c.Param("word") + ) + + // langCode, word := getLanguageAndWord(r) words, err := transliterate(langCode, word) if err != nil { - renderError(w, err) - } else { - renderJSON(w, - transliterationResponse{standardResponse: newStandardResponse(""), Result: words.([]string), Input: word}) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) } + + return c.JSON(http.StatusOK, transliterationResponse{standardResponse: newStandardResponse(), Result: words.([]string), Input: word}) } -func reverseTransliterationHandler(w http.ResponseWriter, r *http.Request) { - langCode, word := getLanguageAndWord(r) +func handleReverseTransliteration(c echo.Context) error { + var ( + langCode = c.Param("langCode") + word = c.Param("word") + ) + result, err := reveseTransliterate(langCode, word) if err != nil { - renderError(w, err) - } else { - response := struct { - standardResponse - Result string `json:"result"` - }{ - newStandardResponse(""), - result.(string), - } - renderJSON(w, response) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) } + + response := struct { + standardResponse + Result string `json:"result"` + }{ + newStandardResponse(), + result.(string), + } + + return c.JSON(http.StatusOK, response) } -func metadataHandler(w http.ResponseWriter, r *http.Request) { - schemeIdentifier, _ := getLanguageAndWord(r) - _, _ = getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { +func handleMetadata(c echo.Context) error { + var schemeIdentifier = c.Param("langCode") + + data, err := getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { details, err := handle.GetCorpusDetails() + if err != nil { - renderError(w, err) - return + return nil, err } - renderJSON(w, &metaResponse{Result: details, standardResponse: newStandardResponse("")}) - return + return &metaResponse{Result: details, standardResponse: newStandardResponse()}, nil }) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } + + return c.JSON(http.StatusOK, data) } -func downloadHandler(w http.ResponseWriter, r *http.Request) { - params := parseParams(r) - if params.downloadStart < 0 { - renderError(w, errors.New("invalid parameters")) - return +func handleDownload(c echo.Context) error { + var ( + langCode = c.Param("langCode") + // word = c.Param("word") + ) + + start, _ := strconv.Atoi(c.Param("downloadStart")) + + if start < 0 { + return echo.NewHTTPError(http.StatusBadRequest, "invalid parameter") } fillCache := func(ctx context.Context, key string, dest groupcache.Sink) error { @@ -224,7 +191,7 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) { return err } - response := downloadResponse{Count: len(words), Words: words, standardResponse: newStandardResponse("")} + response := downloadResponse{Count: len(words), Words: words, standardResponse: newStandardResponse()} b, err := json.Marshal(response) if err != nil { @@ -265,67 +232,82 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) { } }) - cacheGroup := cacheGroups[params.langCode] + cacheGroup := cacheGroups[langCode] ctx := varnamCacheContext{} var data []byte - if err := cacheGroup.Get(&ctx, fmt.Sprintf("%s+%d", params.langCode, params.downloadStart), groupcache.AllocatingByteSliceSink(&data)); err != nil { + if err := cacheGroup.Get(&ctx, fmt.Sprintf("%s+%d", langCode, start), groupcache.AllocatingByteSliceSink(&data)); err != nil { if err == errCacheSkipped { - renderGzippedJSON(w, ctx.Data) - return - } + c.Response().Header().Set("Content-Type", "application/json; charset=utf-8") + c.Response().Header().Set("Content-Encoding", "gzip") - renderError(w, err) + return c.JSON(http.StatusOK, ctx.Data) + } - return + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } - renderGzippedJSON(w, data) + c.Response().Header().Set("Content-Type", "application/json; charset=utf-8") + c.Response().Header().Set("Content-Encoding", "gzip") + + return c.JSON(http.StatusOK, data) } -func languagesHandler(w http.ResponseWriter, r *http.Request) { - renderJSON(w, schemeDetails) +func handleLanguages(c echo.Context) error { + return c.JSON(http.StatusOK, schemeDetails) } -func learnHandler(w http.ResponseWriter, r *http.Request) { +func handlLearn(c echo.Context) error { var args Args - decoder := json.NewDecoder(r.Body) - - if e := decoder.Decode(&args); e != nil { - renderError(w, e) - return + if err := c.Bind(&args); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } ch, ok := learnChannels[args.LangCode] if !ok { - renderError(w, errors.New("unable to find language")) - - return + return echo.NewHTTPError(http.StatusBadRequest, "unable to find language") } go func(word string) { ch <- word }(args.Word) - renderJSON(w, "success") + return c.JSON(http.StatusOK, "success") } -func toggleDownloadEnabledStatus(w http.ResponseWriter, r *http.Request, status bool) { - params := parseParams(r) - err := varnamdConfig.setDownloadStatus(params.langCode, status) +func toggleDownloadEnabledStatus(langCode string, status bool) (interface{}, error) { + if err := varnamdConfig.setDownloadStatus(langCode, status); err != nil { + return nil, err + } + + return newStandardResponse(), nil +} + +func handleEnableDownload(c echo.Context) error { + var ( + langCode = c.Param("langCode") + ) + + data, err := toggleDownloadEnabledStatus(langCode, true) if err != nil { - renderError(w, err) - } else { - renderJSON(w, newStandardResponse("")) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } -} -func enableDownload(w http.ResponseWriter, r *http.Request) { - toggleDownloadEnabledStatus(w, r, true) + return c.JSON(http.StatusOK, data) } -func disableDownload(w http.ResponseWriter, r *http.Request) { - toggleDownloadEnabledStatus(w, r, false) +func handleDisableDownload(c echo.Context) error { + var ( + langCode = c.Param("langCode") + ) + + data, err := toggleDownloadEnabledStatus(langCode, false) + + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } + + return c.JSON(http.StatusOK, data) } diff --git a/server.go b/server.go index fe7c4f9..d595f70 100644 --- a/server.go +++ b/server.go @@ -5,53 +5,46 @@ import ( "log" "net/http" "os" + "path/filepath" - "github.com/gorilla/mux" + "github.com/labstack/echo/v4" ) func startDaemon() { initLanguageChannels() initLearnChannels() - r := mux.NewRouter() + e := echo.New() - r.HandleFunc("/tl/{langCode}/{word}", transliterationHandler).Methods("GET") - r.HandleFunc("/rtl/{langCode}/{word}", reverseTransliterationHandler).Methods("GET") - r.HandleFunc("/meta/{langCode}", metadataHandler).Methods("GET") - r.HandleFunc("/download/{langCode}/{downloadStart}", downloadHandler).Methods("GET") - r.HandleFunc("/learn", learnHandler).Methods("POST") - r.HandleFunc("/languages", languagesHandler).Methods("GET") - r.HandleFunc("/status", statusHandler).Methods("GET") + e.GET("/tl/{langCode}/{word}", handleTransliteration) + e.GET("/rtl/{langCode}/{word}", handleReverseTransliteration) + e.GET("/meta/{langCode}", handleMetadata) + e.GET("/download/{langCode}/{downloadStart}", handleDownload) + e.POST("/learn", handlLearn) + e.GET("/languages", handleLanguages) + e.GET("/status", handleStatus) - if enableInternalApis { - r.HandleFunc("/sync/download/{langCode}/enable", enableDownload).Methods("POST") - r.HandleFunc("/sync/download/{langCode}/disable", disableDownload).Methods("POST") + if _, err := os.Stat(uiDir); err != nil { + log.Fatal("UI path doesnot exist", err) } - addUI(r) + e.Static("/", filepath.Clean(uiDir)) + + if enableInternalApis { + e.POST("/sync/download/{langCode}/enable", handleEnableDownload) + e.POST("/sync/download/{langCode}/disable", handleDisableDownload) + } address := fmt.Sprintf("%s:%d", host, port) log.Printf("Listening on %s", address) if enableSSL { - if err := http.ListenAndServeTLS(address, certFilePath, keyFilePath, recoverHandler(corsHandler(r))); err != nil { + if err := http.ListenAndServeTLS(address, certFilePath, keyFilePath, recoverHandler(corsHandler(e))); err != nil { log.Fatalln(err) } } else { - if err := http.ListenAndServe(address, recoverHandler(corsHandler(r))); err != nil { + if err := http.ListenAndServe(address, recoverHandler(corsHandler(e))); err != nil { log.Fatalln(err) } } } - -func addUI(r *mux.Router) { - if uiDir == "" { - return - } - - if _, err := os.Stat(uiDir); err != nil { - log.Fatalln("UI path doesnot exist", err) - } - - r.PathPrefix("/").Handler(http.FileServer(http.Dir(uiDir))) -} From 070c3f9a848c0d6f395b2fe648748d984ee7456c Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Fri, 17 Jul 2020 10:33:06 +0530 Subject: [PATCH 03/56] fix: issues in echo server config, html, css files formatted --- main.go | 4 +- server.go | 28 ++- ui/index.html | 215 ++++++------------- ui/javascripts/copycss.js | 18 +- ui/stylesheets/addon.css | 26 +-- ui/stylesheets/ambiance.css | 121 ++++++++--- ui/stylesheets/bootstrap-responsive.css | 49 +++-- ui/stylesheets/codemirror.css | 164 ++++++++++---- ui/stylesheets/print.css | 15 +- ui/stylesheets/styles.css | 273 +++++++++++++++++------- 10 files changed, 543 insertions(+), 370 deletions(-) diff --git a/main.go b/main.go index b75144b..b61f612 100644 --- a/main.go +++ b/main.go @@ -109,8 +109,8 @@ func redirectLogToFile() { func init() { flag.IntVar(&port, "p", 8080, "Run daemon in specified port") flag.IntVar(&maxHandleCount, "max-handle-count", 10, "Maximum number of handles can be opened for each language") - flag.StringVar(&host, "host", "", "Host for the varnam daemon server") - flag.StringVar(&uiDir, "ui", "", "UI directory path") + flag.StringVar(&host, "host", "localhost", "Host for the varnam daemon server") + flag.StringVar(&uiDir, "ui", "ui", "UI directory path") flag.BoolVar(&enableInternalApis, "enable-internal-apis", false, "Enable internal APIs") flag.BoolVar(&enableSSL, "enable-ssl", false, "Enables SSL") flag.StringVar(&certFilePath, "cert-file-path", "", "Certificate file path") diff --git a/server.go b/server.go index d595f70..77bc8e9 100644 --- a/server.go +++ b/server.go @@ -8,6 +8,7 @@ import ( "path/filepath" "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" ) func startDaemon() { @@ -24,7 +25,7 @@ func startDaemon() { e.GET("/languages", handleLanguages) e.GET("/status", handleStatus) - if _, err := os.Stat(uiDir); err != nil { + if _, err := os.Stat(filepath.Clean(uiDir)); err != nil { log.Fatal("UI path doesnot exist", err) } @@ -38,13 +39,30 @@ func startDaemon() { address := fmt.Sprintf("%s:%d", host, port) log.Printf("Listening on %s", address) + e.Use(middleware.Recover()) + e.Use(middleware.Logger()) + + e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ + AllowOrigins: []string{"*"}, + AllowMethods: []string{http.MethodOptions}, + })) + + e.Use(middleware.SecureWithConfig(middleware.SecureConfig{ + XSSProtection: "", + ContentTypeNosniff: "", + XFrameOptions: "", + HSTSMaxAge: 3600, + // ContentSecurityPolicy: "default-src 'self'", + })) + if enableSSL { - if err := http.ListenAndServeTLS(address, certFilePath, keyFilePath, recoverHandler(corsHandler(e))); err != nil { - log.Fatalln(err) + if err := e.StartTLS(address, certFilePath, keyFilePath); err != nil { + log.Fatal(err) } } else { - if err := http.ListenAndServe(address, recoverHandler(corsHandler(e))); err != nil { - log.Fatalln(err) + if err := e.Start(address); err != nil { + log.Fatal(err) } } + } diff --git a/ui/index.html b/ui/index.html index e4be1b9..26235f1 100644 --- a/ui/index.html +++ b/ui/index.html @@ -1,163 +1,76 @@ - - - - - - - - - - - - - Varnam - Type in Indian languages - - - + + + + + + - -
- Your Internet connection might have interrupted - -
-
-
+ Varnam - Type in Indian languages + -
-
- - English - - - -
-
- -
-
- - -
+ +
+ Your Internet connection might have interrupted + +
+
+
+
+
+ + English + + +
-
-
-
- +
+
-
- +
+ +
-
- - - - - - +
+
+ +
+
+ +
+ +
+
+ + + + + + diff --git a/ui/javascripts/copycss.js b/ui/javascripts/copycss.js index 3e092fb..2a62c9c 100644 --- a/ui/javascripts/copycss.js +++ b/ui/javascripts/copycss.js @@ -1,6 +1,6 @@ -(function($) { +(function ($) { - $.fn.getStyles = function(only, except) { + $.fn.getStyles = function (only, except) { // the map to return with requested styles and values as KVP var product = {}; @@ -30,12 +30,12 @@ // convenience methods to turn css case ('background-image') to camel ('backgroundImage') var pattern = /\-([a-z])/g; - var uc = function(a, b) { - return b.toUpperCase(); - }; - var camelize = function(string) { - return string.replace(pattern, uc); - }; + var uc = function (a, b) { + return b.toUpperCase(); + }; + var camelize = function (string) { + return string.replace(pattern, uc); + }; // make sure we're getting a good reference if (style = window.getComputedStyle(dom, null)) { @@ -88,7 +88,7 @@ }; // sugar - source is the selector, dom element or jQuery instance to copy from - only and except are optional - $.fn.copyCSS = function(source, only, except) { + $.fn.copyCSS = function (source, only, except) { var styles = $(source).getStyles(only, except); this.css(styles); }; diff --git a/ui/stylesheets/addon.css b/ui/stylesheets/addon.css index 730ded8..4f0b70f 100644 --- a/ui/stylesheets/addon.css +++ b/ui/stylesheets/addon.css @@ -1,17 +1,17 @@ .CodeMirror-completions { - position: absolute; - z-index: 10; - overflow: hidden; - -webkit-box-shadow: 2px 3px 5px rgba(0,0,0,.2); - -moz-box-shadow: 2px 3px 5px rgba(0,0,0,.2); - box-shadow: 2px 3px 5px rgba(0,0,0,.2); + position: absolute; + z-index: 10; + overflow: hidden; + -webkit-box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2); + -moz-box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2); + box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2); } .CodeMirror-completions select { - background: #fafafa; - outline: none; - border: none; - padding: 0; - margin: 0; - font-family: monospace; -} \ No newline at end of file + background: #fafafa; + outline: none; + border: none; + padding: 0; + margin: 0; + font-family: monospace; +} diff --git a/ui/stylesheets/ambiance.css b/ui/stylesheets/ambiance.css index 3b93f9c..6e020c3 100644 --- a/ui/stylesheets/ambiance.css +++ b/ui/stylesheets/ambiance.css @@ -2,47 +2,101 @@ /* Color scheme */ -.cm-s-ambiance .cm-keyword { color: #cda869; } -.cm-s-ambiance .cm-atom { color: #CF7EA9; } -.cm-s-ambiance .cm-number { color: #78CF8A; } -.cm-s-ambiance .cm-def { color: #aac6e3; } -.cm-s-ambiance .cm-variable { color: #ffb795; } -.cm-s-ambiance .cm-variable-2 { color: #eed1b3; } -.cm-s-ambiance .cm-variable-3 { color: #faded3; } -.cm-s-ambiance .cm-property { color: #eed1b3; } -.cm-s-ambiance .cm-operator {color: #fa8d6a;} -.cm-s-ambiance .cm-comment { color: #555; font-style:italic; } -.cm-s-ambiance .cm-string { color: #8f9d6a; } -.cm-s-ambiance .cm-string-2 { color: #9d937c; } -.cm-s-ambiance .cm-meta { color: #D2A8A1; } -.cm-s-ambiance .cm-error { color: #AF2018; } -.cm-s-ambiance .cm-qualifier { color: yellow; } -.cm-s-ambiance .cm-builtin { color: #9999cc; } -.cm-s-ambiance .cm-bracket { color: #24C2C7; } -.cm-s-ambiance .cm-tag { color: #fee4ff } -.cm-s-ambiance .cm-attribute { color: #9B859D; } -.cm-s-ambiance .cm-header {color: blue;} -.cm-s-ambiance .cm-quote { color: #24C2C7; } -.cm-s-ambiance .cm-hr { color: pink; } -.cm-s-ambiance .cm-link { color: #F4C20B; } -.cm-s-ambiance .cm-special { color: #FF9D00; } +.cm-s-ambiance .cm-keyword { + color: #cda869; +} +.cm-s-ambiance .cm-atom { + color: #cf7ea9; +} +.cm-s-ambiance .cm-number { + color: #78cf8a; +} +.cm-s-ambiance .cm-def { + color: #aac6e3; +} +.cm-s-ambiance .cm-variable { + color: #ffb795; +} +.cm-s-ambiance .cm-variable-2 { + color: #eed1b3; +} +.cm-s-ambiance .cm-variable-3 { + color: #faded3; +} +.cm-s-ambiance .cm-property { + color: #eed1b3; +} +.cm-s-ambiance .cm-operator { + color: #fa8d6a; +} +.cm-s-ambiance .cm-comment { + color: #555; + font-style: italic; +} +.cm-s-ambiance .cm-string { + color: #8f9d6a; +} +.cm-s-ambiance .cm-string-2 { + color: #9d937c; +} +.cm-s-ambiance .cm-meta { + color: #d2a8a1; +} +.cm-s-ambiance .cm-error { + color: #af2018; +} +.cm-s-ambiance .cm-qualifier { + color: yellow; +} +.cm-s-ambiance .cm-builtin { + color: #9999cc; +} +.cm-s-ambiance .cm-bracket { + color: #24c2c7; +} +.cm-s-ambiance .cm-tag { + color: #fee4ff; +} +.cm-s-ambiance .cm-attribute { + color: #9b859d; +} +.cm-s-ambiance .cm-header { + color: blue; +} +.cm-s-ambiance .cm-quote { + color: #24c2c7; +} +.cm-s-ambiance .cm-hr { + color: pink; +} +.cm-s-ambiance .cm-link { + color: #f4c20b; +} +.cm-s-ambiance .cm-special { + color: #ff9d00; +} -.cm-s-ambiance .CodeMirror-matchingbracket { color: #0f0; } -.cm-s-ambiance .CodeMirror-nonmatchingbracket { color: #f22; } +.cm-s-ambiance .CodeMirror-matchingbracket { + color: #0f0; +} +.cm-s-ambiance .CodeMirror-nonmatchingbracket { + color: #f22; +} .cm-s-ambiance .CodeMirror-selected { background: rgba(255, 255, 255, 0.15); } .CodeMirror-focused .cm-s-ambiance .CodeMirror-selected { - background: rgba(255, 255, 255, 0.10); + background: rgba(255, 255, 255, 0.1); } /* Editor styling */ .cm-s-ambiance { - line-height: 1.40em; - font-family: Monaco, Menlo,"Andale Mono","lucida console","Courier New",monospace !important; - color: #E6E1DC; + line-height: 1.4em; + font-family: Monaco, Menlo, "Andale Mono", "lucida console", "Courier New", + monospace !important; + color: #e6e1dc; background-color: #202020; -webkit-box-shadow: inset 0 0 10px black; -moz-box-shadow: inset 0 0 10px black; @@ -51,10 +105,10 @@ } .cm-s-ambiance .CodeMirror-gutter { - background: #3D3D3D; + background: #3d3d3d; padding: 0 5px; text-shadow: #333 1px 1px; - border-right: 1px solid #4D4D4D; + border-right: 1px solid #4d4d4d; box-shadow: 0 10px 20px black; } @@ -64,11 +118,10 @@ } .cm-s-ambiance .CodeMirror-lines { - } .cm-s-ambiance .CodeMirror-lines .CodeMirror-cursor { - border-left: 1px solid #7991E8; + border-left: 1px solid #7991e8; } .cm-s-ambiance .activeline { diff --git a/ui/stylesheets/bootstrap-responsive.css b/ui/stylesheets/bootstrap-responsive.css index 06e55c0..4429722 100644 --- a/ui/stylesheets/bootstrap-responsive.css +++ b/ui/stylesheets/bootstrap-responsive.css @@ -35,9 +35,9 @@ width: 100%; min-height: 28px; -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - -ms-box-sizing: border-box; - box-sizing: border-box; + -moz-box-sizing: border-box; + -ms-box-sizing: border-box; + box-sizing: border-box; } .hidden { @@ -185,9 +185,9 @@ width: 100%; min-height: 28px; -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - -ms-box-sizing: border-box; - box-sizing: border-box; + -moz-box-sizing: border-box; + -ms-box-sizing: border-box; + box-sizing: border-box; } .input-prepend input, .input-append input, @@ -312,9 +312,9 @@ margin-left: 2.762430939%; *margin-left: 2.709239449638298%; -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - -ms-box-sizing: border-box; - box-sizing: border-box; + -moz-box-sizing: border-box; + -ms-box-sizing: border-box; + box-sizing: border-box; } .row-fluid [class*="span"]:first-child { margin-left: 0; @@ -548,9 +548,9 @@ margin-left: 2.564102564%; *margin-left: 2.510911074638298%; -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - -ms-box-sizing: border-box; - box-sizing: border-box; + -moz-box-sizing: border-box; + -ms-box-sizing: border-box; + box-sizing: border-box; } .row-fluid [class*="span"]:first-child { margin-left: 0; @@ -732,15 +732,15 @@ font-weight: bold; color: #999999; -webkit-border-radius: 3px; - -moz-border-radius: 3px; - border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; } .nav-collapse .btn { padding: 4px 10px 4px; font-weight: normal; -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; } .nav-collapse .dropdown-menu li + li a { margin-bottom: 2px; @@ -765,11 +765,11 @@ background-color: transparent; border: none; -webkit-border-radius: 0; - -moz-border-radius: 0; - border-radius: 0; + -moz-border-radius: 0; + border-radius: 0; -webkit-box-shadow: none; - -moz-box-shadow: none; - box-shadow: none; + -moz-box-shadow: none; + box-shadow: none; } .nav-collapse .dropdown-menu:before, .nav-collapse .dropdown-menu:after { @@ -785,9 +785,12 @@ margin: 9px 0; border-top: 1px solid #222222; border-bottom: 1px solid #222222; - -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); - -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); - box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); + -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), + 0 1px 0 rgba(255, 255, 255, 0.1); + -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), + 0 1px 0 rgba(255, 255, 255, 0.1); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), + 0 1px 0 rgba(255, 255, 255, 0.1); } .navbar .nav-collapse .nav.pull-right { float: none; diff --git a/ui/stylesheets/codemirror.css b/ui/stylesheets/codemirror.css index 05ad0ed..524c749 100644 --- a/ui/stylesheets/codemirror.css +++ b/ui/stylesheets/codemirror.css @@ -20,7 +20,8 @@ /* Vertical scrollbar */ .CodeMirror-scrollbar { position: absolute; - right: 0; top: 0; + right: 0; + top: 0; overflow-x: hidden; overflow-y: scroll; z-index: 5; @@ -47,7 +48,9 @@ } .CodeMirror-gutter { - position: absolute; left: 0; top: 0; + position: absolute; + left: 0; + top: 0; z-index: 10; background-color: #f7f7f7; border-right: 1px solid #eee; @@ -57,12 +60,12 @@ .CodeMirror-gutter-text { color: #aaa; text-align: right; - padding: .4em .2em .4em .4em; + padding: 0.4em 0.2em 0.4em 0.4em; white-space: pre !important; cursor: default; } .CodeMirror-lines { - padding: .4em; + padding: 0.4em; white-space: pre; cursor: text; } @@ -72,10 +75,14 @@ -webkit-border-radius: 0; -o-border-radius: 0; border-radius: 0; - border-width: 0; margin: 0; padding: 0; background: transparent; + border-width: 0; + margin: 0; + padding: 0; + background: transparent; font-family: inherit; font-size: inherit; - padding: 0; margin: 0; + padding: 0; + margin: 0; white-space: pre; word-wrap: normal; line-height: inherit; @@ -107,67 +114,132 @@ width: auto; border: 0; background: transparent; - background: rgba(0, 200, 0, .4); + background: rgba(0, 200, 0, 0.4); filter: progid:DXImageTransform.Microsoft.gradient(startColorstr=#6600c800, endColorstr=#4c00c800); } /* Kludge to turn off filter in ie9+, which also accepts rgba */ .cm-keymap-fat-cursor pre.CodeMirror-cursor:not(#nonsense_id) { filter: progid:DXImageTransform.Microsoft.gradient(enabled=false); } -.CodeMirror pre.CodeMirror-cursor.CodeMirror-overwrite {} +.CodeMirror pre.CodeMirror-cursor.CodeMirror-overwrite { +} .CodeMirror-focused pre.CodeMirror-cursor { visibility: visible; } -div.CodeMirror-selected { background: #d9d9d9; } -.CodeMirror-focused div.CodeMirror-selected { background: #d7d4f0; } +div.CodeMirror-selected { + background: #d9d9d9; +} +.CodeMirror-focused div.CodeMirror-selected { + background: #d7d4f0; +} .CodeMirror-searching { background: #ffa; - background: rgba(255, 255, 0, .4); + background: rgba(255, 255, 0, 0.4); } /* Default theme */ -.cm-s-default span.cm-keyword {color: #708;} -.cm-s-default span.cm-atom {color: #219;} -.cm-s-default span.cm-number {color: #164;} -.cm-s-default span.cm-def {color: #00f;} -.cm-s-default span.cm-variable {color: black;} -.cm-s-default span.cm-variable-2 {color: #05a;} -.cm-s-default span.cm-variable-3 {color: #085;} -.cm-s-default span.cm-property {color: black;} -.cm-s-default span.cm-operator {color: black;} -.cm-s-default span.cm-comment {color: #a50;} -.cm-s-default span.cm-string {color: #a11;} -.cm-s-default span.cm-string-2 {color: #f50;} -.cm-s-default span.cm-meta {color: #555;} -.cm-s-default span.cm-error {color: #f00;} -.cm-s-default span.cm-qualifier {color: #555;} -.cm-s-default span.cm-builtin {color: #30a;} -.cm-s-default span.cm-bracket {color: #997;} -.cm-s-default span.cm-tag {color: #170;} -.cm-s-default span.cm-attribute {color: #00c;} -.cm-s-default span.cm-header {color: blue;} -.cm-s-default span.cm-quote {color: #090;} -.cm-s-default span.cm-hr {color: #999;} -.cm-s-default span.cm-link {color: #00c;} - -span.cm-header, span.cm-strong {font-weight: bold;} -span.cm-em {font-style: italic;} -span.cm-emstrong {font-style: italic; font-weight: bold;} -span.cm-link {text-decoration: underline;} - -span.cm-invalidchar {color: #f00;} - -div.CodeMirror span.CodeMirror-matchingbracket {color: #0f0;} -div.CodeMirror span.CodeMirror-nonmatchingbracket {color: #f22;} +.cm-s-default span.cm-keyword { + color: #708; +} +.cm-s-default span.cm-atom { + color: #219; +} +.cm-s-default span.cm-number { + color: #164; +} +.cm-s-default span.cm-def { + color: #00f; +} +.cm-s-default span.cm-variable { + color: black; +} +.cm-s-default span.cm-variable-2 { + color: #05a; +} +.cm-s-default span.cm-variable-3 { + color: #085; +} +.cm-s-default span.cm-property { + color: black; +} +.cm-s-default span.cm-operator { + color: black; +} +.cm-s-default span.cm-comment { + color: #a50; +} +.cm-s-default span.cm-string { + color: #a11; +} +.cm-s-default span.cm-string-2 { + color: #f50; +} +.cm-s-default span.cm-meta { + color: #555; +} +.cm-s-default span.cm-error { + color: #f00; +} +.cm-s-default span.cm-qualifier { + color: #555; +} +.cm-s-default span.cm-builtin { + color: #30a; +} +.cm-s-default span.cm-bracket { + color: #997; +} +.cm-s-default span.cm-tag { + color: #170; +} +.cm-s-default span.cm-attribute { + color: #00c; +} +.cm-s-default span.cm-header { + color: blue; +} +.cm-s-default span.cm-quote { + color: #090; +} +.cm-s-default span.cm-hr { + color: #999; +} +.cm-s-default span.cm-link { + color: #00c; +} -@media print { +span.cm-header, +span.cm-strong { + font-weight: bold; +} +span.cm-em { + font-style: italic; +} +span.cm-emstrong { + font-style: italic; + font-weight: bold; +} +span.cm-link { + text-decoration: underline; +} + +span.cm-invalidchar { + color: #f00; +} +div.CodeMirror span.CodeMirror-matchingbracket { + color: #0f0; +} +div.CodeMirror span.CodeMirror-nonmatchingbracket { + color: #f22; +} + +@media print { /* Hide the cursor when printing */ .CodeMirror pre.CodeMirror-cursor { visibility: hidden; } - } diff --git a/ui/stylesheets/print.css b/ui/stylesheets/print.css index 280e58d..256e962 100644 --- a/ui/stylesheets/print.css +++ b/ui/stylesheets/print.css @@ -1,16 +1,19 @@ -div.toolbar, #editor_div, header, .navbar{ - display:none; +div.toolbar, +#editor_div, +header, +.navbar { + display: none; } -#preview_div{ - display:block !important; - width:100%; +#preview_div { + display: block !important; + width: 100%; margin: 0; border: 0; padding: 0; } -#preview{ +#preview { border: 0; margin: 0; box-shadow: none; diff --git a/ui/stylesheets/styles.css b/ui/stylesheets/styles.css index a5a7e3d..eca7b67 100644 --- a/ui/stylesheets/styles.css +++ b/ui/stylesheets/styles.css @@ -1,146 +1,257 @@ body { - background: url("../images/stressed_linen.png") repeat scroll 0 0 #151515; - /*color: #EAEAEA;*/ - color: #B3B3B3; - line-height: 1.5; - margin: 0 0 10px; - padding: 0; - font-size: 16px; - font-weight: normal; + background: url("../images/stressed_linen.png") repeat scroll 0 0 #151515; + /*color: #EAEAEA;*/ + color: #b3b3b3; + line-height: 1.5; + margin: 0 0 10px; + padding: 0; + font-size: 16px; + font-weight: normal; } code { - border: 0px; - background-color: rgba(186, 196, 204, 0.12); + border: 0px; + background-color: rgba(186, 196, 204, 0.12); } .container { - max-width: 800px; - margin: 0 auto; - width: 90%; + max-width: 800px; + margin: 0 auto; + width: 90%; } header h1:before { - content: "./ "; - font-size: 24px; + content: "./ "; + font-size: 24px; } .main-content { - background: none repeat scroll 0 0 rgba(186, 196, 204, 0.12); - border-radius: 10px 10px 10px 10px; - margin-top: 15px; - padding-left: 30px; - padding-right: 30px; + background: none repeat scroll 0 0 rgba(186, 196, 204, 0.12); + border-radius: 10px 10px 10px 10px; + margin-top: 15px; + padding-left: 30px; + padding-right: 30px; } .footer { - font-size: 12px; - text-align: right; - padding-left: 50%; + font-size: 12px; + text-align: right; + padding-left: 50%; } h1 { - font-size: 30px; + font-size: 30px; } h1 a { - color: inherit; - text-decoration: none; + color: inherit; + text-decoration: none; } h1 a:hover { - color: inherit; - text-decoration: none; + color: inherit; + text-decoration: none; } -h2, h3 { - font-size: 18px; +h2, +h3 { + font-size: 18px; } -h1, h2, h3 { - /*color: #0088CC;*/ - color: #B5E853; +h1, +h2, +h3 { + /*color: #0088CC;*/ + color: #b5e853; } .main-content h3 { - font-size: 18px; + font-size: 18px; } .logo { - font-family: Monaco,"Bitstream Vera Sans Mono","Lucida Console",Terminal,monospace; + font-family: Monaco, "Bitstream Vera Sans Mono", "Lucida Console", Terminal, + monospace; } -header .logo h2{ - color: #666666; - font-weight: 300; - font-size:18px; - letter-spacing: -0.03em; - text-shadow: 0 1px 1px rgba(0, 0, 0, 0.1), 0 0 5px rgba(181, 232, 83, 0.1), 0 0 10px rgba(181, 232, 83, 0.1); - line-height: 30px; +header .logo h2 { + color: #666666; + font-weight: 300; + font-size: 18px; + letter-spacing: -0.03em; + text-shadow: 0 1px 1px rgba(0, 0, 0, 0.1), 0 0 5px rgba(181, 232, 83, 0.1), + 0 0 10px rgba(181, 232, 83, 0.1); + line-height: 30px; } #mainlinks a { - border-color: #000000 rgba(0, 0, 0, 0.7) rgba(0, 0, 0, 0.7); - border-image: none; - border-left: 2px solid rgba(0, 0, 0, 0.7); - border-radius: 50px 50px 50px 50px; - border-right: 2px solid rgba(0, 0, 0, 0.7); - border-style: solid; - border-width: 2px; - box-shadow: 0 1px 0 rgba(255, 255, 255, 0.05) inset; - color: #B3B3B3; - display: inline-block; - font-family: Helvetica,Arial,sans-serif; - font-size: 15px; - padding: 8px 18px; - text-decoration: none; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.75); - background-color: #1E1C1C; + border-color: #000000 rgba(0, 0, 0, 0.7) rgba(0, 0, 0, 0.7); + border-image: none; + border-left: 2px solid rgba(0, 0, 0, 0.7); + border-radius: 50px 50px 50px 50px; + border-right: 2px solid rgba(0, 0, 0, 0.7); + border-style: solid; + border-width: 2px; + box-shadow: 0 1px 0 rgba(255, 255, 255, 0.05) inset; + color: #b3b3b3; + display: inline-block; + font-family: Helvetica, Arial, sans-serif; + font-size: 15px; + padding: 8px 18px; + text-decoration: none; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.75); + background-color: #1e1c1c; } p { - margin: 0 0 1.5em; + margin: 0 0 1.5em; } .navigation { - float: right; + float: right; } .editor { - clear: both; - margin-right: 10px; + clear: both; + margin-right: 10px; } .repolinks { - list-style: none; + list-style: none; } .repolinks li:before { - content: "- "; - color: red; - font-weight: bold; - font-size: smaller; + content: "- "; + color: red; + font-weight: bold; + font-size: smaller; } -nav ul{list-style-type:none;float:right;} -nav ul li {display:inline-block; } -.headlines {margin-top:200px;} -.headlines p{font-size:25px; color:#fff;} -.main-container{margin-top:40px;} -.banner { background:#ccc;} +nav ul { + list-style-type: none; + float: right; +} +nav ul li { + display: inline-block; +} +.headlines { + margin-top: 200px; +} +.headlines p { + font-size: 25px; + color: #fff; +} +.main-container { + margin-top: 40px; +} +.banner { + background: #ccc; +} -header.inner {height:100px;background: url("../images/bg.jpg") no-repeat scroll left -165px transparent;} -header.inner .logo{height:100px;} -header.inner .logo h1{line-height:100px;} +header.inner { + height: 100px; + background: url("../images/bg.jpg") no-repeat scroll left -165px transparent; +} +header.inner .logo { + height: 100px; +} +header.inner .logo h1 { + line-height: 100px; +} pre { - background-color: black; - color: #666; + background-color: black; + color: #666; } .main-intro { - text-align: center; - padding-top: 10px; - padding-bottom: 0px; + text-align: center; + padding-top: 10px; + padding-bottom: 0px; +} + +.CodeMirror-completions { + position: absolute; + z-index: 10; + overflow: hidden; + -webkit-box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2); + -moz-box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2); + box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2); +} + +.CodeMirror-completions select { + background: #fafafa; + outline: none; + border: none; + padding: 0; + margin: 0; + font-family: monospace; +} + +.CodeMirror { + background: #ededed; + border: #333 solid 1px; +} + +iframe { + /* width: 100%;*/ + display: block; + width: 100%; + height: 400px; + background-color: #fff; + /*padding: 5px;*/ + overflow: auto; + /*margin-right:5px;*/ + border: 1px solid black; + box-shadow: rgba(0, 0, 0, 0.75) 0 2px 5px; + border-radius: 3px; +} + +.CodeMirror pre { + line-height: inherit; + color: #b3b3b3; +} + +.CodeMirror-wrap pre { + word-break: normal; +} + +.CodeMirror-gutter { + background: none repeat scroll 0 0 rgba(186, 196, 204, 0.12); +} + +.navbar h1 { + color: #000; + /* margin-top: 5px; */ + display: inline; +} + +.toolbar { + position: relative; + padding-right: 10px; +} + +.caption { + position: absolute; + bottom: 17px; + left: 155px; +} + +.navbar .navbar-inner { + background: none repeat scroll 0 0 #ffffff; + border-radius: 0 0 0 0; + box-shadow: none; + color: #000000; + padding-top: 10px; + text-shadow: none; } +.networkerror { + z-index: 9; + position: fixed; + top: 0px; + left: 18%; + display: none; + margin: 0 auto; + width: 60%; +} From a3889f42b66a303e7b0fe1bcba98565aa5391f96 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 9 Aug 2020 09:00:15 +0530 Subject: [PATCH 04/56] corrected wildcard for handlers --- server.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server.go b/server.go index 77bc8e9..07b0ce9 100644 --- a/server.go +++ b/server.go @@ -17,10 +17,10 @@ func startDaemon() { e := echo.New() - e.GET("/tl/{langCode}/{word}", handleTransliteration) - e.GET("/rtl/{langCode}/{word}", handleReverseTransliteration) - e.GET("/meta/{langCode}", handleMetadata) - e.GET("/download/{langCode}/{downloadStart}", handleDownload) + e.GET("/tl/:langCode/:word", handleTransliteration) + e.GET("/rtl/:langCode/:word", handleReverseTransliteration) + e.GET("/meta/:langCode:", handleMetadata) + e.GET("/download/:langCode/:downloadStart", handleDownload) e.POST("/learn", handlLearn) e.GET("/languages", handleLanguages) e.GET("/status", handleStatus) From 196c62c1cbd10042df49a7169d876de6110c06f0 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Thu, 3 Sep 2020 16:33:56 +0530 Subject: [PATCH 05/56] made flag parsing before initializing configs --- http_handlers.go | 8 +++++++- main.go | 6 +++--- varnam_learn.go | 6 ------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/http_handlers.go b/http_handlers.go index 0c26ba0..6d33e3a 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -55,6 +55,12 @@ type downloadResponse struct { standardResponse } +// Args to read. +type args struct { + LangCode string `json:"lang"` + Word string `json:"word"` +} + func corsHandler(h http.Handler) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") @@ -259,7 +265,7 @@ func handleLanguages(c echo.Context) error { } func handlLearn(c echo.Context) error { - var args Args + var args args if err := c.Bind(&args); err != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) diff --git a/main.go b/main.go index b61f612..e1a9386 100644 --- a/main.go +++ b/main.go @@ -138,12 +138,12 @@ func startSyncDispatcher() { } func main() { - varnamdConfig = initConfig() - startedAt = time.Now() - runtime.GOMAXPROCS(runtime.NumCPU()) flag.Parse() + varnamdConfig = initConfig() + startedAt = time.Now() + if version { fmt.Println(varnamdVersion) os.Exit(0) diff --git a/varnam_learn.go b/varnam_learn.go index bb8143d..31d6226 100644 --- a/varnam_learn.go +++ b/varnam_learn.go @@ -6,12 +6,6 @@ import ( "github.com/varnamproject/libvarnam-golang" ) -// Args to read. -type Args struct { - LangCode string `json:"lang"` - Word string `json:"word"` -} - var ( learnChannels map[string]chan string ) From 39ae0a0375db0a3360598c47e19825c7bf1072bb Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Thu, 3 Sep 2020 17:17:38 +0530 Subject: [PATCH 06/56] fixed json key --- http_handlers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http_handlers.go b/http_handlers.go index 6d33e3a..04fd41d 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -58,7 +58,7 @@ type downloadResponse struct { // Args to read. type args struct { LangCode string `json:"lang"` - Word string `json:"word"` + Text string `json:"text"` } func corsHandler(h http.Handler) http.HandlerFunc { @@ -277,7 +277,7 @@ func handlLearn(c echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, "unable to find language") } - go func(word string) { ch <- word }(args.Word) + go func(word string) { ch <- word }(args.Text) return c.JSON(http.StatusOK, "success") } From 705e8ec4f8e09f353bcb62fb0f714c79e5d72bd5 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Thu, 3 Sep 2020 21:19:06 +0530 Subject: [PATCH 07/56] fixed lint errors, added Content-Type as json for learn endpoint manually --- http_handlers.go | 65 +++++++--------------------------------------- main.go | 5 ++-- server.go | 1 - sync.go | 54 ++++++++++---------------------------- varnam_handlers.go | 25 ++++++++++-------- varnam_learn.go | 5 ++-- 6 files changed, 40 insertions(+), 115 deletions(-) diff --git a/http_handlers.go b/http_handlers.go index 04fd41d..8832ed3 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -7,9 +7,7 @@ import ( "encoding/json" "errors" "fmt" - "log" "net/http" - "runtime" "strconv" "strings" "time" @@ -61,44 +59,6 @@ type args struct { Text string `json:"text"` } -func corsHandler(h http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Access-Control-Allow-Origin", "*") - - if r.Method == "OPTIONS" { - w.WriteHeader(http.StatusOK) - } else { - h.ServeHTTP(w, r) - } - } -} - -func recoverHandler(next http.Handler) http.Handler { - fn := func(w http.ResponseWriter, r *http.Request) { - defer func() { - if err := recover(); err != nil { - stack := make([]byte, 1024) - stack = stack[:runtime.Stack(stack, false)] - log.Printf("panic: %s\n%s", err, stack) - http.Error(w, http.StatusText(500), 500) - } - }() - next.ServeHTTP(w, r) - } - - return http.HandlerFunc(fn) -} - -// func getLangCode(r *http.Request) string { -// params := mux.Vars(r) -// return params["langCode"] -// } - -// func getWord(r *http.Request) string { -// params := mux.Vars(r) -// return params["word"] -// } - func handleStatus(c echo.Context) error { uptime := time.Since(startedAt) @@ -121,9 +81,7 @@ func handleTransliteration(c echo.Context) error { word = c.Param("word") ) - // langCode, word := getLanguageAndWord(r) words, err := transliterate(langCode, word) - if err != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) } @@ -138,7 +96,6 @@ func handleReverseTransliteration(c echo.Context) error { ) result, err := reveseTransliterate(langCode, word) - if err != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) } @@ -159,7 +116,6 @@ func handleMetadata(c echo.Context) error { data, err := getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { details, err := handle.GetCorpusDetails() - if err != nil { return nil, err } @@ -176,11 +132,9 @@ func handleMetadata(c echo.Context) error { func handleDownload(c echo.Context) error { var ( langCode = c.Param("langCode") - // word = c.Param("word") + start, _ = strconv.Atoi(c.Param("downloadStart")) ) - start, _ := strconv.Atoi(c.Param("downloadStart")) - if start < 0 { return echo.NewHTTPError(http.StatusBadRequest, "invalid parameter") } @@ -191,15 +145,15 @@ func handleDownload(c echo.Context) error { parts := strings.Split(key, "+") schemeID := parts[0] downloadStart, _ := strconv.Atoi(parts[1]) - words, err := getWords(schemeID, downloadStart) + words, err := getWords(schemeID, downloadStart) if err != nil { return err } response := downloadResponse{Count: len(words), Words: words, standardResponse: newStandardResponse()} - b, err := json.Marshal(response) + b, err := json.Marshal(response) if err != nil { return err } @@ -265,19 +219,20 @@ func handleLanguages(c echo.Context) error { } func handlLearn(c echo.Context) error { - var args args + var a args + + c.Request().Header.Set("Content-Type", "application/json") - if err := c.Bind(&args); err != nil { + if err := c.Bind(&a); err != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } - ch, ok := learnChannels[args.LangCode] - + ch, ok := learnChannels[a.LangCode] if !ok { return echo.NewHTTPError(http.StatusBadRequest, "unable to find language") } - go func(word string) { ch <- word }(args.Text) + go func(word string) { ch <- word }(a.Text) return c.JSON(http.StatusOK, "success") } @@ -296,7 +251,6 @@ func handleEnableDownload(c echo.Context) error { ) data, err := toggleDownloadEnabledStatus(langCode, true) - if err != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } @@ -310,7 +264,6 @@ func handleDisableDownload(c echo.Context) error { ) data, err := toggleDownloadEnabledStatus(langCode, false) - if err != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } diff --git a/main.go b/main.go index e1a9386..7bf84d3 100644 --- a/main.go +++ b/main.go @@ -85,9 +85,8 @@ func getConfigDir() string { func getLogsDir() string { d := getConfigDir() logsDir := path.Join(d, "logs") - err := os.MkdirAll(logsDir, 0750) - if err != nil { + if err := os.MkdirAll(logsDir, 0750); err != nil { panic(err) } @@ -97,8 +96,8 @@ func getLogsDir() string { func redirectLogToFile() { year, month, day := time.Now().Date() logfile := path.Join(getLogsDir(), fmt.Sprintf("%d-%d-%d.log", year, month, day)) - f, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + f, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) if err != nil { panic(err) } diff --git a/server.go b/server.go index 07b0ce9..47fc4c0 100644 --- a/server.go +++ b/server.go @@ -64,5 +64,4 @@ func startDaemon() { log.Fatal(err) } } - } diff --git a/sync.go b/sync.go index 75729d5..9dc3c47 100644 --- a/sync.go +++ b/sync.go @@ -28,7 +28,6 @@ func newSyncDispatcher(interval time.Duration) *syncDispatcher { func (s *syncDispatcher) start() { err := createSyncMetadataDir() - if err != nil { fmt.Printf("Failed to create sync metadata directory. Sync will be disabled.\nActual error: %s\n", err.Error()) return @@ -37,10 +36,8 @@ func (s *syncDispatcher) start() { for s := range varnamdConfig.schemesToDownload { // download cache directory for each of the languages err = createLearnQueueDir(s) - if err != nil { fmt.Printf("Failed to create learn queue directory for '%s'. Sync will be disabled.\nActual error: %s\n", s, err.Error()) - return } } @@ -181,8 +178,7 @@ func downloadWordsAndUpdateOffset(langCode string, offset int) (string, error) { return "", err } - err = setDownloadOffset(langCode, offset+count) - if err != nil { + if err = setDownloadOffset(langCode, offset+count); err != nil { log.Printf("Error setting download offset for '%s'. %s\n", langCode, err.Error()) return "", err } @@ -191,12 +187,12 @@ func downloadWordsAndUpdateOffset(langCode string, offset int) (string, error) { } func getCorpusDetails(langCode string) (*libvarnam.CorpusDetails, error) { + var m metaResponse + url := fmt.Sprintf("%s/meta/%s", varnamdConfig.upstream, langCode) log.Printf("Fetching corpus details for '%s'\n", langCode) - var m metaResponse err := getJSONResponse(url, &m) - if err != nil { return nil, err } @@ -212,14 +208,11 @@ func downloadWords(langCode string, offset int) (totalWordsDownloaded int, downl var response downloadResponse url := fmt.Sprintf("%s/download/%s/%d", varnamdConfig.upstream, langCode, offset) - err = getJSONResponse(url, &response) - - if err != nil { + if err = getJSONResponse(url, &response); err != nil { return 0, "", err } downloadedFilePath, err = transformAndPersistWords(langCode, offset, &response) - if err != nil { log.Printf("Download was successful, but failed to persist to local learn queue. %s\n", err.Error()) return 0, "", err @@ -230,8 +223,8 @@ func downloadWords(langCode string, offset int) (totalWordsDownloaded int, downl func transformAndPersistWords(langCode string, offset int, dresp *downloadResponse) (string, error) { learnQueueDir := getLearnQueueDir(langCode) - targetFile, err := os.Create(path.Join(learnQueueDir, fmt.Sprintf("%s.%d", langCode, offset))) + targetFile, err := os.Create(path.Join(learnQueueDir, fmt.Sprintf("%s.%d", langCode, offset))) if err != nil { return "", err } @@ -239,8 +232,7 @@ func transformAndPersistWords(langCode string, offset int, dresp *downloadRespon defer func() { _ = targetFile.Close() }() for _, word := range dresp.Words { - _, err = targetFile.WriteString(fmt.Sprintf("%s %d\n", word.Word, word.Confidence)) - if err != nil { + if _, err = targetFile.WriteString(fmt.Sprintf("%s %d\n", word.Word, word.Confidence)); err != nil { return "", err } } @@ -252,8 +244,8 @@ func getFilesFromLearnQueue(langCode string) []string { var files []string learnQueueDir := getLearnQueueDir(langCode) - queueContents, err := ioutil.ReadDir(learnQueueDir) + queueContents, err := ioutil.ReadDir(learnQueueDir) if err != nil { return nil } @@ -270,8 +262,7 @@ func getFilesFromLearnQueue(langCode string) []string { func getJSONResponse(url string, output interface{}) error { log.Printf("GET: '%s'\n", url) - resp, err := http.Get(url) - + resp, err := http.Get(url) // #nosec G107 if err != nil { return err } @@ -279,19 +270,14 @@ func getJSONResponse(url string, output interface{}) error { defer func() { _ = resp.Body.Close() }() jsonDecoder := json.NewDecoder(resp.Body) - err = jsonDecoder.Decode(output) - if err != nil { - return err - } - - return nil + return jsonDecoder.Decode(output) } func getDownloadOffset(langCode string) int { filePath := getDownloadOffsetMetadataFile(langCode) - content, err := ioutil.ReadFile(filepath.Clean(filePath)) + content, err := ioutil.ReadFile(filepath.Clean(filePath)) if err != nil { return 0 } @@ -316,13 +302,7 @@ func getDownloadOffsetMetadataFile(langCode string) string { func createLearnQueueDir(langCode string) error { queueDir := getLearnQueueDir(langCode) - err := os.MkdirAll(queueDir, 0750) - - if err != nil { - return err - } - - return nil + return os.MkdirAll(queueDir, 0750) } func getLearnQueueDir(langCode string) string { @@ -334,18 +314,10 @@ func getLearnQueueDir(langCode string) string { func createSyncMetadataDir() error { syncDir := getSyncMetadataDir() - err := os.MkdirAll(syncDir, 0750) - - if err != nil { - return err - } - - return nil + return os.MkdirAll(syncDir, 0750) } func getSyncMetadataDir() string { configDir := getConfigDir() - syncDir := path.Join(configDir, "sync") - - return syncDir + return path.Join(configDir, "sync") } diff --git a/varnam_handlers.go b/varnam_handlers.go index f6a7188..1594219 100644 --- a/varnam_handlers.go +++ b/varnam_handlers.go @@ -52,6 +52,7 @@ func initLanguageChannels() { if err != nil { panic("Unable to init varnam for language " + scheme.LangCode + "." + err.Error()) } + languageChannels[scheme.Identifier] <- handle } } @@ -59,9 +60,11 @@ func initLanguageChannels() { func getOrCreateHandler(schemeIdentifier string, f func(handle *libvarnam.Varnam) (data interface{}, err error)) (data interface{}, err error) { ch, ok := languageChannels[schemeIdentifier] + if !ok { return nil, errors.New("invalid scheme identifier") } + select { case handle := <-ch: data, err = f(handle) @@ -69,14 +72,18 @@ func getOrCreateHandler(schemeIdentifier string, f func(handle *libvarnam.Varnam go func() { ch <- handle }() case <-time.After(800 * time.Millisecond): var handle *libvarnam.Varnam - handle, err = libvarnam.Init(schemeIdentifier) + handle, err = libvarnam.Init(schemeIdentifier) if err != nil { log.Println(err) return nil, errors.New("unable to initialize varnam handle") } data, err = f(handle) + if err != nil { + log.Println(err) + return nil, errors.New("unable to complete varnam handle") + } go sendHandlerToChannel(schemeIdentifier, handle, ch) } @@ -84,10 +91,9 @@ func getOrCreateHandler(schemeIdentifier string, f func(handle *libvarnam.Varnam return } -func transliterate(schemeIdentifier string, word string) (data interface{}, err error) { +func transliterate(schemeIdentifier string, word string) (interface{}, error) { return getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { - data, err = handle.Transliterate(word) - return + return handle.Transliterate(word) }) } @@ -97,7 +103,6 @@ func getWords(schemeIdentifier string, downloadStart int) ([]*word, error) { }) db, err := sql.Open("sqlite3", filepath.(string)) - if err != nil { return nil, err } @@ -106,14 +111,13 @@ func getWords(schemeIdentifier string, downloadStart int) ([]*word, error) { // Making an index for all learned words so that download is faster // this needs to be removed when there is more clarity on how learned words needs to be handled - _, err = db.Exec("create index if not exists varnamd_download_only_learned on patterns_content (learned) where learned = 1;") - if err != nil { + if _, err = db.Exec("create index if not exists varnamd_download_only_learned on patterns_content (learned) where learned = 1;"); err != nil { return nil, err } q := `select id, word, confidence from words where id in (select distinct(word_id) from patterns_content where learned = 1) order by id asc limit ? offset ?;` - rows, err := db.Query(q, downloadPageSize, downloadStart) + rows, err := db.Query(q, downloadPageSize, downloadStart) if err != nil { return nil, err } @@ -140,10 +144,9 @@ func getWords(schemeIdentifier string, downloadStart int) ([]*word, error) { return words, nil } -func reveseTransliterate(schemeIdentifier string, word string) (data interface{}, err error) { +func reveseTransliterate(schemeIdentifier string, word string) (interface{}, error) { return getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { - data, err = handle.ReverseTransliterate(word) - return + return handle.ReverseTransliterate(word) }) } diff --git a/varnam_learn.go b/varnam_learn.go index 31d6226..6dbba4f 100644 --- a/varnam_learn.go +++ b/varnam_learn.go @@ -14,8 +14,8 @@ func initLearnChannels() { learnChannels = make(map[string]chan string) for _, scheme := range schemeDetails { learnChannels[scheme.Identifier] = make(chan string, 100) - handle, err := libvarnam.Init(scheme.Identifier) + handle, err := libvarnam.Init(scheme.Identifier) if err != nil { log.Fatal("Unable to initialize varnam for lang", scheme.LangCode) } @@ -26,8 +26,7 @@ func initLearnChannels() { func listenForWords(lang string, handle *libvarnam.Varnam) { for word := range learnChannels[lang] { - err := handle.Learn(word) - if err != nil { + if err := handle.Learn(word); err != nil { log.Printf("Failed to learn %s. %s\n", word, err.Error()) } } From 13622dbea13c22171627584fb5dd643848eddc97 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Thu, 3 Sep 2020 22:55:12 +0530 Subject: [PATCH 08/56] fixed content encoding issue in gzip content encoding --- http_handlers.go | 9 ++------- varnam_handlers.go | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/http_handlers.go b/http_handlers.go index 8832ed3..2ff78e6 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -196,22 +196,17 @@ func handleDownload(c echo.Context) error { ctx := varnamCacheContext{} var data []byte - if err := cacheGroup.Get(&ctx, fmt.Sprintf("%s+%d", langCode, start), groupcache.AllocatingByteSliceSink(&data)); err != nil { if err == errCacheSkipped { - c.Response().Header().Set("Content-Type", "application/json; charset=utf-8") c.Response().Header().Set("Content-Encoding", "gzip") - - return c.JSON(http.StatusOK, ctx.Data) + return c.Blob(http.StatusOK, "application/json; charset=utf-8", ctx.Data) } return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } - c.Response().Header().Set("Content-Type", "application/json; charset=utf-8") c.Response().Header().Set("Content-Encoding", "gzip") - - return c.JSON(http.StatusOK, data) + return c.Blob(http.StatusOK, "application/json; charset=utf-8", data) } func handleLanguages(c echo.Context) error { diff --git a/varnam_handlers.go b/varnam_handlers.go index 1594219..0f52e9c 100644 --- a/varnam_handlers.go +++ b/varnam_handlers.go @@ -50,7 +50,7 @@ func initLanguageChannels() { for i := 0; i < maxHandleCount; i++ { handle, err := libvarnam.Init(scheme.Identifier) if err != nil { - panic("Unable to init varnam for language " + scheme.LangCode + "." + err.Error()) + panic("Unable to init varnam for language" + scheme.LangCode + ". " + err.Error()) } languageChannels[scheme.Identifier] <- handle From 547961c92baf0b6ce913e4e33923d9afc5e6124b Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Fri, 4 Sep 2020 09:23:14 +0530 Subject: [PATCH 09/56] corrected error check pattern --- sync.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/sync.go b/sync.go index 9dc3c47..dd1295d 100644 --- a/sync.go +++ b/sync.go @@ -27,16 +27,14 @@ func newSyncDispatcher(interval time.Duration) *syncDispatcher { } func (s *syncDispatcher) start() { - err := createSyncMetadataDir() - if err != nil { + if err := createSyncMetadataDir(); err != nil { fmt.Printf("Failed to create sync metadata directory. Sync will be disabled.\nActual error: %s\n", err.Error()) return } for s := range varnamdConfig.schemesToDownload { // download cache directory for each of the languages - err = createLearnQueueDir(s) - if err != nil { + if err := createLearnQueueDir(s); err != nil { fmt.Printf("Failed to create learn queue directory for '%s'. Sync will be disabled.\nActual error: %s\n", s, err.Error()) return } @@ -57,10 +55,6 @@ func (s *syncDispatcher) start() { }() } -// func (s *syncDispatcher) stop() { -// close(s.quit) -// } - func (s *syncDispatcher) runNow() { s.force <- true } @@ -129,7 +123,6 @@ func downloadAllWords(langCode string, corpusSize int, output chan string) { } filePath, err := downloadWordsAndUpdateOffset(langCode, offset) - if err != nil { break } @@ -162,8 +155,7 @@ func learnFromFile(langCode, fileToLearn string) { log.Printf("Learned from '%s'. TotalWords: %d, Failed: %d. Took %s\n", fileToLearn, learnStatus.TotalWords, learnStatus.Failed, end.Sub(start)) } - err = os.Remove(fileToLearn) - if err != nil { + if err = os.Remove(fileToLearn); err != nil { log.Printf("Error deleting '%s'. %s\n", fileToLearn, err.Error()) } @@ -192,8 +184,7 @@ func getCorpusDetails(langCode string) (*libvarnam.CorpusDetails, error) { url := fmt.Sprintf("%s/meta/%s", varnamdConfig.upstream, langCode) log.Printf("Fetching corpus details for '%s'\n", langCode) - err := getJSONResponse(url, &m) - if err != nil { + if err := getJSONResponse(url, &m); err != nil { return nil, err } From 96b908de86b4b046f572d173affdcd502d1aff9c Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Fri, 11 Sep 2020 17:35:48 +0530 Subject: [PATCH 10/56] added cache for tl --- cache.go | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 6 +++++ http_handlers.go | 13 ++++++++--- main.go | 11 +++++++++- server.go | 11 +++++++++- 6 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 cache.go diff --git a/cache.go b/cache.go new file mode 100644 index 0000000..c380b0a --- /dev/null +++ b/cache.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + "strings" + "time" + + "github.com/coocood/freecache" +) + +const ( + defaultCacheSize = 100 << 20 // 100 MB cache + defaultExpiry = time.Hour * 24 // Cache for 24 hours + wordSeparator = "|" // Assuming the separator wont be used in any case. +) + +// Cache objects. +type Cache interface { + Set(lang, word string, val []string) error + Get(lang, word string) ([]string, error) +} + +// MemCache impliments Cache interface. +type MemCache struct { + fc *freecache.Cache +} + +// NewMemCache will create object of new Cache impliemtation. +func NewMemCache() *MemCache { + return newCacheWithSize(defaultCacheSize) +} + +func newCacheWithSize(size int) *MemCache { + return &MemCache{fc: freecache.NewCache(size)} +} + +// Set lang-word as val to cache. +func (c *MemCache) Set(lang, word string, val []string) error { + var value = strings.Join(val, "|") + return c.setWithExpiry(lang, word, []byte(value), int(defaultExpiry)) +} + +func (c *MemCache) setWithExpiry(lang, word string, val []byte, expiry int) error { + var key = fmt.Sprintf("%s-%s", lang, word) + return c.fc.Set([]byte(key), val, expiry) +} + +// Get lang-word from cache. +func (c *MemCache) Get(lang, word string) ([]string, error) { + var key = fmt.Sprintf("%s-%s", lang, word) + val, err := c.fc.Get([]byte(key)) + if err != nil { + return nil, err + } + + return strings.Split(string(val), wordSeparator), nil +} diff --git a/go.mod b/go.mod index a7fbcdd..64da2d8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/varnamproject/varnamd go 1.14 require ( + github.com/coocood/freecache v1.1.1 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/protobuf v1.4.2 // indirect github.com/labstack/echo/v4 v4.1.16 diff --git a/go.sum b/go.sum index a234f57..eefcea3 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,10 @@ +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc= +github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= @@ -35,6 +40,7 @@ github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/ github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/http_handlers.go b/http_handlers.go index 2ff78e6..20de0a4 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -79,14 +79,21 @@ func handleTransliteration(c echo.Context) error { var ( langCode = c.Param("langCode") word = c.Param("word") + app = c.Get("app").(*App) ) - words, err := transliterate(langCode, word) + words, err := app.cache.Get(langCode, word) if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) + w, err := transliterate(langCode, word) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) + } + + words = w.([]string) + app.cache.Set(langCode, word, words) } - return c.JSON(http.StatusOK, transliterationResponse{standardResponse: newStandardResponse(), Result: words.([]string), Input: word}) + return c.JSON(http.StatusOK, transliterationResponse{standardResponse: newStandardResponse(), Result: words, Input: word}) } func handleReverseTransliteration(c echo.Context) error { diff --git a/main.go b/main.go index 7bf84d3..9de0c43 100644 --- a/main.go +++ b/main.go @@ -30,6 +30,11 @@ var ( syncDispatcherRunning bool ) +// App is a singleton to share across handlers. +type App struct { + cache Cache +} + // varnamd configurations // this is populated from various command line flags type config struct { @@ -154,6 +159,10 @@ func main() { log.Printf("varnamd %s", varnamdVersion) + app := &App{ + cache: NewMemCache(), + } + startSyncDispatcher() - startDaemon() + startDaemon(app) } diff --git a/server.go b/server.go index 47fc4c0..a92ae4f 100644 --- a/server.go +++ b/server.go @@ -11,7 +11,8 @@ import ( "github.com/labstack/echo/v4/middleware" ) -func startDaemon() { +func startDaemon(app *App) { + initLanguageChannels() initLearnChannels() @@ -42,6 +43,14 @@ func startDaemon() { e.Use(middleware.Recover()) e.Use(middleware.Logger()) + // Custom middleware to set sigleton to app's context. + e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + c.Set("app", app) + return next(c) + } + }) + e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ AllowOrigins: []string{"*"}, AllowMethods: []string{http.MethodOptions}, From a6e274c19bceb690639786d2f1da893d1529d3ca Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Fri, 11 Sep 2020 22:51:59 +0530 Subject: [PATCH 11/56] added caching for rtl --- cache.go | 8 ++++---- http_handlers.go | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/cache.go b/cache.go index c380b0a..5994de4 100644 --- a/cache.go +++ b/cache.go @@ -11,12 +11,12 @@ import ( const ( defaultCacheSize = 100 << 20 // 100 MB cache defaultExpiry = time.Hour * 24 // Cache for 24 hours - wordSeparator = "|" // Assuming the separator wont be used in any case. + wordSeparator = "<>" // Assuming the separator wont be used in any case. ) // Cache objects. type Cache interface { - Set(lang, word string, val []string) error + Set(lang, word string, val ...string) error Get(lang, word string) ([]string, error) } @@ -35,8 +35,8 @@ func newCacheWithSize(size int) *MemCache { } // Set lang-word as val to cache. -func (c *MemCache) Set(lang, word string, val []string) error { - var value = strings.Join(val, "|") +func (c *MemCache) Set(lang, word string, val ...string) error { + var value = strings.Join(val, wordSeparator) return c.setWithExpiry(lang, word, []byte(value), int(defaultExpiry)) } diff --git a/http_handlers.go b/http_handlers.go index 20de0a4..5debf4f 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -90,7 +90,7 @@ func handleTransliteration(c echo.Context) error { } words = w.([]string) - app.cache.Set(langCode, word, words) + app.cache.Set(langCode, word, words...) } return c.JSON(http.StatusOK, transliterationResponse{standardResponse: newStandardResponse(), Result: words, Input: word}) @@ -100,11 +100,22 @@ func handleReverseTransliteration(c echo.Context) error { var ( langCode = c.Param("langCode") word = c.Param("word") + app = c.Get("app").(*App) ) - result, err := reveseTransliterate(langCode, word) + result, err := app.cache.Get(langCode, word) if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) + res, err := reveseTransliterate(langCode, word) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) + } + + result = []string{res.(string)} + app.cache.Set(langCode, word, res.(string)) + } + + if len(result) <= 0 { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("no transliteration found for lanugage: %s, word: %s", langCode, word)) } response := struct { @@ -112,7 +123,7 @@ func handleReverseTransliteration(c echo.Context) error { Result string `json:"result"` }{ newStandardResponse(), - result.(string), + result[0], } return c.JSON(http.StatusOK, response) From 87ff1695c3e7d10d2248f952e02ca4422a24ccce Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sat, 12 Sep 2020 10:41:10 +0530 Subject: [PATCH 12/56] changed handler to another function --- main.go | 2 ++ server.go | 36 +++++++++++++++++++++++------------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/main.go b/main.go index 9de0c43..d47c2d3 100644 --- a/main.go +++ b/main.go @@ -33,6 +33,7 @@ var ( // App is a singleton to share across handlers. type App struct { cache Cache + log *log.Logger } // varnamd configurations @@ -161,6 +162,7 @@ func main() { app := &App{ cache: NewMemCache(), + log: log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), } startSyncDispatcher() diff --git a/server.go b/server.go index a92ae4f..485952d 100644 --- a/server.go +++ b/server.go @@ -16,6 +16,27 @@ func startDaemon(app *App) { initLanguageChannels() initLearnChannels() + e, err := initHandlers(app) + if err != nil { + app.log.Fatalf("error initializing handlers: %s", err) + } + + address := fmt.Sprintf("%s:%d", host, port) + log.Printf("Listening on %s", address) + + if enableSSL { + if err := e.StartTLS(address, certFilePath, keyFilePath); err != nil { + log.Fatal(err) + } + } else { + if err := e.Start(address); err != nil { + log.Fatal(err) + } + } + +} + +func initHandlers(app *App) (*echo.Echo, error) { e := echo.New() e.GET("/tl/:langCode/:word", handleTransliteration) @@ -27,7 +48,7 @@ func startDaemon(app *App) { e.GET("/status", handleStatus) if _, err := os.Stat(filepath.Clean(uiDir)); err != nil { - log.Fatal("UI path doesnot exist", err) + return nil, err } e.Static("/", filepath.Clean(uiDir)) @@ -37,9 +58,6 @@ func startDaemon(app *App) { e.POST("/sync/download/{langCode}/disable", handleDisableDownload) } - address := fmt.Sprintf("%s:%d", host, port) - log.Printf("Listening on %s", address) - e.Use(middleware.Recover()) e.Use(middleware.Logger()) @@ -64,13 +82,5 @@ func startDaemon(app *App) { // ContentSecurityPolicy: "default-src 'self'", })) - if enableSSL { - if err := e.StartTLS(address, certFilePath, keyFilePath); err != nil { - log.Fatal(err) - } - } else { - if err := e.Start(address); err != nil { - log.Fatal(err) - } - } + return e, nil } From 5479a9e4896798ac4fb15271baaee55d57a65839 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sat, 12 Sep 2020 11:07:20 +0530 Subject: [PATCH 13/56] added logging to handlers --- cache.go | 1 + http_handlers.go | 33 ++++++++++++++++++++++++++++----- server.go | 9 +++------ 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/cache.go b/cache.go index 5994de4..ca36f9d 100644 --- a/cache.go +++ b/cache.go @@ -48,6 +48,7 @@ func (c *MemCache) setWithExpiry(lang, word string, val []byte, expiry int) erro // Get lang-word from cache. func (c *MemCache) Get(lang, word string) ([]string, error) { var key = fmt.Sprintf("%s-%s", lang, word) + val, err := c.fc.Get([]byte(key)) if err != nil { return nil, err diff --git a/http_handlers.go b/http_handlers.go index 5debf4f..aedc2ea 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -86,11 +86,12 @@ func handleTransliteration(c echo.Context) error { if err != nil { w, err := transliterate(langCode, word) if err != nil { + app.log.Printf("error in transliterationg, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) } - words = w.([]string) - app.cache.Set(langCode, word, words...) + words, _ = w.([]string) + _ = app.cache.Set(langCode, word, words...) } return c.JSON(http.StatusOK, transliterationResponse{standardResponse: newStandardResponse(), Result: words, Input: word}) @@ -107,14 +108,16 @@ func handleReverseTransliteration(c echo.Context) error { if err != nil { res, err := reveseTransliterate(langCode, word) if err != nil { + app.log.Printf("error in reverse transliterationg, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) } result = []string{res.(string)} - app.cache.Set(langCode, word, res.(string)) + _ = app.cache.Set(langCode, word, res.(string)) } if len(result) <= 0 { + app.log.Printf("no reverse transliteration found for lang: %s word: %s", langCode, word) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("no transliteration found for lanugage: %s, word: %s", langCode, word)) } @@ -130,7 +133,10 @@ func handleReverseTransliteration(c echo.Context) error { } func handleMetadata(c echo.Context) error { - var schemeIdentifier = c.Param("langCode") + var ( + schemeIdentifier = c.Param("langCode") + app = c.Get("app").(*App) + ) data, err := getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { details, err := handle.GetCorpusDetails() @@ -141,6 +147,7 @@ func handleMetadata(c echo.Context) error { return &metaResponse{Result: details, standardResponse: newStandardResponse()}, nil }) if err != nil { + app.log.Printf("error in getting corpus details for: %s, err: %s", schemeIdentifier, err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } @@ -151,6 +158,8 @@ func handleDownload(c echo.Context) error { var ( langCode = c.Param("langCode") start, _ = strconv.Atoi(c.Param("downloadStart")) + + app = c.Get("app").(*App) ) if start < 0 { @@ -220,10 +229,13 @@ func handleDownload(c echo.Context) error { return c.Blob(http.StatusOK, "application/json; charset=utf-8", ctx.Data) } + app.log.Printf("error in fetching deta from cache: %s, err: %s", langCode, err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } c.Response().Header().Set("Content-Encoding", "gzip") + return c.Blob(http.StatusOK, "application/json; charset=utf-8", data) } @@ -232,16 +244,22 @@ func handleLanguages(c echo.Context) error { } func handlLearn(c echo.Context) error { - var a args + var ( + a args + + app = c.Get("app").(*App) + ) c.Request().Header.Set("Content-Type", "application/json") if err := c.Bind(&a); err != nil { + app.log.Printf("error in binding request details for learn, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } ch, ok := learnChannels[a.LangCode] if !ok { + app.log.Printf("unknown language requested to learn: %s", a.LangCode) return echo.NewHTTPError(http.StatusBadRequest, "unable to find language") } @@ -261,10 +279,13 @@ func toggleDownloadEnabledStatus(langCode string, status bool) (interface{}, err func handleEnableDownload(c echo.Context) error { var ( langCode = c.Param("langCode") + + app = c.Get("app").(*App) ) data, err := toggleDownloadEnabledStatus(langCode, true) if err != nil { + app.log.Printf("failed to toggle download enable, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } @@ -274,10 +295,12 @@ func handleEnableDownload(c echo.Context) error { func handleDisableDownload(c echo.Context) error { var ( langCode = c.Param("langCode") + app = c.Get("app").(*App) ) data, err := toggleDownloadEnabledStatus(langCode, false) if err != nil { + app.log.Printf("failed to disable download, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } diff --git a/server.go b/server.go index 485952d..c9f4c0d 100644 --- a/server.go +++ b/server.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "log" "net/http" "os" "path/filepath" @@ -12,7 +11,6 @@ import ( ) func startDaemon(app *App) { - initLanguageChannels() initLearnChannels() @@ -22,18 +20,17 @@ func startDaemon(app *App) { } address := fmt.Sprintf("%s:%d", host, port) - log.Printf("Listening on %s", address) + app.log.Printf("Listening on %s", address) if enableSSL { if err := e.StartTLS(address, certFilePath, keyFilePath); err != nil { - log.Fatal(err) + app.log.Fatal(err) } } else { if err := e.Start(address); err != nil { - log.Fatal(err) + app.log.Fatal(err) } } - } func initHandlers(app *App) (*echo.Echo, error) { From c20038f4ce6ac12ea5db386780383ab26097eebb Mon Sep 17 00:00:00 2001 From: Athul Cyriac Ajay Date: Sat, 12 Sep 2020 16:24:48 +0530 Subject: [PATCH 14/56] Refactored function name typo --- http_handlers.go | 2 +- server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/http_handlers.go b/http_handlers.go index 2ff78e6..25dd79a 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -213,7 +213,7 @@ func handleLanguages(c echo.Context) error { return c.JSON(http.StatusOK, schemeDetails) } -func handlLearn(c echo.Context) error { +func handleLearn(c echo.Context) error { var a args c.Request().Header.Set("Content-Type", "application/json") diff --git a/server.go b/server.go index 47fc4c0..b0529c4 100644 --- a/server.go +++ b/server.go @@ -21,7 +21,7 @@ func startDaemon() { e.GET("/rtl/:langCode/:word", handleReverseTransliteration) e.GET("/meta/:langCode:", handleMetadata) e.GET("/download/:langCode/:downloadStart", handleDownload) - e.POST("/learn", handlLearn) + e.POST("/learn", handleLearn) e.GET("/languages", handleLanguages) e.GET("/status", handleStatus) From c85fae87bef4f899253a454b6e1bf111eb0fb530 Mon Sep 17 00:00:00 2001 From: Athul Cyriac Ajay Date: Sat, 12 Sep 2020 20:12:49 +0530 Subject: [PATCH 15/56] Implement Train Endpoint --- go.mod | 2 +- go.sum | 10 ++++++++-- http_handlers.go | 37 ++++++++++++++++++++++++++++++++++++- server.go | 1 + sync.go | 2 +- varnam_handlers.go | 8 +++++++- varnam_learn.go | 2 +- 7 files changed, 55 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 64da2d8..c7257c7 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/varnamproject/varnamd go 1.14 require ( + github.com/athul/go-libvarnam v0.0.0-20200912100303-48fdad5ac70e github.com/coocood/freecache v1.1.1 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/protobuf v1.4.2 // indirect @@ -10,7 +11,6 @@ require ( github.com/mattn/go-colorable v0.1.7 // indirect github.com/mattn/go-sqlite3 v1.14.0 github.com/valyala/fasttemplate v1.2.0 // indirect - github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect diff --git a/go.sum b/go.sum index eefcea3..18b3b15 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,9 @@ +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/athul/go-libvarnam v0.0.0-20200912100303-48fdad5ac70e h1:zKSW8/mOy4aYFghtsAFXNrq84IbjmFvqX3lAy0hqb7M= +github.com/athul/go-libvarnam v0.0.0-20200912100303-48fdad5ac70e/go.mod h1:QRvlfDTVGxOgZYm8toOEXFVAmSQXCvjq3zWBZBfHYig= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc= @@ -40,18 +43,19 @@ github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/ github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.0 h1:y3yXRCoDvC2HTtIHvL2cc7Zd+bqA+zqDO6oQzsJO07E= github.com/valyala/fasttemplate v1.2.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932 h1:Z6VhoopafnsFFYf8KwGty1dmJteTCM/mLGRAvc81QPk= -github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932/go.mod h1:p0y0nS7FVUFgPK6dOkEY/NiOI7+aBeYgrl4ACVFuLIo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -95,3 +99,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/http_handlers.go b/http_handlers.go index 5debf4f..ace317c 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -12,9 +12,9 @@ import ( "strings" "time" + "github.com/athul/go-libvarnam" "github.com/golang/groupcache" "github.com/labstack/echo/v4" - "github.com/varnamproject/libvarnam-golang" ) var errCacheSkipped = errors.New("cache skipped") @@ -59,6 +59,13 @@ type args struct { Text string `json:"text"` } +//TrainArgs read the incoming data +type TrainArgs struct { + Lang string `json:"lang" form:"lang"` + Pattern string `json:"pattern" form:"pattern"` + Word string `json:"word" form:"word"` +} + func handleStatus(c echo.Context) error { uptime := time.Since(startedAt) @@ -250,6 +257,34 @@ func handlLearn(c echo.Context) error { return c.JSON(http.StatusOK, "success") } +func handleTrain(c echo.Context) error { + var targs TrainArgs + fmt.Println(1) + // c.Request().Header.Set("Content-Type", "application/json") + + // if err := c.Bind(&targs); err != nil { + // fmt.Println("Error 1") + // return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + // } + targs.Lang = c.FormValue("lang") + targs.Pattern = c.FormValue("pattern") + targs.Word = c.FormValue("word") + + fmt.Println(targs) + handle, err := libvarnam.Init(targs.Lang) + if err != nil { + fmt.Println("Error 2") + return fmt.Errorf("failed to get data") + } + fmt.Println(3) + if err := handle.Train(targs.Pattern, targs.Word); err != nil { + fmt.Println("Error Trains............") + return fmt.Errorf("failed to Train %s. %s", targs.Word, err.Error()) + } + fmt.Println(4) + return c.JSON(200, "Word Trained") + +} func toggleDownloadEnabledStatus(langCode string, status bool) (interface{}, error) { if err := varnamdConfig.setDownloadStatus(langCode, status); err != nil { return nil, err diff --git a/server.go b/server.go index 485952d..95f4332 100644 --- a/server.go +++ b/server.go @@ -46,6 +46,7 @@ func initHandlers(app *App) (*echo.Echo, error) { e.POST("/learn", handlLearn) e.GET("/languages", handleLanguages) e.GET("/status", handleStatus) + e.POST("/train", handleTrain) if _, err := os.Stat(filepath.Clean(uiDir)); err != nil { return nil, err diff --git a/sync.go b/sync.go index dd1295d..4d39707 100644 --- a/sync.go +++ b/sync.go @@ -13,7 +13,7 @@ import ( "strings" "time" - "github.com/varnamproject/libvarnam-golang" + "github.com/athul/go-libvarnam" ) type syncDispatcher struct { diff --git a/varnam_handlers.go b/varnam_handlers.go index 0f52e9c..b401384 100644 --- a/varnam_handlers.go +++ b/varnam_handlers.go @@ -7,9 +7,9 @@ import ( "sync" "time" + "github.com/athul/go-libvarnam" "github.com/golang/groupcache" _ "github.com/mattn/go-sqlite3" - "github.com/varnamproject/libvarnam-golang" ) type word struct { @@ -97,6 +97,12 @@ func transliterate(schemeIdentifier string, word string) (interface{}, error) { }) } +// func trainwords(schemeIdentifier, word, pattern string) (interface{}, error) { +// return getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { +// return handle.Train(pattern,word) +// }) +// } + func getWords(schemeIdentifier string, downloadStart int) ([]*word, error) { filepath, _ := getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { return handle.GetSuggestionsFilePath(), nil diff --git a/varnam_learn.go b/varnam_learn.go index 6dbba4f..4917440 100644 --- a/varnam_learn.go +++ b/varnam_learn.go @@ -3,7 +3,7 @@ package main import ( "log" - "github.com/varnamproject/libvarnam-golang" + "github.com/athul/go-libvarnam" ) var ( From 30617548edfba1bc43bc3e24d5eb230aa6ca7b5d Mon Sep 17 00:00:00 2001 From: Athul Cyriac Ajay Date: Sun, 13 Sep 2020 14:22:57 +0530 Subject: [PATCH 16/56] Moved the Train Request types to JSON --- http_handlers.go | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/http_handlers.go b/http_handlers.go index ace317c..49a72a5 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -259,29 +259,19 @@ func handlLearn(c echo.Context) error { func handleTrain(c echo.Context) error { var targs TrainArgs - fmt.Println(1) - // c.Request().Header.Set("Content-Type", "application/json") - - // if err := c.Bind(&targs); err != nil { - // fmt.Println("Error 1") - // return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) - // } - targs.Lang = c.FormValue("lang") - targs.Pattern = c.FormValue("pattern") - targs.Word = c.FormValue("word") - - fmt.Println(targs) + + c.Request().Header.Set("Content-Type", "application/json") + + if err := c.Bind(&targs); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } handle, err := libvarnam.Init(targs.Lang) if err != nil { - fmt.Println("Error 2") return fmt.Errorf("failed to get data") } - fmt.Println(3) if err := handle.Train(targs.Pattern, targs.Word); err != nil { - fmt.Println("Error Trains............") return fmt.Errorf("failed to Train %s. %s", targs.Word, err.Error()) } - fmt.Println(4) return c.JSON(200, "Word Trained") } From a8e1659f728373fbddb20ae4aa8c215dfe74120e Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 14:28:37 +0530 Subject: [PATCH 17/56] removed go-libvarnam dep --- go.mod | 1 - go.sum | 4 +- http_handlers.go | 2 +- libvarnam/libvarnam.go | 201 +++++++++++++++++++++++++++++++++++++++++ sync.go | 2 +- varnam_handlers.go | 2 +- varnam_learn.go | 2 +- 7 files changed, 207 insertions(+), 7 deletions(-) create mode 100644 libvarnam/libvarnam.go diff --git a/go.mod b/go.mod index 64da2d8..f4f8e1f 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/mattn/go-colorable v0.1.7 // indirect github.com/mattn/go-sqlite3 v1.14.0 github.com/valyala/fasttemplate v1.2.0 // indirect - github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect diff --git a/go.sum b/go.sum index eefcea3..8a1d42e 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= @@ -40,6 +41,7 @@ github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/ github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= @@ -50,8 +52,6 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.0 h1:y3yXRCoDvC2HTtIHvL2cc7Zd+bqA+zqDO6oQzsJO07E= github.com/valyala/fasttemplate v1.2.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932 h1:Z6VhoopafnsFFYf8KwGty1dmJteTCM/mLGRAvc81QPk= -github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932/go.mod h1:p0y0nS7FVUFgPK6dOkEY/NiOI7+aBeYgrl4ACVFuLIo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/http_handlers.go b/http_handlers.go index aedc2ea..58dd50b 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -14,7 +14,7 @@ import ( "github.com/golang/groupcache" "github.com/labstack/echo/v4" - "github.com/varnamproject/libvarnam-golang" + "github.com/varnamproject/varnamd/libvarnam" ) var errCacheSkipped = errors.New("cache skipped") diff --git a/libvarnam/libvarnam.go b/libvarnam/libvarnam.go new file mode 100644 index 0000000..4c1bdc1 --- /dev/null +++ b/libvarnam/libvarnam.go @@ -0,0 +1,201 @@ +package libvarnam + +// #cgo pkg-config: varnam +// #include +import "C" +import "fmt" + +// Varnam app binding. +type Varnam struct { + handle *C.varnam +} + +// VarnamError satisfies error interface. +type VarnamError struct { + errorCode int + message string +} + +// SchemeDetails returns language and other changes. +type SchemeDetails struct { + LangCode string + Identifier string + DisplayName string + Author string + CompiledDate string + IsStable bool +} + +// CorpusDetails returns corpus details. +type CorpusDetails struct { + WordsCount int `json:"wordsCount"` +} + +// LearnStatus . +type LearnStatus struct { + TotalWords int + Failed int +} + +func (e *VarnamError) Error() string { + return e.message +} + +// GetSuggestionsFilePath returns suggestions. +func (v *Varnam) GetSuggestionsFilePath() string { + return C.GoString(C.varnam_get_suggestions_file(v.handle)) +} + +// GetCorpusDetails will return corpus details. +func (v *Varnam) GetCorpusDetails() (*CorpusDetails, error) { + var details *C.vcorpus_details + + rc := C.varnam_get_corpus_details(v.handle, &details) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return nil, &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return &CorpusDetails{WordsCount: int(details.wordsCount)}, nil +} + +// Transliterate given string to corresponding language. +func (v *Varnam) Transliterate(text string) ([]string, error) { + var va *C.varray + rc := C.varnam_transliterate(v.handle, C.CString(text), &va) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return nil, &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + var ( + i C.int + words []string + ) + + for i = 0; i < C.varray_length(va); i++ { + word := (*C.vword)(C.varray_get(va, i)) + words = append(words, C.GoString(word.text)) + } + + return words, nil +} + +// ReverseTransliterate given string. +func (v *Varnam) ReverseTransliterate(text string) (string, error) { + var output *C.char + rc := C.varnam_reverse_transliterate(v.handle, C.CString(text), &output) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return "", &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return C.GoString(output), nil +} + +// LearnFromFile learns from file from the given filepath. +func (v *Varnam) LearnFromFile(filePath string) (*LearnStatus, error) { + var status C.vlearn_status + rc := C.varnam_learn_from_file(v.handle, C.CString(filePath), &status, nil, nil) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return nil, &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return &LearnStatus{TotalWords: int(status.total_words), Failed: int(status.failed)}, nil +} + +// Init initializes varnam bindings. +func Init(schemeIdentifier string) (*Varnam, error) { + var ( + v *C.varnam + msg *C.char + ) + + rc := C.varnam_init_from_id(C.CString(schemeIdentifier), &v, &msg) + + if rc != C.VARNAM_SUCCESS { + return nil, &VarnamError{errorCode: (int)(rc), message: C.GoString(msg)} + } + + return &Varnam{handle: v}, nil +} + +// GetAllSchemeDetails returns all scheme related details. +func GetAllSchemeDetails() []*SchemeDetails { + allHandles := C.varnam_get_all_handles() + + if allHandles == nil { + return []*SchemeDetails{} + } + + var schemeDetails []*SchemeDetails + + length := int(C.varray_length(allHandles)) + + for i := 0; i < length; i++ { + var detail *C.vscheme_details + + handle := (*C.varnam)(C.varray_get(allHandles, C.int(i))) + rc := C.varnam_get_scheme_details(handle, &detail) + + if rc != C.VARNAM_SUCCESS { + return []*SchemeDetails{} + } + + schemeDetails = append(schemeDetails, &SchemeDetails{ + LangCode: C.GoString(detail.langCode), Identifier: C.GoString(detail.identifier), + DisplayName: C.GoString(detail.displayName), Author: C.GoString(detail.author), + CompiledDate: C.GoString(detail.compiledDate), IsStable: detail.isStable > 0}) + + C.varnam_destroy(handle) + } + + return schemeDetails +} + +// Learn from given input text. +func (v *Varnam) Learn(text string) error { + rc := C.varnam_learn(v.handle, C.CString(text)) + + if rc != 0 { + errorCode := (int)(rc) + + return &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return nil +} + +func (v *Varnam) getVarnamError(errorCode int) string { + errormessage := C.varnam_get_last_error(v.handle) + varnamErrorMsg := C.GoString(errormessage) + + return fmt.Sprintf("%d:%s", errorCode, varnamErrorMsg) +} + +// Destroy closes handle. +func (v *Varnam) Destroy() { + C.varnam_destroy(v.handle) +} + +// Train methods trains with the word and pattern,eg: pattern=chrome,word=ക്രോം +func (v *Varnam) Train(pattern, word string) error { + rc := C.varnam_train(v.handle, C.CString(pattern), C.CString(word)) + + if rc != 0 { + errorCode := (int)(rc) + + return &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return nil +} diff --git a/sync.go b/sync.go index dd1295d..5b9c4ae 100644 --- a/sync.go +++ b/sync.go @@ -13,7 +13,7 @@ import ( "strings" "time" - "github.com/varnamproject/libvarnam-golang" + "github.com/varnamproject/varnamd/libvarnam" ) type syncDispatcher struct { diff --git a/varnam_handlers.go b/varnam_handlers.go index 0f52e9c..c2508ff 100644 --- a/varnam_handlers.go +++ b/varnam_handlers.go @@ -9,7 +9,7 @@ import ( "github.com/golang/groupcache" _ "github.com/mattn/go-sqlite3" - "github.com/varnamproject/libvarnam-golang" + "github.com/varnamproject/varnamd/libvarnam" ) type word struct { diff --git a/varnam_learn.go b/varnam_learn.go index 6dbba4f..fdb8f29 100644 --- a/varnam_learn.go +++ b/varnam_learn.go @@ -3,7 +3,7 @@ package main import ( "log" - "github.com/varnamproject/libvarnam-golang" + "github.com/varnamproject/varnamd/libvarnam" ) var ( From e8b8b9c080b26a8a5217f32d316c91ec37321604 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 15:38:10 +0530 Subject: [PATCH 18/56] migrated to koanf --- config.toml | 10 +++ defs.go | 4 -- go.mod | 2 + go.sum | 23 +++++++ http_handlers.go | 2 +- init.go | 54 +++++++++++++++ main.go | 171 ++++++++++++++++++++--------------------------- server.go | 16 ++--- 8 files changed, 170 insertions(+), 112 deletions(-) create mode 100644 config.toml delete mode 100644 defs.go create mode 100644 init.go diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..025fc39 --- /dev/null +++ b/config.toml @@ -0,0 +1,10 @@ +[app] + address = ":8080" + enable-internal-api = false + enable-ssl = false + cert-path = "" + key-file-path = "" + max-handles = 10 + upstream-url = "https://api.varnamproject.com" + download-enabled-schemes = "" + sync-interval = "30s" \ No newline at end of file diff --git a/defs.go b/defs.go deleted file mode 100644 index f3cbb5d..0000000 --- a/defs.go +++ /dev/null @@ -1,4 +0,0 @@ -package main - -const varnamdVersion = "0.0.8" -const downloadPageSize = 100 diff --git a/go.mod b/go.mod index f4f8e1f..6ab74b4 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,11 @@ require ( github.com/coocood/freecache v1.1.1 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/protobuf v1.4.2 // indirect + github.com/knadh/koanf v0.12.1 github.com/labstack/echo/v4 v4.1.16 github.com/mattn/go-colorable v0.1.7 // indirect github.com/mattn/go-sqlite3 v1.14.0 + github.com/spf13/pflag v1.0.5 github.com/valyala/fasttemplate v1.2.0 // indirect golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect diff --git a/go.sum b/go.sum index 8a1d42e..646906c 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= @@ -8,8 +10,13 @@ github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEka github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -23,6 +30,10 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/knadh/koanf v0.12.1 h1:9N0asqPUvZ0jL9thTbMZ0FAT3VrGfR+aYm5JTInT+Gs= +github.com/knadh/koanf v0.12.1/go.mod h1:31bzRSM7vS5Vm9LNLo7B2Re1zhLOZT6EQKeodixBikE= github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz0o= github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= @@ -39,11 +50,19 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= +github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rhnvrm/simples3 v0.5.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -70,10 +89,12 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= @@ -95,3 +116,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/http_handlers.go b/http_handlers.go index 58dd50b..67e7a28 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -67,7 +67,7 @@ func handleStatus(c echo.Context) error { Uptime string `json:"uptime"` standardResponse }{ - varnamdVersion, + buildVersion + "-" + buildDate, uptime.String(), newStandardResponse(), } diff --git a/init.go b/init.go new file mode 100644 index 0000000..4c76531 --- /dev/null +++ b/init.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "os" + "path" + "runtime" + "strings" + "time" +) + +func initConfig(cfg appConfig) *config { + toDownload := make(map[string]bool) + schemes := strings.Split(cfg.DownloadEnabledSchemes, ",") + + for _, scheme := range schemes { + s := strings.TrimSpace(scheme) + + if s != "" { + if !isValidSchemeIdentifier(s) { + panic(fmt.Sprintf("%s is not a valid libvarnam supported scheme", s)) + } + + toDownload[s] = true + } + } + + return &config{upstream: cfg.UpstreamURL, schemesToDownload: toDownload, + syncInterval: time.Duration(cfg.SyncInterval)} +} + +func (c *config) setDownloadStatus(langCode string, status bool) error { + if !isValidSchemeIdentifier(langCode) { + return fmt.Errorf("%s is not a valid libvarnam supported scheme", langCode) + } + + c.schemesToDownload[langCode] = status + + if status { + // when varnamd was started without any langcodes to sync, the dispatcher won't be running + // in that case, we need to start the dispatcher since we have a new lang code to download now + startSyncDispatcher() + } + + return nil +} + +func getConfigDir() string { + if runtime.GOOS == "windows" { + return path.Join(os.Getenv("localappdata"), ".varnamd") + } + + return path.Join(os.Getenv("HOME"), ".varnamd") +} diff --git a/main.go b/main.go index d47c2d3..3547740 100644 --- a/main.go +++ b/main.go @@ -1,33 +1,48 @@ package main import ( - "flag" "fmt" "log" "os" - "path" "runtime" - "strings" "time" + + flag "github.com/spf13/pflag" + + "github.com/knadh/koanf" + "github.com/knadh/koanf/parsers/toml" + "github.com/knadh/koanf/providers/file" + "github.com/knadh/koanf/providers/posflag" ) +const ( + downloadPageSize = 100 +) + +type appConfig struct { + Address string `koanf:"address"` + + EnableInternalApis bool `koanf:"enable-internal-api"` // internal APIs are not exposed to public + EnableSSL bool `koanf:"enable-ssl"` + CertFilePath string `koanf:"cert-path"` + KeyFilePath string `koanf:"key-file-path"` + + DownloadEnabledSchemes string `koanf:"download-enabled-schemes"` + SyncInterval time.Duration `koanf:"sync-interval"` + UpstreamURL string `koanf:"upstream-url"` +} + var ( - port int - version bool - maxHandleCount int - host string - uiDir string - enableInternalApis bool // internal APIs are not exposed to public - enableSSL bool - certFilePath string - keyFilePath string - logToFile bool // logs will be written to file when true - varnamdConfig *config // config instance used across the application - startedAt time.Time - downloadEnabledSchemes string // comma separated list of scheme identifier for which download will be performed - syncIntervalInSecs int - upstreamURL string - syncDispatcherRunning bool + kf = koanf.New(".") + + uiDir = "ui/" + + varnamdConfig *config // config instance used across the application + syncDispatcherRunning bool + startedAt time.Time + buildVersion string + buildDate string + maxHandleCount int ) // App is a singleton to share across handlers. @@ -44,87 +59,43 @@ type config struct { syncInterval time.Duration } -func initConfig() *config { - toDownload := make(map[string]bool) - schemes := strings.Split(downloadEnabledSchemes, ",") - - for _, scheme := range schemes { - s := strings.TrimSpace(scheme) - - if s != "" { - if !isValidSchemeIdentifier(s) { - panic(fmt.Sprintf("%s is not a valid libvarnam supported scheme", s)) - } - - toDownload[s] = true - } +func init() { + // Setup flags to read from user to start the application. + // Initialize 'config' flagset. + flagSet := flag.NewFlagSet("config", flag.ContinueOnError) + flagSet.Usage = func() { + log.Fatal(flagSet.FlagUsages()) } - return &config{upstream: upstreamURL, schemesToDownload: toDownload, - syncInterval: time.Duration(syncIntervalInSecs)} -} - -func (c *config) setDownloadStatus(langCode string, status bool) error { - if !isValidSchemeIdentifier(langCode) { - return fmt.Errorf("%s is not a valid libvarnam supported scheme", langCode) - } + // Create config flag to read 'config.toml' from user. + flagSet.String("config", "config.toml", "Path to the TOML configuration file") - c.schemesToDownload[langCode] = status + // Create flag for version check. + flagSet.Bool("version", false, "Current version of the build") - if status { - // when varnamd was started without any langcodes to sync, the dispatcher won't be running - // in that case, we need to start the dispatcher since we have a new lang code to download now - startSyncDispatcher() + err := flagSet.Parse(os.Args[1:]) + if err != nil { + log.Fatalf("error parsing flags: %v", err) } - return nil -} - -func getConfigDir() string { - if runtime.GOOS == "windows" { - return path.Join(os.Getenv("localappdata"), ".varnamd") + // Load commandline params user given. + if err = kf.Load(posflag.Provider(flagSet, ".", kf), nil); err != nil { + log.Fatal(err.Error()) } - return path.Join(os.Getenv("HOME"), ".varnamd") -} - -func getLogsDir() string { - d := getConfigDir() - logsDir := path.Join(d, "logs") - - if err := os.MkdirAll(logsDir, 0750); err != nil { - panic(err) + // Handle --version flag. Print build version, build date and die. + if kf.Bool("version") { + fmt.Printf("Commit: %v\nBuild: %v\n", buildVersion, buildDate) + os.Exit(0) } - return logsDir -} - -func redirectLogToFile() { - year, month, day := time.Now().Date() - logfile := path.Join(getLogsDir(), fmt.Sprintf("%d-%d-%d.log", year, month, day)) + // Load the config file. + log.Printf("reading config: %s", kf.String("config")) - f, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - panic(err) + if err = kf.Load(file.Provider(kf.String("config")), toml.Parser()); err != nil { + log.Fatalf("error reading config: %v", err) } - log.SetOutput(f) -} - -func init() { - flag.IntVar(&port, "p", 8080, "Run daemon in specified port") - flag.IntVar(&maxHandleCount, "max-handle-count", 10, "Maximum number of handles can be opened for each language") - flag.StringVar(&host, "host", "localhost", "Host for the varnam daemon server") - flag.StringVar(&uiDir, "ui", "ui", "UI directory path") - flag.BoolVar(&enableInternalApis, "enable-internal-apis", false, "Enable internal APIs") - flag.BoolVar(&enableSSL, "enable-ssl", false, "Enables SSL") - flag.StringVar(&certFilePath, "cert-file-path", "", "Certificate file path") - flag.StringVar(&keyFilePath, "key-file-path", "", "Key file path") - flag.StringVar(&upstreamURL, "upstream", "https://api.varnamproject.com", "Provide an upstream server") - flag.StringVar(&downloadEnabledSchemes, "enable-download", "", "Comma separated language identifier for which varnamd will download words from upstream") - flag.IntVar(&syncIntervalInSecs, "sync-interval", 30, "Download interval in seconds") - flag.BoolVar(&logToFile, "log-to-file", true, "If true, logs will be written to a file") - flag.BoolVar(&version, "version", false, "Print the version and exit") } func syncRequired() bool { @@ -143,22 +114,26 @@ func startSyncDispatcher() { } func main() { - runtime.GOMAXPROCS(runtime.NumCPU()) - flag.Parse() + var ( + config appConfig + ) - varnamdConfig = initConfig() - startedAt = time.Now() + runtime.GOMAXPROCS(runtime.NumCPU()) - if version { - fmt.Println(varnamdVersion) - os.Exit(0) + // Read configuration using Koanf. + if err := kf.Unmarshal("app", &config); err != nil { + log.Fatal(err.Error()) } - if logToFile { - redirectLogToFile() + maxHandleCount = kf.Int("app.max-handles") + if maxHandleCount <= 0 { + maxHandleCount = 10 } - log.Printf("varnamd %s", varnamdVersion) + varnamdConfig = initConfig(config) + startedAt = time.Now() + + log.Printf("varnamd %s-%s", buildVersion, buildDate) app := &App{ cache: NewMemCache(), @@ -166,5 +141,5 @@ func main() { } startSyncDispatcher() - startDaemon(app) + startDaemon(app, config) } diff --git a/server.go b/server.go index c9f4c0d..0441c60 100644 --- a/server.go +++ b/server.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "net/http" "os" "path/filepath" @@ -10,30 +9,29 @@ import ( "github.com/labstack/echo/v4/middleware" ) -func startDaemon(app *App) { +func startDaemon(app *App, cfg appConfig) { initLanguageChannels() initLearnChannels() - e, err := initHandlers(app) + e, err := initHandlers(app, cfg.EnableInternalApis) if err != nil { app.log.Fatalf("error initializing handlers: %s", err) } - address := fmt.Sprintf("%s:%d", host, port) - app.log.Printf("Listening on %s", address) + app.log.Printf("Listening on %s", cfg.Address) - if enableSSL { - if err := e.StartTLS(address, certFilePath, keyFilePath); err != nil { + if cfg.EnableSSL { + if err := e.StartTLS(cfg.Address, cfg.CertFilePath, cfg.KeyFilePath); err != nil { app.log.Fatal(err) } } else { - if err := e.Start(address); err != nil { + if err := e.Start(cfg.Address); err != nil { app.log.Fatal(err) } } } -func initHandlers(app *App) (*echo.Echo, error) { +func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { e := echo.New() e.GET("/tl/:langCode/:word", handleTransliteration) From 8741e8b9e9038bd37b7ea823259a987ebce74244 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 15:57:38 +0530 Subject: [PATCH 19/56] added stuffsbin --- go.mod | 1 + go.sum | 2 ++ init.go | 29 +++++++++++++++++++++++++++++ main.go | 9 +++++++-- server.go | 11 +++-------- 5 files changed, 42 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 6ab74b4..4938e4b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/protobuf v1.4.2 // indirect github.com/knadh/koanf v0.12.1 + github.com/knadh/stuffbin v1.1.0 github.com/labstack/echo/v4 v4.1.16 github.com/mattn/go-colorable v0.1.7 // indirect github.com/mattn/go-sqlite3 v1.14.0 diff --git a/go.sum b/go.sum index 646906c..da866de 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/knadh/koanf v0.12.1 h1:9N0asqPUvZ0jL9thTbMZ0FAT3VrGfR+aYm5JTInT+Gs= github.com/knadh/koanf v0.12.1/go.mod h1:31bzRSM7vS5Vm9LNLo7B2Re1zhLOZT6EQKeodixBikE= +github.com/knadh/stuffbin v1.1.0 h1:f5S5BHzZALjuJEgTIOMC9NidEnBJM7Ze6Lu1GHR/lwU= +github.com/knadh/stuffbin v1.1.0/go.mod h1:yVCFaWaKPubSNibBsTAJ939q2ABHudJQxRWZWV5yh+4= github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz0o= github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= diff --git a/init.go b/init.go index 4c76531..9b56d6c 100644 --- a/init.go +++ b/init.go @@ -2,11 +2,14 @@ package main import ( "fmt" + "log" "os" "path" "runtime" "strings" "time" + + "github.com/knadh/stuffbin" ) func initConfig(cfg appConfig) *config { @@ -52,3 +55,29 @@ func getConfigDir() string { return path.Join(os.Getenv("HOME"), ".varnamd") } + +// initVFS initializes the stuffbin virtual FileSystem to provide +// access to bunded static assets to the app. +func initVFS() (stuffbin.FileSystem, error) { + files := []string{ + "ui/", + } + + binPath, err := os.Executable() + if err != nil { + return nil, err + } + + fs, err := stuffbin.UnStuff(binPath) + if err != nil { + log.Printf("unable to initialize embedded filesystem: %v", err) + log.Printf("using local filesystem") + + fs, err = stuffbin.NewLocalFS("/", files...) + if err != nil { + return nil, err + } + } + + return fs, nil +} diff --git a/main.go b/main.go index 3547740..935aca7 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( "github.com/knadh/koanf/parsers/toml" "github.com/knadh/koanf/providers/file" "github.com/knadh/koanf/providers/posflag" + "github.com/knadh/stuffbin" ) const ( @@ -35,8 +36,6 @@ type appConfig struct { var ( kf = koanf.New(".") - uiDir = "ui/" - varnamdConfig *config // config instance used across the application syncDispatcherRunning bool startedAt time.Time @@ -49,6 +48,7 @@ var ( type App struct { cache Cache log *log.Logger + fs stuffbin.FileSystem } // varnamd configurations @@ -135,9 +135,14 @@ func main() { log.Printf("varnamd %s-%s", buildVersion, buildDate) + fs, err := initVFS() + if err != nil { + log.Fatal(err.Error()) + } app := &App{ cache: NewMemCache(), log: log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), + fs: fs, } startSyncDispatcher() diff --git a/server.go b/server.go index 0441c60..971d9bb 100644 --- a/server.go +++ b/server.go @@ -2,8 +2,6 @@ package main import ( "net/http" - "os" - "path/filepath" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" @@ -32,8 +30,9 @@ func startDaemon(app *App, cfg appConfig) { } func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { - e := echo.New() + fSrv := app.fs.FileServer() + e := echo.New() e.GET("/tl/:langCode/:word", handleTransliteration) e.GET("/rtl/:langCode/:word", handleReverseTransliteration) e.GET("/meta/:langCode:", handleMetadata) @@ -42,11 +41,7 @@ func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { e.GET("/languages", handleLanguages) e.GET("/status", handleStatus) - if _, err := os.Stat(filepath.Clean(uiDir)); err != nil { - return nil, err - } - - e.Static("/", filepath.Clean(uiDir)) + e.GET("/", echo.WrapHandler(fSrv)) if enableInternalApis { e.POST("/sync/download/{langCode}/enable", handleEnableDownload) From 1f4acc4489814fbdcfc3f162a457f1a7807d1710 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 16:05:59 +0530 Subject: [PATCH 20/56] added makefile --- Makefile | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d207778 --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +BIN := varnam.bin +HASH := $(shell git rev-parse HEAD | cut -c 1-8) +COMMIT_DATE := $(shell git show -s --format=%ci ${HASH}) +BUILD_DATE := $(shell date '+%Y-%m-%d %H:%M:%S') +VERSION := ${HASH} (${COMMIT_DATE}) +STATIC := ui + +deps: + go get -u github.com/knadh/stuffbin/... + +.PHONY: build +build: ## Build the binary (default) + go build -o ${BIN} -ldflags="-X 'main.buildVersion=${VERSION}' -X 'main.buildDate=${BUILD_DATE}'" + stuffbin -a stuff -in ${BIN} -out ${BIN} ${STATIC} + +.PHONY: run +run: build + ./${BIN} + +.PHONY: clean +clean: ## Remove temporary files and the binary + go clean + +# Absolutely awesome: http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html +.PHONY: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.DEFAULT_GOAL := build \ No newline at end of file From f5e7045ac2a60bbb8b825f0b862f99944568e2f7 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 16:07:50 +0530 Subject: [PATCH 21/56] size reduced --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d207778..457d6ab 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ deps: .PHONY: build build: ## Build the binary (default) - go build -o ${BIN} -ldflags="-X 'main.buildVersion=${VERSION}' -X 'main.buildDate=${BUILD_DATE}'" + go build -o ${BIN} -ldflags="-X 'main.buildVersion=${VERSION}' -X 'main.buildDate=${BUILD_DATE}' -s -w" stuffbin -a stuff -in ${BIN} -out ${BIN} ${STATIC} .PHONY: run From 3e8a5775d76a041d02798a17a1e8cdf6bd6af662 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 17:30:20 +0530 Subject: [PATCH 22/56] added root handler --- Makefile | 2 +- http_handlers.go | 14 ++++++++++++++ init.go | 2 +- server.go | 6 +++--- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 457d6ab..bbd179b 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ HASH := $(shell git rev-parse HEAD | cut -c 1-8) COMMIT_DATE := $(shell git show -s --format=%ci ${HASH}) BUILD_DATE := $(shell date '+%Y-%m-%d %H:%M:%S') VERSION := ${HASH} (${COMMIT_DATE}) -STATIC := ui +STATIC := ui:/ deps: go get -u github.com/knadh/stuffbin/... diff --git a/http_handlers.go b/http_handlers.go index 67e7a28..42398d6 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -306,3 +306,17 @@ func handleDisableDownload(c echo.Context) error { return c.JSON(http.StatusOK, data) } + +// handleIndex is the root handler that renders the Javascript frontend. +func handleIndex(c echo.Context) error { + app, _ := c.Get("app").(*App) + + b, err := app.fs.Read("/index.html") + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + c.Response().Header().Set("Content-Type", "text/html") + + return c.String(http.StatusOK, string(b)) +} diff --git a/init.go b/init.go index 9b56d6c..710b777 100644 --- a/init.go +++ b/init.go @@ -60,7 +60,7 @@ func getConfigDir() string { // access to bunded static assets to the app. func initVFS() (stuffbin.FileSystem, error) { files := []string{ - "ui/", + "ui:/", } binPath, err := os.Executable() diff --git a/server.go b/server.go index 971d9bb..f243c92 100644 --- a/server.go +++ b/server.go @@ -30,8 +30,6 @@ func startDaemon(app *App, cfg appConfig) { } func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { - fSrv := app.fs.FileServer() - e := echo.New() e.GET("/tl/:langCode/:word", handleTransliteration) e.GET("/rtl/:langCode/:word", handleReverseTransliteration) @@ -41,7 +39,9 @@ func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { e.GET("/languages", handleLanguages) e.GET("/status", handleStatus) - e.GET("/", echo.WrapHandler(fSrv)) + e.GET("/", handleIndex) + + e.GET("/*", echo.WrapHandler(app.fs.FileServer())) if enableInternalApis { e.POST("/sync/download/{langCode}/enable", handleEnableDownload) From 6745ef7503572d7adc6bf74940a5645f2c3da904 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 17:48:47 +0530 Subject: [PATCH 23/56] added defaults --- main.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 935aca7..5655173 100644 --- a/main.go +++ b/main.go @@ -93,7 +93,7 @@ func init() { log.Printf("reading config: %s", kf.String("config")) if err = kf.Load(file.Provider(kf.String("config")), toml.Parser()); err != nil { - log.Fatalf("error reading config: %v", err) + log.Printf("error reading config: %v", err) } } @@ -125,6 +125,23 @@ func main() { log.Fatal(err.Error()) } + // If address is empty run on localhost port 8080. + if config.Address == "" { + config.Address = ":8080" + } + + if config.EnableSSL && (config.CertFilePath == "" || config.KeyFilePath == "") { + config.EnableSSL = false + } + + if config.SyncInterval < 1*time.Second { + config.SyncInterval = 30 * time.Second + } + + if config.UpstreamURL == "" { + config.UpstreamURL = "https://api.varnamproject.com" + } + maxHandleCount = kf.Int("app.max-handles") if maxHandleCount <= 0 { maxHandleCount = 10 From 41e2b4c2cecdf17899ea1a753212177ba32c55d4 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sat, 12 Sep 2020 11:07:20 +0530 Subject: [PATCH 24/56] added logging to handlers --- cache.go | 1 + http_handlers.go | 33 ++++++++++++++++++++++++++++----- server.go | 9 +++------ 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/cache.go b/cache.go index 5994de4..ca36f9d 100644 --- a/cache.go +++ b/cache.go @@ -48,6 +48,7 @@ func (c *MemCache) setWithExpiry(lang, word string, val []byte, expiry int) erro // Get lang-word from cache. func (c *MemCache) Get(lang, word string) ([]string, error) { var key = fmt.Sprintf("%s-%s", lang, word) + val, err := c.fc.Get([]byte(key)) if err != nil { return nil, err diff --git a/http_handlers.go b/http_handlers.go index 49a72a5..ac2321e 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -93,11 +93,12 @@ func handleTransliteration(c echo.Context) error { if err != nil { w, err := transliterate(langCode, word) if err != nil { + app.log.Printf("error in transliterationg, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) } - words = w.([]string) - app.cache.Set(langCode, word, words...) + words, _ = w.([]string) + _ = app.cache.Set(langCode, word, words...) } return c.JSON(http.StatusOK, transliterationResponse{standardResponse: newStandardResponse(), Result: words, Input: word}) @@ -114,14 +115,16 @@ func handleReverseTransliteration(c echo.Context) error { if err != nil { res, err := reveseTransliterate(langCode, word) if err != nil { + app.log.Printf("error in reverse transliterationg, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) } result = []string{res.(string)} - app.cache.Set(langCode, word, res.(string)) + _ = app.cache.Set(langCode, word, res.(string)) } if len(result) <= 0 { + app.log.Printf("no reverse transliteration found for lang: %s word: %s", langCode, word) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("no transliteration found for lanugage: %s, word: %s", langCode, word)) } @@ -137,7 +140,10 @@ func handleReverseTransliteration(c echo.Context) error { } func handleMetadata(c echo.Context) error { - var schemeIdentifier = c.Param("langCode") + var ( + schemeIdentifier = c.Param("langCode") + app = c.Get("app").(*App) + ) data, err := getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { details, err := handle.GetCorpusDetails() @@ -148,6 +154,7 @@ func handleMetadata(c echo.Context) error { return &metaResponse{Result: details, standardResponse: newStandardResponse()}, nil }) if err != nil { + app.log.Printf("error in getting corpus details for: %s, err: %s", schemeIdentifier, err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } @@ -158,6 +165,8 @@ func handleDownload(c echo.Context) error { var ( langCode = c.Param("langCode") start, _ = strconv.Atoi(c.Param("downloadStart")) + + app = c.Get("app").(*App) ) if start < 0 { @@ -227,10 +236,13 @@ func handleDownload(c echo.Context) error { return c.Blob(http.StatusOK, "application/json; charset=utf-8", ctx.Data) } + app.log.Printf("error in fetching deta from cache: %s, err: %s", langCode, err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } c.Response().Header().Set("Content-Encoding", "gzip") + return c.Blob(http.StatusOK, "application/json; charset=utf-8", data) } @@ -239,16 +251,22 @@ func handleLanguages(c echo.Context) error { } func handlLearn(c echo.Context) error { - var a args + var ( + a args + + app = c.Get("app").(*App) + ) c.Request().Header.Set("Content-Type", "application/json") if err := c.Bind(&a); err != nil { + app.log.Printf("error in binding request details for learn, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } ch, ok := learnChannels[a.LangCode] if !ok { + app.log.Printf("unknown language requested to learn: %s", a.LangCode) return echo.NewHTTPError(http.StatusBadRequest, "unable to find language") } @@ -286,10 +304,13 @@ func toggleDownloadEnabledStatus(langCode string, status bool) (interface{}, err func handleEnableDownload(c echo.Context) error { var ( langCode = c.Param("langCode") + + app = c.Get("app").(*App) ) data, err := toggleDownloadEnabledStatus(langCode, true) if err != nil { + app.log.Printf("failed to toggle download enable, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } @@ -299,10 +320,12 @@ func handleEnableDownload(c echo.Context) error { func handleDisableDownload(c echo.Context) error { var ( langCode = c.Param("langCode") + app = c.Get("app").(*App) ) data, err := toggleDownloadEnabledStatus(langCode, false) if err != nil { + app.log.Printf("failed to disable download, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } diff --git a/server.go b/server.go index 95f4332..1fae24b 100644 --- a/server.go +++ b/server.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "log" "net/http" "os" "path/filepath" @@ -12,7 +11,6 @@ import ( ) func startDaemon(app *App) { - initLanguageChannels() initLearnChannels() @@ -22,18 +20,17 @@ func startDaemon(app *App) { } address := fmt.Sprintf("%s:%d", host, port) - log.Printf("Listening on %s", address) + app.log.Printf("Listening on %s", address) if enableSSL { if err := e.StartTLS(address, certFilePath, keyFilePath); err != nil { - log.Fatal(err) + app.log.Fatal(err) } } else { if err := e.Start(address); err != nil { - log.Fatal(err) + app.log.Fatal(err) } } - } func initHandlers(app *App) (*echo.Echo, error) { From 2140ed0021e611eb452bec8e8fd217d53cef3cfa Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Tue, 15 Sep 2020 01:44:58 +0530 Subject: [PATCH 25/56] added logger for errors in train endpoint --- http_handlers.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/http_handlers.go b/http_handlers.go index a426652..4201485 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -11,7 +11,7 @@ import ( "strconv" "strings" "time" - + "github.com/golang/groupcache" "github.com/labstack/echo/v4" "github.com/varnamproject/varnamd/libvarnam" @@ -281,13 +281,16 @@ func handleTrain(c echo.Context) error { c.Request().Header.Set("Content-Type", "application/json") if err := c.Bind(&targs); err != nil { + app.log.Printf("error reading request, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } handle, err := libvarnam.Init(targs.Lang) if err != nil { + app.log.Printf("libvarnam init failed, err: %s", err.Error()) return fmt.Errorf("failed to get data") } if err := handle.Train(targs.Pattern, targs.Word); err != nil { + app.log.Printf("error training word: %s, pattern: %s, err:%s", targs.Word, targs.Pattern, err.Error()) return fmt.Errorf("failed to Train %s. %s", targs.Word, err.Error()) } return c.JSON(200, "Word Trained") From eacef025d6c3ef5fc605035ec78f0ffebfbc2a8f Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Fri, 11 Sep 2020 17:35:48 +0530 Subject: [PATCH 26/56] added cache for tl --- cache.go | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 6 +++++ http_handlers.go | 13 ++++++++--- main.go | 11 +++++++++- server.go | 11 +++++++++- 6 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 cache.go diff --git a/cache.go b/cache.go new file mode 100644 index 0000000..c380b0a --- /dev/null +++ b/cache.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + "strings" + "time" + + "github.com/coocood/freecache" +) + +const ( + defaultCacheSize = 100 << 20 // 100 MB cache + defaultExpiry = time.Hour * 24 // Cache for 24 hours + wordSeparator = "|" // Assuming the separator wont be used in any case. +) + +// Cache objects. +type Cache interface { + Set(lang, word string, val []string) error + Get(lang, word string) ([]string, error) +} + +// MemCache impliments Cache interface. +type MemCache struct { + fc *freecache.Cache +} + +// NewMemCache will create object of new Cache impliemtation. +func NewMemCache() *MemCache { + return newCacheWithSize(defaultCacheSize) +} + +func newCacheWithSize(size int) *MemCache { + return &MemCache{fc: freecache.NewCache(size)} +} + +// Set lang-word as val to cache. +func (c *MemCache) Set(lang, word string, val []string) error { + var value = strings.Join(val, "|") + return c.setWithExpiry(lang, word, []byte(value), int(defaultExpiry)) +} + +func (c *MemCache) setWithExpiry(lang, word string, val []byte, expiry int) error { + var key = fmt.Sprintf("%s-%s", lang, word) + return c.fc.Set([]byte(key), val, expiry) +} + +// Get lang-word from cache. +func (c *MemCache) Get(lang, word string) ([]string, error) { + var key = fmt.Sprintf("%s-%s", lang, word) + val, err := c.fc.Get([]byte(key)) + if err != nil { + return nil, err + } + + return strings.Split(string(val), wordSeparator), nil +} diff --git a/go.mod b/go.mod index a7fbcdd..64da2d8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/varnamproject/varnamd go 1.14 require ( + github.com/coocood/freecache v1.1.1 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/protobuf v1.4.2 // indirect github.com/labstack/echo/v4 v4.1.16 diff --git a/go.sum b/go.sum index a234f57..eefcea3 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,10 @@ +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc= +github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= @@ -35,6 +40,7 @@ github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/ github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/http_handlers.go b/http_handlers.go index 25dd79a..b2e0198 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -79,14 +79,21 @@ func handleTransliteration(c echo.Context) error { var ( langCode = c.Param("langCode") word = c.Param("word") + app = c.Get("app").(*App) ) - words, err := transliterate(langCode, word) + words, err := app.cache.Get(langCode, word) if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) + w, err := transliterate(langCode, word) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) + } + + words = w.([]string) + app.cache.Set(langCode, word, words) } - return c.JSON(http.StatusOK, transliterationResponse{standardResponse: newStandardResponse(), Result: words.([]string), Input: word}) + return c.JSON(http.StatusOK, transliterationResponse{standardResponse: newStandardResponse(), Result: words, Input: word}) } func handleReverseTransliteration(c echo.Context) error { diff --git a/main.go b/main.go index 7bf84d3..9de0c43 100644 --- a/main.go +++ b/main.go @@ -30,6 +30,11 @@ var ( syncDispatcherRunning bool ) +// App is a singleton to share across handlers. +type App struct { + cache Cache +} + // varnamd configurations // this is populated from various command line flags type config struct { @@ -154,6 +159,10 @@ func main() { log.Printf("varnamd %s", varnamdVersion) + app := &App{ + cache: NewMemCache(), + } + startSyncDispatcher() - startDaemon() + startDaemon(app) } diff --git a/server.go b/server.go index b0529c4..9d03005 100644 --- a/server.go +++ b/server.go @@ -11,7 +11,8 @@ import ( "github.com/labstack/echo/v4/middleware" ) -func startDaemon() { +func startDaemon(app *App) { + initLanguageChannels() initLearnChannels() @@ -42,6 +43,14 @@ func startDaemon() { e.Use(middleware.Recover()) e.Use(middleware.Logger()) + // Custom middleware to set sigleton to app's context. + e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + c.Set("app", app) + return next(c) + } + }) + e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ AllowOrigins: []string{"*"}, AllowMethods: []string{http.MethodOptions}, From 9ad527fc88c315ff412b4b0333bfc206a6cf55e5 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Fri, 11 Sep 2020 22:51:59 +0530 Subject: [PATCH 27/56] added caching for rtl --- cache.go | 8 ++++---- http_handlers.go | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/cache.go b/cache.go index c380b0a..5994de4 100644 --- a/cache.go +++ b/cache.go @@ -11,12 +11,12 @@ import ( const ( defaultCacheSize = 100 << 20 // 100 MB cache defaultExpiry = time.Hour * 24 // Cache for 24 hours - wordSeparator = "|" // Assuming the separator wont be used in any case. + wordSeparator = "<>" // Assuming the separator wont be used in any case. ) // Cache objects. type Cache interface { - Set(lang, word string, val []string) error + Set(lang, word string, val ...string) error Get(lang, word string) ([]string, error) } @@ -35,8 +35,8 @@ func newCacheWithSize(size int) *MemCache { } // Set lang-word as val to cache. -func (c *MemCache) Set(lang, word string, val []string) error { - var value = strings.Join(val, "|") +func (c *MemCache) Set(lang, word string, val ...string) error { + var value = strings.Join(val, wordSeparator) return c.setWithExpiry(lang, word, []byte(value), int(defaultExpiry)) } diff --git a/http_handlers.go b/http_handlers.go index b2e0198..f67620b 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -90,7 +90,7 @@ func handleTransliteration(c echo.Context) error { } words = w.([]string) - app.cache.Set(langCode, word, words) + app.cache.Set(langCode, word, words...) } return c.JSON(http.StatusOK, transliterationResponse{standardResponse: newStandardResponse(), Result: words, Input: word}) @@ -100,11 +100,22 @@ func handleReverseTransliteration(c echo.Context) error { var ( langCode = c.Param("langCode") word = c.Param("word") + app = c.Get("app").(*App) ) - result, err := reveseTransliterate(langCode, word) + result, err := app.cache.Get(langCode, word) if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) + res, err := reveseTransliterate(langCode, word) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) + } + + result = []string{res.(string)} + app.cache.Set(langCode, word, res.(string)) + } + + if len(result) <= 0 { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("no transliteration found for lanugage: %s, word: %s", langCode, word)) } response := struct { @@ -112,7 +123,7 @@ func handleReverseTransliteration(c echo.Context) error { Result string `json:"result"` }{ newStandardResponse(), - result.(string), + result[0], } return c.JSON(http.StatusOK, response) From c0e625d063a79f451b98e71eacbfecf30f647671 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sat, 12 Sep 2020 10:41:10 +0530 Subject: [PATCH 28/56] changed handler to another function --- main.go | 2 ++ server.go | 36 +++++++++++++++++++++++------------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/main.go b/main.go index 9de0c43..d47c2d3 100644 --- a/main.go +++ b/main.go @@ -33,6 +33,7 @@ var ( // App is a singleton to share across handlers. type App struct { cache Cache + log *log.Logger } // varnamd configurations @@ -161,6 +162,7 @@ func main() { app := &App{ cache: NewMemCache(), + log: log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), } startSyncDispatcher() diff --git a/server.go b/server.go index 9d03005..0826d74 100644 --- a/server.go +++ b/server.go @@ -16,6 +16,27 @@ func startDaemon(app *App) { initLanguageChannels() initLearnChannels() + e, err := initHandlers(app) + if err != nil { + app.log.Fatalf("error initializing handlers: %s", err) + } + + address := fmt.Sprintf("%s:%d", host, port) + log.Printf("Listening on %s", address) + + if enableSSL { + if err := e.StartTLS(address, certFilePath, keyFilePath); err != nil { + log.Fatal(err) + } + } else { + if err := e.Start(address); err != nil { + log.Fatal(err) + } + } + +} + +func initHandlers(app *App) (*echo.Echo, error) { e := echo.New() e.GET("/tl/:langCode/:word", handleTransliteration) @@ -27,7 +48,7 @@ func startDaemon(app *App) { e.GET("/status", handleStatus) if _, err := os.Stat(filepath.Clean(uiDir)); err != nil { - log.Fatal("UI path doesnot exist", err) + return nil, err } e.Static("/", filepath.Clean(uiDir)) @@ -37,9 +58,6 @@ func startDaemon(app *App) { e.POST("/sync/download/{langCode}/disable", handleDisableDownload) } - address := fmt.Sprintf("%s:%d", host, port) - log.Printf("Listening on %s", address) - e.Use(middleware.Recover()) e.Use(middleware.Logger()) @@ -64,13 +82,5 @@ func startDaemon(app *App) { // ContentSecurityPolicy: "default-src 'self'", })) - if enableSSL { - if err := e.StartTLS(address, certFilePath, keyFilePath); err != nil { - log.Fatal(err) - } - } else { - if err := e.Start(address); err != nil { - log.Fatal(err) - } - } + return e, nil } From 8be76f3920476e97960ed772b8dfe88c906524d8 Mon Sep 17 00:00:00 2001 From: Athul Cyriac Ajay Date: Sat, 12 Sep 2020 20:12:49 +0530 Subject: [PATCH 29/56] Implement Train Endpoint --- go.mod | 2 +- go.sum | 10 ++++++++-- http_handlers.go | 37 ++++++++++++++++++++++++++++++++++++- server.go | 1 + sync.go | 2 +- varnam_handlers.go | 8 +++++++- varnam_learn.go | 2 +- 7 files changed, 55 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 64da2d8..c7257c7 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/varnamproject/varnamd go 1.14 require ( + github.com/athul/go-libvarnam v0.0.0-20200912100303-48fdad5ac70e github.com/coocood/freecache v1.1.1 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/protobuf v1.4.2 // indirect @@ -10,7 +11,6 @@ require ( github.com/mattn/go-colorable v0.1.7 // indirect github.com/mattn/go-sqlite3 v1.14.0 github.com/valyala/fasttemplate v1.2.0 // indirect - github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect diff --git a/go.sum b/go.sum index eefcea3..18b3b15 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,9 @@ +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/athul/go-libvarnam v0.0.0-20200912100303-48fdad5ac70e h1:zKSW8/mOy4aYFghtsAFXNrq84IbjmFvqX3lAy0hqb7M= +github.com/athul/go-libvarnam v0.0.0-20200912100303-48fdad5ac70e/go.mod h1:QRvlfDTVGxOgZYm8toOEXFVAmSQXCvjq3zWBZBfHYig= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc= @@ -40,18 +43,19 @@ github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/ github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.0 h1:y3yXRCoDvC2HTtIHvL2cc7Zd+bqA+zqDO6oQzsJO07E= github.com/valyala/fasttemplate v1.2.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932 h1:Z6VhoopafnsFFYf8KwGty1dmJteTCM/mLGRAvc81QPk= -github.com/varnamproject/libvarnam-golang v0.0.0-20150608021401-b24ba4c3d932/go.mod h1:p0y0nS7FVUFgPK6dOkEY/NiOI7+aBeYgrl4ACVFuLIo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -95,3 +99,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/http_handlers.go b/http_handlers.go index f67620b..5908977 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -12,9 +12,9 @@ import ( "strings" "time" + "github.com/athul/go-libvarnam" "github.com/golang/groupcache" "github.com/labstack/echo/v4" - "github.com/varnamproject/libvarnam-golang" ) var errCacheSkipped = errors.New("cache skipped") @@ -59,6 +59,13 @@ type args struct { Text string `json:"text"` } +//TrainArgs read the incoming data +type TrainArgs struct { + Lang string `json:"lang" form:"lang"` + Pattern string `json:"pattern" form:"pattern"` + Word string `json:"word" form:"word"` +} + func handleStatus(c echo.Context) error { uptime := time.Since(startedAt) @@ -250,6 +257,34 @@ func handleLearn(c echo.Context) error { return c.JSON(http.StatusOK, "success") } +func handleTrain(c echo.Context) error { + var targs TrainArgs + fmt.Println(1) + // c.Request().Header.Set("Content-Type", "application/json") + + // if err := c.Bind(&targs); err != nil { + // fmt.Println("Error 1") + // return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + // } + targs.Lang = c.FormValue("lang") + targs.Pattern = c.FormValue("pattern") + targs.Word = c.FormValue("word") + + fmt.Println(targs) + handle, err := libvarnam.Init(targs.Lang) + if err != nil { + fmt.Println("Error 2") + return fmt.Errorf("failed to get data") + } + fmt.Println(3) + if err := handle.Train(targs.Pattern, targs.Word); err != nil { + fmt.Println("Error Trains............") + return fmt.Errorf("failed to Train %s. %s", targs.Word, err.Error()) + } + fmt.Println(4) + return c.JSON(200, "Word Trained") + +} func toggleDownloadEnabledStatus(langCode string, status bool) (interface{}, error) { if err := varnamdConfig.setDownloadStatus(langCode, status); err != nil { return nil, err diff --git a/server.go b/server.go index 0826d74..4b29b80 100644 --- a/server.go +++ b/server.go @@ -46,6 +46,7 @@ func initHandlers(app *App) (*echo.Echo, error) { e.POST("/learn", handleLearn) e.GET("/languages", handleLanguages) e.GET("/status", handleStatus) + e.POST("/train", handleTrain) if _, err := os.Stat(filepath.Clean(uiDir)); err != nil { return nil, err diff --git a/sync.go b/sync.go index dd1295d..4d39707 100644 --- a/sync.go +++ b/sync.go @@ -13,7 +13,7 @@ import ( "strings" "time" - "github.com/varnamproject/libvarnam-golang" + "github.com/athul/go-libvarnam" ) type syncDispatcher struct { diff --git a/varnam_handlers.go b/varnam_handlers.go index 0f52e9c..b401384 100644 --- a/varnam_handlers.go +++ b/varnam_handlers.go @@ -7,9 +7,9 @@ import ( "sync" "time" + "github.com/athul/go-libvarnam" "github.com/golang/groupcache" _ "github.com/mattn/go-sqlite3" - "github.com/varnamproject/libvarnam-golang" ) type word struct { @@ -97,6 +97,12 @@ func transliterate(schemeIdentifier string, word string) (interface{}, error) { }) } +// func trainwords(schemeIdentifier, word, pattern string) (interface{}, error) { +// return getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { +// return handle.Train(pattern,word) +// }) +// } + func getWords(schemeIdentifier string, downloadStart int) ([]*word, error) { filepath, _ := getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { return handle.GetSuggestionsFilePath(), nil diff --git a/varnam_learn.go b/varnam_learn.go index 6dbba4f..4917440 100644 --- a/varnam_learn.go +++ b/varnam_learn.go @@ -3,7 +3,7 @@ package main import ( "log" - "github.com/varnamproject/libvarnam-golang" + "github.com/athul/go-libvarnam" ) var ( From bbb706e2767d2332971816fd504b67530583f0a9 Mon Sep 17 00:00:00 2001 From: Athul Cyriac Ajay Date: Sun, 13 Sep 2020 14:22:57 +0530 Subject: [PATCH 30/56] Moved the Train Request types to JSON --- http_handlers.go | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/http_handlers.go b/http_handlers.go index 5908977..1cde305 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -259,29 +259,19 @@ func handleLearn(c echo.Context) error { func handleTrain(c echo.Context) error { var targs TrainArgs - fmt.Println(1) - // c.Request().Header.Set("Content-Type", "application/json") - - // if err := c.Bind(&targs); err != nil { - // fmt.Println("Error 1") - // return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) - // } - targs.Lang = c.FormValue("lang") - targs.Pattern = c.FormValue("pattern") - targs.Word = c.FormValue("word") - - fmt.Println(targs) + + c.Request().Header.Set("Content-Type", "application/json") + + if err := c.Bind(&targs); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } handle, err := libvarnam.Init(targs.Lang) if err != nil { - fmt.Println("Error 2") return fmt.Errorf("failed to get data") } - fmt.Println(3) if err := handle.Train(targs.Pattern, targs.Word); err != nil { - fmt.Println("Error Trains............") return fmt.Errorf("failed to Train %s. %s", targs.Word, err.Error()) } - fmt.Println(4) return c.JSON(200, "Word Trained") } From 2930197001aa662dd5c5bec498b162fe5a4d5c66 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sat, 12 Sep 2020 11:07:20 +0530 Subject: [PATCH 31/56] added logging to handlers --- cache.go | 1 + http_handlers.go | 35 +++++++++++++++++++++++++++++------ server.go | 9 +++------ 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/cache.go b/cache.go index 5994de4..ca36f9d 100644 --- a/cache.go +++ b/cache.go @@ -48,6 +48,7 @@ func (c *MemCache) setWithExpiry(lang, word string, val []byte, expiry int) erro // Get lang-word from cache. func (c *MemCache) Get(lang, word string) ([]string, error) { var key = fmt.Sprintf("%s-%s", lang, word) + val, err := c.fc.Get([]byte(key)) if err != nil { return nil, err diff --git a/http_handlers.go b/http_handlers.go index 1cde305..ac2321e 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -93,11 +93,12 @@ func handleTransliteration(c echo.Context) error { if err != nil { w, err := transliterate(langCode, word) if err != nil { + app.log.Printf("error in transliterationg, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) } - words = w.([]string) - app.cache.Set(langCode, word, words...) + words, _ = w.([]string) + _ = app.cache.Set(langCode, word, words...) } return c.JSON(http.StatusOK, transliterationResponse{standardResponse: newStandardResponse(), Result: words, Input: word}) @@ -114,14 +115,16 @@ func handleReverseTransliteration(c echo.Context) error { if err != nil { res, err := reveseTransliterate(langCode, word) if err != nil { + app.log.Printf("error in reverse transliterationg, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) } result = []string{res.(string)} - app.cache.Set(langCode, word, res.(string)) + _ = app.cache.Set(langCode, word, res.(string)) } if len(result) <= 0 { + app.log.Printf("no reverse transliteration found for lang: %s word: %s", langCode, word) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("no transliteration found for lanugage: %s, word: %s", langCode, word)) } @@ -137,7 +140,10 @@ func handleReverseTransliteration(c echo.Context) error { } func handleMetadata(c echo.Context) error { - var schemeIdentifier = c.Param("langCode") + var ( + schemeIdentifier = c.Param("langCode") + app = c.Get("app").(*App) + ) data, err := getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { details, err := handle.GetCorpusDetails() @@ -148,6 +154,7 @@ func handleMetadata(c echo.Context) error { return &metaResponse{Result: details, standardResponse: newStandardResponse()}, nil }) if err != nil { + app.log.Printf("error in getting corpus details for: %s, err: %s", schemeIdentifier, err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } @@ -158,6 +165,8 @@ func handleDownload(c echo.Context) error { var ( langCode = c.Param("langCode") start, _ = strconv.Atoi(c.Param("downloadStart")) + + app = c.Get("app").(*App) ) if start < 0 { @@ -227,10 +236,13 @@ func handleDownload(c echo.Context) error { return c.Blob(http.StatusOK, "application/json; charset=utf-8", ctx.Data) } + app.log.Printf("error in fetching deta from cache: %s, err: %s", langCode, err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } c.Response().Header().Set("Content-Encoding", "gzip") + return c.Blob(http.StatusOK, "application/json; charset=utf-8", data) } @@ -238,17 +250,23 @@ func handleLanguages(c echo.Context) error { return c.JSON(http.StatusOK, schemeDetails) } -func handleLearn(c echo.Context) error { - var a args +func handlLearn(c echo.Context) error { + var ( + a args + + app = c.Get("app").(*App) + ) c.Request().Header.Set("Content-Type", "application/json") if err := c.Bind(&a); err != nil { + app.log.Printf("error in binding request details for learn, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } ch, ok := learnChannels[a.LangCode] if !ok { + app.log.Printf("unknown language requested to learn: %s", a.LangCode) return echo.NewHTTPError(http.StatusBadRequest, "unable to find language") } @@ -286,10 +304,13 @@ func toggleDownloadEnabledStatus(langCode string, status bool) (interface{}, err func handleEnableDownload(c echo.Context) error { var ( langCode = c.Param("langCode") + + app = c.Get("app").(*App) ) data, err := toggleDownloadEnabledStatus(langCode, true) if err != nil { + app.log.Printf("failed to toggle download enable, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } @@ -299,10 +320,12 @@ func handleEnableDownload(c echo.Context) error { func handleDisableDownload(c echo.Context) error { var ( langCode = c.Param("langCode") + app = c.Get("app").(*App) ) data, err := toggleDownloadEnabledStatus(langCode, false) if err != nil { + app.log.Printf("failed to disable download, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } diff --git a/server.go b/server.go index 4b29b80..5622fac 100644 --- a/server.go +++ b/server.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "log" "net/http" "os" "path/filepath" @@ -12,7 +11,6 @@ import ( ) func startDaemon(app *App) { - initLanguageChannels() initLearnChannels() @@ -22,18 +20,17 @@ func startDaemon(app *App) { } address := fmt.Sprintf("%s:%d", host, port) - log.Printf("Listening on %s", address) + app.log.Printf("Listening on %s", address) if enableSSL { if err := e.StartTLS(address, certFilePath, keyFilePath); err != nil { - log.Fatal(err) + app.log.Fatal(err) } } else { if err := e.Start(address); err != nil { - log.Fatal(err) + app.log.Fatal(err) } } - } func initHandlers(app *App) (*echo.Echo, error) { From 769bb561f01fe7c8475946eb2e358658424dce65 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 14:28:37 +0530 Subject: [PATCH 32/56] removed go-libvarnam dep --- http_handlers.go | 2 +- libvarnam/libvarnam.go | 201 +++++++++++++++++++++++++++++++++++++++++ sync.go | 2 +- varnam_handlers.go | 2 +- varnam_learn.go | 2 +- 5 files changed, 205 insertions(+), 4 deletions(-) create mode 100644 libvarnam/libvarnam.go diff --git a/http_handlers.go b/http_handlers.go index ac2321e..987e3bf 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -12,9 +12,9 @@ import ( "strings" "time" - "github.com/athul/go-libvarnam" "github.com/golang/groupcache" "github.com/labstack/echo/v4" + "github.com/varnamproject/varnamd/libvarnam" ) var errCacheSkipped = errors.New("cache skipped") diff --git a/libvarnam/libvarnam.go b/libvarnam/libvarnam.go new file mode 100644 index 0000000..4c1bdc1 --- /dev/null +++ b/libvarnam/libvarnam.go @@ -0,0 +1,201 @@ +package libvarnam + +// #cgo pkg-config: varnam +// #include +import "C" +import "fmt" + +// Varnam app binding. +type Varnam struct { + handle *C.varnam +} + +// VarnamError satisfies error interface. +type VarnamError struct { + errorCode int + message string +} + +// SchemeDetails returns language and other changes. +type SchemeDetails struct { + LangCode string + Identifier string + DisplayName string + Author string + CompiledDate string + IsStable bool +} + +// CorpusDetails returns corpus details. +type CorpusDetails struct { + WordsCount int `json:"wordsCount"` +} + +// LearnStatus . +type LearnStatus struct { + TotalWords int + Failed int +} + +func (e *VarnamError) Error() string { + return e.message +} + +// GetSuggestionsFilePath returns suggestions. +func (v *Varnam) GetSuggestionsFilePath() string { + return C.GoString(C.varnam_get_suggestions_file(v.handle)) +} + +// GetCorpusDetails will return corpus details. +func (v *Varnam) GetCorpusDetails() (*CorpusDetails, error) { + var details *C.vcorpus_details + + rc := C.varnam_get_corpus_details(v.handle, &details) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return nil, &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return &CorpusDetails{WordsCount: int(details.wordsCount)}, nil +} + +// Transliterate given string to corresponding language. +func (v *Varnam) Transliterate(text string) ([]string, error) { + var va *C.varray + rc := C.varnam_transliterate(v.handle, C.CString(text), &va) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return nil, &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + var ( + i C.int + words []string + ) + + for i = 0; i < C.varray_length(va); i++ { + word := (*C.vword)(C.varray_get(va, i)) + words = append(words, C.GoString(word.text)) + } + + return words, nil +} + +// ReverseTransliterate given string. +func (v *Varnam) ReverseTransliterate(text string) (string, error) { + var output *C.char + rc := C.varnam_reverse_transliterate(v.handle, C.CString(text), &output) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return "", &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return C.GoString(output), nil +} + +// LearnFromFile learns from file from the given filepath. +func (v *Varnam) LearnFromFile(filePath string) (*LearnStatus, error) { + var status C.vlearn_status + rc := C.varnam_learn_from_file(v.handle, C.CString(filePath), &status, nil, nil) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return nil, &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return &LearnStatus{TotalWords: int(status.total_words), Failed: int(status.failed)}, nil +} + +// Init initializes varnam bindings. +func Init(schemeIdentifier string) (*Varnam, error) { + var ( + v *C.varnam + msg *C.char + ) + + rc := C.varnam_init_from_id(C.CString(schemeIdentifier), &v, &msg) + + if rc != C.VARNAM_SUCCESS { + return nil, &VarnamError{errorCode: (int)(rc), message: C.GoString(msg)} + } + + return &Varnam{handle: v}, nil +} + +// GetAllSchemeDetails returns all scheme related details. +func GetAllSchemeDetails() []*SchemeDetails { + allHandles := C.varnam_get_all_handles() + + if allHandles == nil { + return []*SchemeDetails{} + } + + var schemeDetails []*SchemeDetails + + length := int(C.varray_length(allHandles)) + + for i := 0; i < length; i++ { + var detail *C.vscheme_details + + handle := (*C.varnam)(C.varray_get(allHandles, C.int(i))) + rc := C.varnam_get_scheme_details(handle, &detail) + + if rc != C.VARNAM_SUCCESS { + return []*SchemeDetails{} + } + + schemeDetails = append(schemeDetails, &SchemeDetails{ + LangCode: C.GoString(detail.langCode), Identifier: C.GoString(detail.identifier), + DisplayName: C.GoString(detail.displayName), Author: C.GoString(detail.author), + CompiledDate: C.GoString(detail.compiledDate), IsStable: detail.isStable > 0}) + + C.varnam_destroy(handle) + } + + return schemeDetails +} + +// Learn from given input text. +func (v *Varnam) Learn(text string) error { + rc := C.varnam_learn(v.handle, C.CString(text)) + + if rc != 0 { + errorCode := (int)(rc) + + return &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return nil +} + +func (v *Varnam) getVarnamError(errorCode int) string { + errormessage := C.varnam_get_last_error(v.handle) + varnamErrorMsg := C.GoString(errormessage) + + return fmt.Sprintf("%d:%s", errorCode, varnamErrorMsg) +} + +// Destroy closes handle. +func (v *Varnam) Destroy() { + C.varnam_destroy(v.handle) +} + +// Train methods trains with the word and pattern,eg: pattern=chrome,word=ക്രോം +func (v *Varnam) Train(pattern, word string) error { + rc := C.varnam_train(v.handle, C.CString(pattern), C.CString(word)) + + if rc != 0 { + errorCode := (int)(rc) + + return &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return nil +} diff --git a/sync.go b/sync.go index 4d39707..5b9c4ae 100644 --- a/sync.go +++ b/sync.go @@ -13,7 +13,7 @@ import ( "strings" "time" - "github.com/athul/go-libvarnam" + "github.com/varnamproject/varnamd/libvarnam" ) type syncDispatcher struct { diff --git a/varnam_handlers.go b/varnam_handlers.go index b401384..a0723cd 100644 --- a/varnam_handlers.go +++ b/varnam_handlers.go @@ -7,9 +7,9 @@ import ( "sync" "time" - "github.com/athul/go-libvarnam" "github.com/golang/groupcache" _ "github.com/mattn/go-sqlite3" + "github.com/varnamproject/varnamd/libvarnam" ) type word struct { diff --git a/varnam_learn.go b/varnam_learn.go index 4917440..fdb8f29 100644 --- a/varnam_learn.go +++ b/varnam_learn.go @@ -3,7 +3,7 @@ package main import ( "log" - "github.com/athul/go-libvarnam" + "github.com/varnamproject/varnamd/libvarnam" ) var ( From 2ed3ad47bfa5e676b2073ccd026a030c04dfa342 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 15:38:10 +0530 Subject: [PATCH 33/56] migrated to koanf --- config.toml | 10 +++ defs.go | 4 -- go.mod | 3 +- go.sum | 29 ++++++-- http_handlers.go | 2 +- init.go | 54 +++++++++++++++ main.go | 171 ++++++++++++++++++++--------------------------- server.go | 16 ++--- 8 files changed, 170 insertions(+), 119 deletions(-) create mode 100644 config.toml delete mode 100644 defs.go create mode 100644 init.go diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..025fc39 --- /dev/null +++ b/config.toml @@ -0,0 +1,10 @@ +[app] + address = ":8080" + enable-internal-api = false + enable-ssl = false + cert-path = "" + key-file-path = "" + max-handles = 10 + upstream-url = "https://api.varnamproject.com" + download-enabled-schemes = "" + sync-interval = "30s" \ No newline at end of file diff --git a/defs.go b/defs.go deleted file mode 100644 index f3cbb5d..0000000 --- a/defs.go +++ /dev/null @@ -1,4 +0,0 @@ -package main - -const varnamdVersion = "0.0.8" -const downloadPageSize = 100 diff --git a/go.mod b/go.mod index c7257c7..6ab74b4 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,14 @@ module github.com/varnamproject/varnamd go 1.14 require ( - github.com/athul/go-libvarnam v0.0.0-20200912100303-48fdad5ac70e github.com/coocood/freecache v1.1.1 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/protobuf v1.4.2 // indirect + github.com/knadh/koanf v0.12.1 github.com/labstack/echo/v4 v4.1.16 github.com/mattn/go-colorable v0.1.7 // indirect github.com/mattn/go-sqlite3 v1.14.0 + github.com/spf13/pflag v1.0.5 github.com/valyala/fasttemplate v1.2.0 // indirect golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect diff --git a/go.sum b/go.sum index 18b3b15..646906c 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,22 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= -github.com/athul/go-libvarnam v0.0.0-20200912100303-48fdad5ac70e h1:zKSW8/mOy4aYFghtsAFXNrq84IbjmFvqX3lAy0hqb7M= -github.com/athul/go-libvarnam v0.0.0-20200912100303-48fdad5ac70e/go.mod h1:QRvlfDTVGxOgZYm8toOEXFVAmSQXCvjq3zWBZBfHYig= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc= github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -25,6 +30,10 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/knadh/koanf v0.12.1 h1:9N0asqPUvZ0jL9thTbMZ0FAT3VrGfR+aYm5JTInT+Gs= +github.com/knadh/koanf v0.12.1/go.mod h1:31bzRSM7vS5Vm9LNLo7B2Re1zhLOZT6EQKeodixBikE= github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz0o= github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= @@ -41,15 +50,21 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= +github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rhnvrm/simples3 v0.5.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= @@ -74,10 +89,12 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= @@ -99,5 +116,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/http_handlers.go b/http_handlers.go index 987e3bf..d903072 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -74,7 +74,7 @@ func handleStatus(c echo.Context) error { Uptime string `json:"uptime"` standardResponse }{ - varnamdVersion, + buildVersion + "-" + buildDate, uptime.String(), newStandardResponse(), } diff --git a/init.go b/init.go new file mode 100644 index 0000000..4c76531 --- /dev/null +++ b/init.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "os" + "path" + "runtime" + "strings" + "time" +) + +func initConfig(cfg appConfig) *config { + toDownload := make(map[string]bool) + schemes := strings.Split(cfg.DownloadEnabledSchemes, ",") + + for _, scheme := range schemes { + s := strings.TrimSpace(scheme) + + if s != "" { + if !isValidSchemeIdentifier(s) { + panic(fmt.Sprintf("%s is not a valid libvarnam supported scheme", s)) + } + + toDownload[s] = true + } + } + + return &config{upstream: cfg.UpstreamURL, schemesToDownload: toDownload, + syncInterval: time.Duration(cfg.SyncInterval)} +} + +func (c *config) setDownloadStatus(langCode string, status bool) error { + if !isValidSchemeIdentifier(langCode) { + return fmt.Errorf("%s is not a valid libvarnam supported scheme", langCode) + } + + c.schemesToDownload[langCode] = status + + if status { + // when varnamd was started without any langcodes to sync, the dispatcher won't be running + // in that case, we need to start the dispatcher since we have a new lang code to download now + startSyncDispatcher() + } + + return nil +} + +func getConfigDir() string { + if runtime.GOOS == "windows" { + return path.Join(os.Getenv("localappdata"), ".varnamd") + } + + return path.Join(os.Getenv("HOME"), ".varnamd") +} diff --git a/main.go b/main.go index d47c2d3..3547740 100644 --- a/main.go +++ b/main.go @@ -1,33 +1,48 @@ package main import ( - "flag" "fmt" "log" "os" - "path" "runtime" - "strings" "time" + + flag "github.com/spf13/pflag" + + "github.com/knadh/koanf" + "github.com/knadh/koanf/parsers/toml" + "github.com/knadh/koanf/providers/file" + "github.com/knadh/koanf/providers/posflag" ) +const ( + downloadPageSize = 100 +) + +type appConfig struct { + Address string `koanf:"address"` + + EnableInternalApis bool `koanf:"enable-internal-api"` // internal APIs are not exposed to public + EnableSSL bool `koanf:"enable-ssl"` + CertFilePath string `koanf:"cert-path"` + KeyFilePath string `koanf:"key-file-path"` + + DownloadEnabledSchemes string `koanf:"download-enabled-schemes"` + SyncInterval time.Duration `koanf:"sync-interval"` + UpstreamURL string `koanf:"upstream-url"` +} + var ( - port int - version bool - maxHandleCount int - host string - uiDir string - enableInternalApis bool // internal APIs are not exposed to public - enableSSL bool - certFilePath string - keyFilePath string - logToFile bool // logs will be written to file when true - varnamdConfig *config // config instance used across the application - startedAt time.Time - downloadEnabledSchemes string // comma separated list of scheme identifier for which download will be performed - syncIntervalInSecs int - upstreamURL string - syncDispatcherRunning bool + kf = koanf.New(".") + + uiDir = "ui/" + + varnamdConfig *config // config instance used across the application + syncDispatcherRunning bool + startedAt time.Time + buildVersion string + buildDate string + maxHandleCount int ) // App is a singleton to share across handlers. @@ -44,87 +59,43 @@ type config struct { syncInterval time.Duration } -func initConfig() *config { - toDownload := make(map[string]bool) - schemes := strings.Split(downloadEnabledSchemes, ",") - - for _, scheme := range schemes { - s := strings.TrimSpace(scheme) - - if s != "" { - if !isValidSchemeIdentifier(s) { - panic(fmt.Sprintf("%s is not a valid libvarnam supported scheme", s)) - } - - toDownload[s] = true - } +func init() { + // Setup flags to read from user to start the application. + // Initialize 'config' flagset. + flagSet := flag.NewFlagSet("config", flag.ContinueOnError) + flagSet.Usage = func() { + log.Fatal(flagSet.FlagUsages()) } - return &config{upstream: upstreamURL, schemesToDownload: toDownload, - syncInterval: time.Duration(syncIntervalInSecs)} -} - -func (c *config) setDownloadStatus(langCode string, status bool) error { - if !isValidSchemeIdentifier(langCode) { - return fmt.Errorf("%s is not a valid libvarnam supported scheme", langCode) - } + // Create config flag to read 'config.toml' from user. + flagSet.String("config", "config.toml", "Path to the TOML configuration file") - c.schemesToDownload[langCode] = status + // Create flag for version check. + flagSet.Bool("version", false, "Current version of the build") - if status { - // when varnamd was started without any langcodes to sync, the dispatcher won't be running - // in that case, we need to start the dispatcher since we have a new lang code to download now - startSyncDispatcher() + err := flagSet.Parse(os.Args[1:]) + if err != nil { + log.Fatalf("error parsing flags: %v", err) } - return nil -} - -func getConfigDir() string { - if runtime.GOOS == "windows" { - return path.Join(os.Getenv("localappdata"), ".varnamd") + // Load commandline params user given. + if err = kf.Load(posflag.Provider(flagSet, ".", kf), nil); err != nil { + log.Fatal(err.Error()) } - return path.Join(os.Getenv("HOME"), ".varnamd") -} - -func getLogsDir() string { - d := getConfigDir() - logsDir := path.Join(d, "logs") - - if err := os.MkdirAll(logsDir, 0750); err != nil { - panic(err) + // Handle --version flag. Print build version, build date and die. + if kf.Bool("version") { + fmt.Printf("Commit: %v\nBuild: %v\n", buildVersion, buildDate) + os.Exit(0) } - return logsDir -} - -func redirectLogToFile() { - year, month, day := time.Now().Date() - logfile := path.Join(getLogsDir(), fmt.Sprintf("%d-%d-%d.log", year, month, day)) + // Load the config file. + log.Printf("reading config: %s", kf.String("config")) - f, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - panic(err) + if err = kf.Load(file.Provider(kf.String("config")), toml.Parser()); err != nil { + log.Fatalf("error reading config: %v", err) } - log.SetOutput(f) -} - -func init() { - flag.IntVar(&port, "p", 8080, "Run daemon in specified port") - flag.IntVar(&maxHandleCount, "max-handle-count", 10, "Maximum number of handles can be opened for each language") - flag.StringVar(&host, "host", "localhost", "Host for the varnam daemon server") - flag.StringVar(&uiDir, "ui", "ui", "UI directory path") - flag.BoolVar(&enableInternalApis, "enable-internal-apis", false, "Enable internal APIs") - flag.BoolVar(&enableSSL, "enable-ssl", false, "Enables SSL") - flag.StringVar(&certFilePath, "cert-file-path", "", "Certificate file path") - flag.StringVar(&keyFilePath, "key-file-path", "", "Key file path") - flag.StringVar(&upstreamURL, "upstream", "https://api.varnamproject.com", "Provide an upstream server") - flag.StringVar(&downloadEnabledSchemes, "enable-download", "", "Comma separated language identifier for which varnamd will download words from upstream") - flag.IntVar(&syncIntervalInSecs, "sync-interval", 30, "Download interval in seconds") - flag.BoolVar(&logToFile, "log-to-file", true, "If true, logs will be written to a file") - flag.BoolVar(&version, "version", false, "Print the version and exit") } func syncRequired() bool { @@ -143,22 +114,26 @@ func startSyncDispatcher() { } func main() { - runtime.GOMAXPROCS(runtime.NumCPU()) - flag.Parse() + var ( + config appConfig + ) - varnamdConfig = initConfig() - startedAt = time.Now() + runtime.GOMAXPROCS(runtime.NumCPU()) - if version { - fmt.Println(varnamdVersion) - os.Exit(0) + // Read configuration using Koanf. + if err := kf.Unmarshal("app", &config); err != nil { + log.Fatal(err.Error()) } - if logToFile { - redirectLogToFile() + maxHandleCount = kf.Int("app.max-handles") + if maxHandleCount <= 0 { + maxHandleCount = 10 } - log.Printf("varnamd %s", varnamdVersion) + varnamdConfig = initConfig(config) + startedAt = time.Now() + + log.Printf("varnamd %s-%s", buildVersion, buildDate) app := &App{ cache: NewMemCache(), @@ -166,5 +141,5 @@ func main() { } startSyncDispatcher() - startDaemon(app) + startDaemon(app, config) } diff --git a/server.go b/server.go index 5622fac..c162d8e 100644 --- a/server.go +++ b/server.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "net/http" "os" "path/filepath" @@ -10,30 +9,29 @@ import ( "github.com/labstack/echo/v4/middleware" ) -func startDaemon(app *App) { +func startDaemon(app *App, cfg appConfig) { initLanguageChannels() initLearnChannels() - e, err := initHandlers(app) + e, err := initHandlers(app, cfg.EnableInternalApis) if err != nil { app.log.Fatalf("error initializing handlers: %s", err) } - address := fmt.Sprintf("%s:%d", host, port) - app.log.Printf("Listening on %s", address) + app.log.Printf("Listening on %s", cfg.Address) - if enableSSL { - if err := e.StartTLS(address, certFilePath, keyFilePath); err != nil { + if cfg.EnableSSL { + if err := e.StartTLS(cfg.Address, cfg.CertFilePath, cfg.KeyFilePath); err != nil { app.log.Fatal(err) } } else { - if err := e.Start(address); err != nil { + if err := e.Start(cfg.Address); err != nil { app.log.Fatal(err) } } } -func initHandlers(app *App) (*echo.Echo, error) { +func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { e := echo.New() e.GET("/tl/:langCode/:word", handleTransliteration) From 65f4ea509499b1ce6c264b3f66445e417c8c5463 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 15:57:38 +0530 Subject: [PATCH 34/56] added stuffsbin --- go.mod | 1 + go.sum | 2 ++ init.go | 29 +++++++++++++++++++++++++++++ main.go | 9 +++++++-- server.go | 11 +++-------- 5 files changed, 42 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 6ab74b4..4938e4b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/protobuf v1.4.2 // indirect github.com/knadh/koanf v0.12.1 + github.com/knadh/stuffbin v1.1.0 github.com/labstack/echo/v4 v4.1.16 github.com/mattn/go-colorable v0.1.7 // indirect github.com/mattn/go-sqlite3 v1.14.0 diff --git a/go.sum b/go.sum index 646906c..da866de 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/knadh/koanf v0.12.1 h1:9N0asqPUvZ0jL9thTbMZ0FAT3VrGfR+aYm5JTInT+Gs= github.com/knadh/koanf v0.12.1/go.mod h1:31bzRSM7vS5Vm9LNLo7B2Re1zhLOZT6EQKeodixBikE= +github.com/knadh/stuffbin v1.1.0 h1:f5S5BHzZALjuJEgTIOMC9NidEnBJM7Ze6Lu1GHR/lwU= +github.com/knadh/stuffbin v1.1.0/go.mod h1:yVCFaWaKPubSNibBsTAJ939q2ABHudJQxRWZWV5yh+4= github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz0o= github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= diff --git a/init.go b/init.go index 4c76531..9b56d6c 100644 --- a/init.go +++ b/init.go @@ -2,11 +2,14 @@ package main import ( "fmt" + "log" "os" "path" "runtime" "strings" "time" + + "github.com/knadh/stuffbin" ) func initConfig(cfg appConfig) *config { @@ -52,3 +55,29 @@ func getConfigDir() string { return path.Join(os.Getenv("HOME"), ".varnamd") } + +// initVFS initializes the stuffbin virtual FileSystem to provide +// access to bunded static assets to the app. +func initVFS() (stuffbin.FileSystem, error) { + files := []string{ + "ui/", + } + + binPath, err := os.Executable() + if err != nil { + return nil, err + } + + fs, err := stuffbin.UnStuff(binPath) + if err != nil { + log.Printf("unable to initialize embedded filesystem: %v", err) + log.Printf("using local filesystem") + + fs, err = stuffbin.NewLocalFS("/", files...) + if err != nil { + return nil, err + } + } + + return fs, nil +} diff --git a/main.go b/main.go index 3547740..935aca7 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( "github.com/knadh/koanf/parsers/toml" "github.com/knadh/koanf/providers/file" "github.com/knadh/koanf/providers/posflag" + "github.com/knadh/stuffbin" ) const ( @@ -35,8 +36,6 @@ type appConfig struct { var ( kf = koanf.New(".") - uiDir = "ui/" - varnamdConfig *config // config instance used across the application syncDispatcherRunning bool startedAt time.Time @@ -49,6 +48,7 @@ var ( type App struct { cache Cache log *log.Logger + fs stuffbin.FileSystem } // varnamd configurations @@ -135,9 +135,14 @@ func main() { log.Printf("varnamd %s-%s", buildVersion, buildDate) + fs, err := initVFS() + if err != nil { + log.Fatal(err.Error()) + } app := &App{ cache: NewMemCache(), log: log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), + fs: fs, } startSyncDispatcher() diff --git a/server.go b/server.go index c162d8e..e4eb9aa 100644 --- a/server.go +++ b/server.go @@ -2,8 +2,6 @@ package main import ( "net/http" - "os" - "path/filepath" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" @@ -32,8 +30,9 @@ func startDaemon(app *App, cfg appConfig) { } func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { - e := echo.New() + fSrv := app.fs.FileServer() + e := echo.New() e.GET("/tl/:langCode/:word", handleTransliteration) e.GET("/rtl/:langCode/:word", handleReverseTransliteration) e.GET("/meta/:langCode:", handleMetadata) @@ -43,11 +42,7 @@ func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { e.GET("/status", handleStatus) e.POST("/train", handleTrain) - if _, err := os.Stat(filepath.Clean(uiDir)); err != nil { - return nil, err - } - - e.Static("/", filepath.Clean(uiDir)) + e.GET("/", echo.WrapHandler(fSrv)) if enableInternalApis { e.POST("/sync/download/{langCode}/enable", handleEnableDownload) From 458072022e18c0a755b8be838c920ee97bef979f Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 16:05:59 +0530 Subject: [PATCH 35/56] added makefile --- Makefile | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d207778 --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +BIN := varnam.bin +HASH := $(shell git rev-parse HEAD | cut -c 1-8) +COMMIT_DATE := $(shell git show -s --format=%ci ${HASH}) +BUILD_DATE := $(shell date '+%Y-%m-%d %H:%M:%S') +VERSION := ${HASH} (${COMMIT_DATE}) +STATIC := ui + +deps: + go get -u github.com/knadh/stuffbin/... + +.PHONY: build +build: ## Build the binary (default) + go build -o ${BIN} -ldflags="-X 'main.buildVersion=${VERSION}' -X 'main.buildDate=${BUILD_DATE}'" + stuffbin -a stuff -in ${BIN} -out ${BIN} ${STATIC} + +.PHONY: run +run: build + ./${BIN} + +.PHONY: clean +clean: ## Remove temporary files and the binary + go clean + +# Absolutely awesome: http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html +.PHONY: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.DEFAULT_GOAL := build \ No newline at end of file From f793e4463b52e11e5c28fe600984be09c8f02de2 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 16:07:50 +0530 Subject: [PATCH 36/56] size reduced --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d207778..457d6ab 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ deps: .PHONY: build build: ## Build the binary (default) - go build -o ${BIN} -ldflags="-X 'main.buildVersion=${VERSION}' -X 'main.buildDate=${BUILD_DATE}'" + go build -o ${BIN} -ldflags="-X 'main.buildVersion=${VERSION}' -X 'main.buildDate=${BUILD_DATE}' -s -w" stuffbin -a stuff -in ${BIN} -out ${BIN} ${STATIC} .PHONY: run From 40fa65e6c268bf0e2020959f03f7c3fbceb6d86d Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 17:30:20 +0530 Subject: [PATCH 37/56] added root handler --- Makefile | 2 +- http_handlers.go | 14 ++++++++++++++ init.go | 2 +- server.go | 6 +++--- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 457d6ab..bbd179b 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ HASH := $(shell git rev-parse HEAD | cut -c 1-8) COMMIT_DATE := $(shell git show -s --format=%ci ${HASH}) BUILD_DATE := $(shell date '+%Y-%m-%d %H:%M:%S') VERSION := ${HASH} (${COMMIT_DATE}) -STATIC := ui +STATIC := ui:/ deps: go get -u github.com/knadh/stuffbin/... diff --git a/http_handlers.go b/http_handlers.go index d903072..158a081 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -331,3 +331,17 @@ func handleDisableDownload(c echo.Context) error { return c.JSON(http.StatusOK, data) } + +// handleIndex is the root handler that renders the Javascript frontend. +func handleIndex(c echo.Context) error { + app, _ := c.Get("app").(*App) + + b, err := app.fs.Read("/index.html") + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + c.Response().Header().Set("Content-Type", "text/html") + + return c.String(http.StatusOK, string(b)) +} diff --git a/init.go b/init.go index 9b56d6c..710b777 100644 --- a/init.go +++ b/init.go @@ -60,7 +60,7 @@ func getConfigDir() string { // access to bunded static assets to the app. func initVFS() (stuffbin.FileSystem, error) { files := []string{ - "ui/", + "ui:/", } binPath, err := os.Executable() diff --git a/server.go b/server.go index e4eb9aa..0c0b82e 100644 --- a/server.go +++ b/server.go @@ -30,8 +30,6 @@ func startDaemon(app *App, cfg appConfig) { } func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { - fSrv := app.fs.FileServer() - e := echo.New() e.GET("/tl/:langCode/:word", handleTransliteration) e.GET("/rtl/:langCode/:word", handleReverseTransliteration) @@ -42,7 +40,9 @@ func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { e.GET("/status", handleStatus) e.POST("/train", handleTrain) - e.GET("/", echo.WrapHandler(fSrv)) + e.GET("/", handleIndex) + + e.GET("/*", echo.WrapHandler(app.fs.FileServer())) if enableInternalApis { e.POST("/sync/download/{langCode}/enable", handleEnableDownload) From d9e7cd271cf1e376324d63eb879f1f189eea2575 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sun, 13 Sep 2020 17:48:47 +0530 Subject: [PATCH 38/56] added defaults --- main.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 935aca7..5655173 100644 --- a/main.go +++ b/main.go @@ -93,7 +93,7 @@ func init() { log.Printf("reading config: %s", kf.String("config")) if err = kf.Load(file.Provider(kf.String("config")), toml.Parser()); err != nil { - log.Fatalf("error reading config: %v", err) + log.Printf("error reading config: %v", err) } } @@ -125,6 +125,23 @@ func main() { log.Fatal(err.Error()) } + // If address is empty run on localhost port 8080. + if config.Address == "" { + config.Address = ":8080" + } + + if config.EnableSSL && (config.CertFilePath == "" || config.KeyFilePath == "") { + config.EnableSSL = false + } + + if config.SyncInterval < 1*time.Second { + config.SyncInterval = 30 * time.Second + } + + if config.UpstreamURL == "" { + config.UpstreamURL = "https://api.varnamproject.com" + } + maxHandleCount = kf.Int("app.max-handles") if maxHandleCount <= 0 { maxHandleCount = 10 From 43ed19aedc8f8485399bbbd2a6e77f545d933168 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Tue, 15 Sep 2020 01:44:58 +0530 Subject: [PATCH 39/56] added logger for errors in train endpoint --- http_handlers.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/http_handlers.go b/http_handlers.go index 158a081..4201485 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -281,13 +281,16 @@ func handleTrain(c echo.Context) error { c.Request().Header.Set("Content-Type", "application/json") if err := c.Bind(&targs); err != nil { + app.log.Printf("error reading request, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } handle, err := libvarnam.Init(targs.Lang) if err != nil { + app.log.Printf("libvarnam init failed, err: %s", err.Error()) return fmt.Errorf("failed to get data") } if err := handle.Train(targs.Pattern, targs.Word); err != nil { + app.log.Printf("error training word: %s, pattern: %s, err:%s", targs.Word, targs.Pattern, err.Error()) return fmt.Errorf("failed to Train %s. %s", targs.Word, err.Error()) } return c.JSON(200, "Word Trained") From d7d6086d5725c62eb9a601fc2e6ba98d9aa01e8f Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Wed, 16 Sep 2020 22:35:18 +0530 Subject: [PATCH 40/56] fixed undefined variable error --- http_handlers.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/http_handlers.go b/http_handlers.go index 4201485..5e684af 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -250,7 +250,7 @@ func handleLanguages(c echo.Context) error { return c.JSON(http.StatusOK, schemeDetails) } -func handlLearn(c echo.Context) error { +func handleLearn(c echo.Context) error { var ( a args @@ -276,7 +276,10 @@ func handlLearn(c echo.Context) error { } func handleTrain(c echo.Context) error { - var targs TrainArgs + var ( + targs TrainArgs + app = c.Get("app").(*App) + ) c.Request().Header.Set("Content-Type", "application/json") From c3439257e94f2371b4ce058f07e5a5524ff64b20 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Thu, 17 Sep 2020 10:58:38 +0530 Subject: [PATCH 41/56] corrected endpoints and rest api structure for train endpoint --- http_handlers.go | 13 +++++++------ server.go | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/http_handlers.go b/http_handlers.go index 5e684af..671bcfd 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -61,9 +61,8 @@ type args struct { //TrainArgs read the incoming data type TrainArgs struct { - Lang string `json:"lang" form:"lang"` - Pattern string `json:"pattern" form:"pattern"` - Word string `json:"word" form:"word"` + Pattern string `json:"pattern"` + Word string `json:"word"` } func handleStatus(c echo.Context) error { @@ -277,8 +276,9 @@ func handleLearn(c echo.Context) error { func handleTrain(c echo.Context) error { var ( - targs TrainArgs - app = c.Get("app").(*App) + targs TrainArgs + app = c.Get("app").(*App) + langCode = c.Param("langCode") ) c.Request().Header.Set("Content-Type", "application/json") @@ -287,7 +287,8 @@ func handleTrain(c echo.Context) error { app.log.Printf("error reading request, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } - handle, err := libvarnam.Init(targs.Lang) + + handle, err := libvarnam.Init(langCode) if err != nil { app.log.Printf("libvarnam init failed, err: %s", err.Error()) return fmt.Errorf("failed to get data") diff --git a/server.go b/server.go index 0c0b82e..57cd4c8 100644 --- a/server.go +++ b/server.go @@ -38,15 +38,15 @@ func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { e.POST("/learn", handleLearn) e.GET("/languages", handleLanguages) e.GET("/status", handleStatus) - e.POST("/train", handleTrain) + e.POST("/train/:langCode", handleTrain) e.GET("/", handleIndex) e.GET("/*", echo.WrapHandler(app.fs.FileServer())) if enableInternalApis { - e.POST("/sync/download/{langCode}/enable", handleEnableDownload) - e.POST("/sync/download/{langCode}/disable", handleDisableDownload) + e.POST("/sync/download/:langCode/enable", handleEnableDownload) + e.POST("/sync/download/:langCode/disable", handleDisableDownload) } e.Use(middleware.Recover()) From 8a3a3311afa417757d6ea4937460483d13661baf Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Thu, 17 Sep 2020 22:23:14 +0530 Subject: [PATCH 42/56] added handler to train in bulk, changed structure of training individual word, pattern mapping to channel --- http_handlers.go | 68 +++++++++++++++++++++++++++++++++++++++------- server.go | 3 +- varnam_handlers.go | 1 - varnam_learn.go | 32 ++++++++++++++++------ 4 files changed, 83 insertions(+), 21 deletions(-) diff --git a/http_handlers.go b/http_handlers.go index 671bcfd..c998276 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -60,11 +60,17 @@ type args struct { } //TrainArgs read the incoming data -type TrainArgs struct { +type trainArgs struct { Pattern string `json:"pattern"` Word string `json:"word"` } +//TrainBulkArgs read the incoming data for bulk training. +type trainBulkArgs struct { + Pattern []string `json:"pattern"` + Word string `json:"word"` +} + func handleStatus(c echo.Context) error { uptime := time.Since(startedAt) @@ -276,7 +282,7 @@ func handleLearn(c echo.Context) error { func handleTrain(c echo.Context) error { var ( - targs TrainArgs + targs trainArgs app = c.Get("app").(*App) langCode = c.Param("langCode") ) @@ -288,18 +294,60 @@ func handleTrain(c echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } - handle, err := libvarnam.Init(langCode) - if err != nil { - app.log.Printf("libvarnam init failed, err: %s", err.Error()) - return fmt.Errorf("failed to get data") - } - if err := handle.Train(targs.Pattern, targs.Word); err != nil { - app.log.Printf("error training word: %s, pattern: %s, err:%s", targs.Word, targs.Pattern, err.Error()) - return fmt.Errorf("failed to Train %s. %s", targs.Word, err.Error()) + ch, ok := trainChannel[langCode] + if !ok { + app.log.Printf("unknown language requested to learn: %s", langCode) + return echo.NewHTTPError(http.StatusBadRequest, "unable to find language to train") } + + go func(args trainArgs) { ch <- args }(targs) + return c.JSON(200, "Word Trained") } + +// handleTrainBulk is an endpoint for training words in the following format. +// {[ +// {word, patterns: []}, +// {word, patterns: []}, +// {word, patterns: []}, +// {word, patterns: []} +// ]} +// It will covert each bulk arg to trainArg and will send to train channel. +// Training is happened at listenForWords method. +func handleTrainBulk(c echo.Context) error { + var ( + bulkArgs []trainBulkArgs + app = c.Get("app").(*App) + langCode = c.Param("langCode") + ) + + if err := c.Bind(&bulkArgs); err != nil { + app.log.Printf("error reading request, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } + + ch, ok := trainChannel[langCode] + if !ok { + app.log.Printf("unknown language requested to learn: %s", langCode) + return echo.NewHTTPError(http.StatusBadRequest, "unable to find language to train") + } + + for _, v := range bulkArgs { + for _, p := range v.Pattern { + go func(args trainArgs) { + ch <- args + }(trainArgs{ + Pattern: p, + Word: v.Word, + }) + } + } + + return c.JSON(200, "Words Trained") + +} + func toggleDownloadEnabledStatus(langCode string, status bool) (interface{}, error) { if err := varnamdConfig.setDownloadStatus(langCode, status); err != nil { return nil, err diff --git a/server.go b/server.go index 57cd4c8..12a3548 100644 --- a/server.go +++ b/server.go @@ -9,7 +9,7 @@ import ( func startDaemon(app *App, cfg appConfig) { initLanguageChannels() - initLearnChannels() + app.initChannels() e, err := initHandlers(app, cfg.EnableInternalApis) if err != nil { @@ -39,6 +39,7 @@ func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { e.GET("/languages", handleLanguages) e.GET("/status", handleStatus) e.POST("/train/:langCode", handleTrain) + e.POST("/train/bulk/:langCode", handleTrainBulk) e.GET("/", handleIndex) diff --git a/varnam_handlers.go b/varnam_handlers.go index a0723cd..ea244c2 100644 --- a/varnam_handlers.go +++ b/varnam_handlers.go @@ -60,7 +60,6 @@ func initLanguageChannels() { func getOrCreateHandler(schemeIdentifier string, f func(handle *libvarnam.Varnam) (data interface{}, err error)) (data interface{}, err error) { ch, ok := languageChannels[schemeIdentifier] - if !ok { return nil, errors.New("invalid scheme identifier") } diff --git a/varnam_learn.go b/varnam_learn.go index fdb8f29..5ae4ff8 100644 --- a/varnam_learn.go +++ b/varnam_learn.go @@ -1,33 +1,47 @@ package main import ( - "log" + "strings" "github.com/varnamproject/varnamd/libvarnam" ) +const defaultChanSize = 1000 + var ( learnChannels map[string]chan string + trainChannel map[string]chan trainArgs ) -func initLearnChannels() { +// initChannels method will initialize learn and train channels. +func (app *App) initChannels() { learnChannels = make(map[string]chan string) + trainChannel = make(map[string]chan trainArgs) + for _, scheme := range schemeDetails { - learnChannels[scheme.Identifier] = make(chan string, 100) + learnChannels[scheme.Identifier] = make(chan string, defaultChanSize) + trainChannel[scheme.Identifier] = make(chan trainArgs, defaultChanSize) handle, err := libvarnam.Init(scheme.Identifier) if err != nil { - log.Fatal("Unable to initialize varnam for lang", scheme.LangCode) + app.log.Fatal("Unable to initialize varnam for lang", scheme.LangCode) } - go listenForWords(scheme.Identifier, handle) + go app.listenForWords(scheme.Identifier, handle) } } -func listenForWords(lang string, handle *libvarnam.Varnam) { - for word := range learnChannels[lang] { - if err := handle.Learn(word); err != nil { - log.Printf("Failed to learn %s. %s\n", word, err.Error()) +func (app *App) listenForWords(lang string, handle *libvarnam.Varnam) { + for { + select { + case word := <-learnChannels[lang]: + if err := handle.Learn(strings.TrimSpace(word)); err != nil { + app.log.Printf("Failed to learn %s. %s\n", word, err.Error()) + } + case args := <-trainChannel[lang]: + if err := handle.Train(strings.TrimSpace(args.Pattern), strings.TrimSpace(args.Word)); err != nil { + app.log.Printf("error training word: %s, pattern: %s, err:%s", args.Word, args.Pattern, err.Error()) + } } } } From 3f7f971628343c8058905407de584294cc52e4a2 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Thu, 17 Sep 2020 23:08:27 +0530 Subject: [PATCH 43/56] added basic auth for training handles --- config.toml | 6 ++++- http_handlers.go | 2 -- init.go | 27 +++++++++++++++++++ main.go | 67 ++++++++++++++++++++++-------------------------- server.go | 58 ++++++++++++++++++++++++++++++++++------- 5 files changed, 111 insertions(+), 49 deletions(-) diff --git a/config.toml b/config.toml index 025fc39..11ce9b2 100644 --- a/config.toml +++ b/config.toml @@ -7,4 +7,8 @@ max-handles = 10 upstream-url = "https://api.varnamproject.com" download-enabled-schemes = "" - sync-interval = "30s" \ No newline at end of file + sync-interval = "30s" + accounts-enabled = true +[users] + [users.admin] + password = "pass" diff --git a/http_handlers.go b/http_handlers.go index c998276..da21c0d 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -303,7 +303,6 @@ func handleTrain(c echo.Context) error { go func(args trainArgs) { ch <- args }(targs) return c.JSON(200, "Word Trained") - } // handleTrainBulk is an endpoint for training words in the following format. @@ -345,7 +344,6 @@ func handleTrainBulk(c echo.Context) error { } return c.JSON(200, "Words Trained") - } func toggleDownloadEnabledStatus(langCode string, status bool) (interface{}, error) { diff --git a/init.go b/init.go index 710b777..6cb0abd 100644 --- a/init.go +++ b/init.go @@ -81,3 +81,30 @@ func initVFS() (stuffbin.FileSystem, error) { return fs, nil } + +func initAppConfig() (appConfig, error) { + var config appConfig + // Read configuration using Koanf. + if err := kf.Unmarshal("app", &config); err != nil { + return config, err + } + + // If address is empty run on localhost port 8080. + if config.Address == "" { + config.Address = ":8080" + } + + if config.EnableSSL && (config.CertFilePath == "" || config.KeyFilePath == "") { + config.EnableSSL = false + } + + if config.SyncInterval < 1*time.Second { + config.SyncInterval = 30 * time.Second + } + + if config.UpstreamURL == "" { + config.UpstreamURL = "https://api.varnamproject.com" + } + + return config, nil +} diff --git a/main.go b/main.go index 5655173..411a498 100644 --- a/main.go +++ b/main.go @@ -16,10 +16,27 @@ import ( "github.com/knadh/stuffbin" ) +type userConfig map[string]string + const ( downloadPageSize = 100 ) +var ( + kf = koanf.New(".") + + varnamdConfig *config // config instance used across the application + syncDispatcherRunning bool + startedAt time.Time + buildVersion string + buildDate string + maxHandleCount int + authEnabled bool + + // User accounts are stored here. + users map[string]userConfig +) + type appConfig struct { Address string `koanf:"address"` @@ -33,17 +50,6 @@ type appConfig struct { UpstreamURL string `koanf:"upstream-url"` } -var ( - kf = koanf.New(".") - - varnamdConfig *config // config instance used across the application - syncDispatcherRunning bool - startedAt time.Time - buildVersion string - buildDate string - maxHandleCount int -) - // App is a singleton to share across handlers. type App struct { cache Cache @@ -60,6 +66,9 @@ type config struct { } func init() { + // Set max processors to number of CPUs to maximize performance. + runtime.GOMAXPROCS(runtime.NumCPU()) + // Setup flags to read from user to start the application. // Initialize 'config' flagset. flagSet := flag.NewFlagSet("config", flag.ContinueOnError) @@ -95,7 +104,6 @@ func init() { if err = kf.Load(file.Provider(kf.String("config")), toml.Parser()); err != nil { log.Printf("error reading config: %v", err) } - } func syncRequired() bool { @@ -114,39 +122,23 @@ func startSyncDispatcher() { } func main() { - var ( - config appConfig - ) - - runtime.GOMAXPROCS(runtime.NumCPU()) - - // Read configuration using Koanf. - if err := kf.Unmarshal("app", &config); err != nil { + config, err := initAppConfig() + if err != nil { log.Fatal(err.Error()) } - // If address is empty run on localhost port 8080. - if config.Address == "" { - config.Address = ":8080" - } - - if config.EnableSSL && (config.CertFilePath == "" || config.KeyFilePath == "") { - config.EnableSSL = false - } - - if config.SyncInterval < 1*time.Second { - config.SyncInterval = 30 * time.Second - } - - if config.UpstreamURL == "" { - config.UpstreamURL = "https://api.varnamproject.com" - } - maxHandleCount = kf.Int("app.max-handles") if maxHandleCount <= 0 { maxHandleCount = 10 } + authEnabled = kf.Bool("app.accounts-enabled") + if authEnabled { + if err = kf.Unmarshal("users", &users); err != nil { + log.Fatal(err.Error()) + } + } + varnamdConfig = initConfig(config) startedAt = time.Now() @@ -156,6 +148,7 @@ func main() { if err != nil { log.Fatal(err.Error()) } + app := &App{ cache: NewMemCache(), log: log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), diff --git a/server.go b/server.go index 12a3548..30bb62a 100644 --- a/server.go +++ b/server.go @@ -1,7 +1,9 @@ package main import ( + "encoding/base64" "net/http" + "strings" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" @@ -11,12 +13,9 @@ func startDaemon(app *App, cfg appConfig) { initLanguageChannels() app.initChannels() - e, err := initHandlers(app, cfg.EnableInternalApis) - if err != nil { - app.log.Fatalf("error initializing handlers: %s", err) - } + e := initHandlers(app, cfg.EnableInternalApis) - app.log.Printf("Listening on %s", cfg.Address) + app.log.Printf("🚀 starting varnamd\nListening on %s", cfg.Address) if cfg.EnableSSL { if err := e.StartTLS(cfg.Address, cfg.CertFilePath, cfg.KeyFilePath); err != nil { @@ -29,7 +28,7 @@ func startDaemon(app *App, cfg appConfig) { } } -func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { +func initHandlers(app *App, enableInternalApis bool) *echo.Echo { e := echo.New() e.GET("/tl/:langCode/:word", handleTransliteration) e.GET("/rtl/:langCode/:word", handleReverseTransliteration) @@ -38,8 +37,8 @@ func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { e.POST("/learn", handleLearn) e.GET("/languages", handleLanguages) e.GET("/status", handleStatus) - e.POST("/train/:langCode", handleTrain) - e.POST("/train/bulk/:langCode", handleTrainBulk) + e.POST("/train/:langCode", authUser(handleTrain)) + e.POST("/train/bulk/:langCode", authUser(handleTrainBulk)) e.GET("/", handleIndex) @@ -74,5 +73,46 @@ func initHandlers(app *App, enableInternalApis bool) (*echo.Echo, error) { // ContentSecurityPolicy: "default-src 'self'", })) - return e, nil + return e +} + +// authUser as a separate method to apply this middleware only for selected endpoints. +func authUser(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + var app = c.Get("app").(*App) + + if authEnabled { + auth := strings.Split(c.Request().Header.Get("Authorization"), " ") + if len(auth) < 2 { + app.log.Printf("authorization header not found") + return echo.NewHTTPError(http.StatusUnauthorized, "authorization header not found") + } + + if strings.ToLower(auth[0]) != "basic" { + app.log.Printf("authorization header not found") + return echo.NewHTTPError(http.StatusUnauthorized, "authorization details not found") + } + + creds, err := base64.StdEncoding.DecodeString(auth[1]) + if err != nil { + app.log.Printf("error decoding auth headers, error: %s", err.Error()) + return echo.NewHTTPError(http.StatusUnauthorized, "authorization failed, failed to decode authstring") + } + + authCreds := strings.Split(string(creds), ":") + + user, ok := users[strings.TrimSpace(authCreds[0])] + if !ok { + app.log.Printf("user not found") + return echo.NewHTTPError(http.StatusUnauthorized, "authorization failed, user not found") + } + + if user["password"] != strings.TrimSpace(authCreds[1]) { + app.log.Printf("password mismatch") + return echo.NewHTTPError(http.StatusUnauthorized, "authorization failed, password mismatch") + } + } + + return next(c) + } } From 1f62ad1074e7a23c1eed7fa4387a5d351a8ec2c2 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Thu, 17 Sep 2020 23:21:21 +0530 Subject: [PATCH 44/56] rolled back address to flags - p and host respectively --- config.toml | 1 - init.go | 2 +- main.go | 5 ++++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/config.toml b/config.toml index 11ce9b2..861996c 100644 --- a/config.toml +++ b/config.toml @@ -1,5 +1,4 @@ [app] - address = ":8080" enable-internal-api = false enable-ssl = false cert-path = "" diff --git a/init.go b/init.go index 6cb0abd..0028f3c 100644 --- a/init.go +++ b/init.go @@ -91,7 +91,7 @@ func initAppConfig() (appConfig, error) { // If address is empty run on localhost port 8080. if config.Address == "" { - config.Address = ":8080" + config.Address = fmt.Sprintf("%s:%d", kf.String("host"), kf.Int("p")) } if config.EnableSSL && (config.CertFilePath == "" || config.KeyFilePath == "") { diff --git a/main.go b/main.go index 411a498..c1745c6 100644 --- a/main.go +++ b/main.go @@ -38,7 +38,7 @@ var ( ) type appConfig struct { - Address string `koanf:"address"` + Address string EnableInternalApis bool `koanf:"enable-internal-api"` // internal APIs are not exposed to public EnableSSL bool `koanf:"enable-ssl"` @@ -79,6 +79,9 @@ func init() { // Create config flag to read 'config.toml' from user. flagSet.String("config", "config.toml", "Path to the TOML configuration file") + flagSet.Int("p", 8080, "Run daemon in specified port") + flagSet.String("host", "", "Host for the varnam daemon server") + // Create flag for version check. flagSet.Bool("version", false, "Current version of the build") From ac3afee8555716b71e1b177e2109c92c7b6ef80a Mon Sep 17 00:00:00 2001 From: Subin Siby Date: Mon, 28 Sep 2020 00:22:21 +0530 Subject: [PATCH 45/56] Add endpoint for downloading vst --- http_handlers.go | 13 +++++++++++++ server.go | 1 + varnam_handlers.go | 6 ++++++ 3 files changed, 20 insertions(+) diff --git a/http_handlers.go b/http_handlers.go index da21c0d..ba52559 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -255,6 +255,19 @@ func handleLanguages(c echo.Context) error { return c.JSON(http.StatusOK, schemeDetails) } +func handleLanguageDownload(c echo.Context) error { + var ( + langCode = c.Param("langCode") + ) + + filepath, err = getSchemeFilePath(langCode) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error: %s", err.Error())) + } + + return c.File(filepath) +} + func handleLearn(c echo.Context) error { var ( a args diff --git a/server.go b/server.go index 30bb62a..0a154e7 100644 --- a/server.go +++ b/server.go @@ -36,6 +36,7 @@ func initHandlers(app *App, enableInternalApis bool) *echo.Echo { e.GET("/download/:langCode/:downloadStart", handleDownload) e.POST("/learn", handleLearn) e.GET("/languages", handleLanguages) + e.GET("/languages/:langCode/download", handleLanguageDownload) e.GET("/status", handleStatus) e.POST("/train/:langCode", authUser(handleTrain)) e.POST("/train/bulk/:langCode", authUser(handleTrainBulk)) diff --git a/varnam_handlers.go b/varnam_handlers.go index ea244c2..f052a6a 100644 --- a/varnam_handlers.go +++ b/varnam_handlers.go @@ -176,3 +176,9 @@ func sendHandlerToChannel(schemeIdentifier string, handle *libvarnam.Varnam, ch default: } } + +func getSchemeFilePath(schemeIdentifier string) { + return getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { + return handle.GetSchemeFilePath(schemeIdentifier), nil + }) +} From e65b78b36634252bd4ec4b2d34dbcaf768a30a93 Mon Sep 17 00:00:00 2001 From: Subin Siby Date: Mon, 28 Sep 2020 00:40:27 +0530 Subject: [PATCH 46/56] Fix https://github.com/varnamproject/varnamd/issues/24 : add endpoint to download scheme file (vst) --- http_handlers.go | 4 ++-- libvarnam/libvarnam.go | 5 +++++ varnam_handlers.go | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/http_handlers.go b/http_handlers.go index ba52559..7359a70 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -260,12 +260,12 @@ func handleLanguageDownload(c echo.Context) error { langCode = c.Param("langCode") ) - filepath, err = getSchemeFilePath(langCode) + filepath, err := getSchemeFilePath(langCode) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error: %s", err.Error())) } - return c.File(filepath) + return c.Attachment(filepath.(string), langCode + ".vst") } func handleLearn(c echo.Context) error { diff --git a/libvarnam/libvarnam.go b/libvarnam/libvarnam.go index 4c1bdc1..3760f25 100644 --- a/libvarnam/libvarnam.go +++ b/libvarnam/libvarnam.go @@ -46,6 +46,11 @@ func (v *Varnam) GetSuggestionsFilePath() string { return C.GoString(C.varnam_get_suggestions_file(v.handle)) } +// GetSchemeFilePath returns the scheme file (.vst) +func (v *Varnam) GetSchemeFilePath() string { + return C.GoString(C.varnam_get_scheme_file(v.handle)) +} + // GetCorpusDetails will return corpus details. func (v *Varnam) GetCorpusDetails() (*CorpusDetails, error) { var details *C.vcorpus_details diff --git a/varnam_handlers.go b/varnam_handlers.go index f052a6a..18cfb6f 100644 --- a/varnam_handlers.go +++ b/varnam_handlers.go @@ -177,8 +177,8 @@ func sendHandlerToChannel(schemeIdentifier string, handle *libvarnam.Varnam, ch } } -func getSchemeFilePath(schemeIdentifier string) { +func getSchemeFilePath(schemeIdentifier string) (interface{}, error) { return getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { - return handle.GetSchemeFilePath(schemeIdentifier), nil + return handle.GetSchemeFilePath(), nil }) } From 8b425289cab5de1dbd5fb2eba760ac16630608ba Mon Sep 17 00:00:00 2001 From: Subin Siby Date: Mon, 28 Sep 2020 02:02:03 +0530 Subject: [PATCH 47/56] Add authenticated endpoint for deleting a word https://github.com/varnamproject/varnamd/issues/21 --- http_handlers.go | 23 +++++++++++++++++++++++ libvarnam/libvarnam.go | 13 +++++++++++++ server.go | 4 +++- varnam_handlers.go | 6 ++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/http_handlers.go b/http_handlers.go index 7359a70..ada2f1a 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -359,6 +359,29 @@ func handleTrainBulk(c echo.Context) error { return c.JSON(200, "Words Trained") } +// Delete a word +func handleDelete(c echo.Context) error { + var ( + a args + + app = c.Get("app").(*App) + ) + + c.Request().Header.Set("Content-Type", "application/json") + + if err := c.Bind(&a); err != nil { + app.log.Printf("error in binding request details for delete, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } + + _, err := deleteWord(a.LangCode, a.Text) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error: %s", err.Error())) + } + + return c.JSON(http.StatusOK, "success") +} + func toggleDownloadEnabledStatus(langCode string, status bool) (interface{}, error) { if err := varnamdConfig.setDownloadStatus(langCode, status); err != nil { return nil, err diff --git a/libvarnam/libvarnam.go b/libvarnam/libvarnam.go index 3760f25..83a8eab 100644 --- a/libvarnam/libvarnam.go +++ b/libvarnam/libvarnam.go @@ -180,6 +180,19 @@ func (v *Varnam) Learn(text string) error { return nil } +// Learn from given input text. +func (v *Varnam) DeleteWord(text string) error { + rc := C.varnam_delete_word(v.handle, C.CString(text)) + + if rc != 0 { + errorCode := (int)(rc) + + return &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return nil +} + func (v *Varnam) getVarnamError(errorCode int) string { errormessage := C.varnam_get_last_error(v.handle) varnamErrorMsg := C.GoString(errormessage) diff --git a/server.go b/server.go index 0a154e7..a3b2394 100644 --- a/server.go +++ b/server.go @@ -34,12 +34,14 @@ func initHandlers(app *App, enableInternalApis bool) *echo.Echo { e.GET("/rtl/:langCode/:word", handleReverseTransliteration) e.GET("/meta/:langCode:", handleMetadata) e.GET("/download/:langCode/:downloadStart", handleDownload) - e.POST("/learn", handleLearn) e.GET("/languages", handleLanguages) e.GET("/languages/:langCode/download", handleLanguageDownload) e.GET("/status", handleStatus) + + e.POST("/learn", handleLearn) e.POST("/train/:langCode", authUser(handleTrain)) e.POST("/train/bulk/:langCode", authUser(handleTrainBulk)) + e.POST("/delete", authUser(handleDelete)) e.GET("/", handleIndex) diff --git a/varnam_handlers.go b/varnam_handlers.go index 18cfb6f..29e8474 100644 --- a/varnam_handlers.go +++ b/varnam_handlers.go @@ -182,3 +182,9 @@ func getSchemeFilePath(schemeIdentifier string) (interface{}, error) { return handle.GetSchemeFilePath(), nil }) } + +func deleteWord(schemeIdentifier string, word string) (interface{}, error) { + return getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { + return nil, handle.DeleteWord(word) + }) +} From 7967718413b583577987804fd39c041e06c7302b Mon Sep 17 00:00:00 2001 From: Subin Siby Date: Mon, 28 Sep 2020 02:57:57 +0530 Subject: [PATCH 48/56] Clear cache after delete or train --- cache.go | 12 ++++++++++++ http_handlers.go | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/cache.go b/cache.go index ca36f9d..14409f5 100644 --- a/cache.go +++ b/cache.go @@ -18,6 +18,8 @@ const ( type Cache interface { Set(lang, word string, val ...string) error Get(lang, word string) ([]string, error) + Delete(lang, word string) (bool, error) + Clear() } // MemCache impliments Cache interface. @@ -56,3 +58,13 @@ func (c *MemCache) Get(lang, word string) ([]string, error) { return strings.Split(string(val), wordSeparator), nil } + +func (c *MemCache) Delete(lang, word string) (bool, error) { + var key = fmt.Sprintf("%s-%s", lang, word) + affected := c.fc.Del([]byte(key)) + return affected, nil +} + +func (c *MemCache) Clear() { + c.fc.Clear() +} diff --git a/http_handlers.go b/http_handlers.go index ada2f1a..b2c09a5 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -315,6 +315,8 @@ func handleTrain(c echo.Context) error { go func(args trainArgs) { ch <- args }(targs) + app.cache.Delete(langCode, targs.Pattern) + return c.JSON(200, "Word Trained") } @@ -379,6 +381,8 @@ func handleDelete(c echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error: %s", err.Error())) } + app.cache.Clear() + return c.JSON(http.StatusOK, "success") } From 25338aebb25b6d245d863041d16dc7ac05c9fcee Mon Sep 17 00:00:00 2001 From: Subin Siby Date: Thu, 1 Oct 2020 02:29:27 +0530 Subject: [PATCH 49/56] Simplified code --- http_handlers.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/http_handlers.go b/http_handlers.go index b2c09a5..8bd64ca 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -265,7 +265,7 @@ func handleLanguageDownload(c echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error: %s", err.Error())) } - return c.Attachment(filepath.(string), langCode + ".vst") + return c.Attachment(filepath.(string), langCode+".vst") } func handleLearn(c echo.Context) error { @@ -376,8 +376,8 @@ func handleDelete(c echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } - _, err := deleteWord(a.LangCode, a.Text) - if err != nil { + if _, err := deleteWord(a.LangCode, a.Text); err != nil { + app.log.Printf("error deleting word, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error: %s", err.Error())) } From 89bdb005a6b202a39ab1cfa761d8d706eabab290 Mon Sep 17 00:00:00 2001 From: Subin Siby Date: Sat, 3 Oct 2020 09:32:38 +0000 Subject: [PATCH 50/56] Endpoint for uploading a file and learn (#5) * Add endpoint for uploading a file and learning from it * Add /learn/upload enpoint for files to be uploaded and learnt * Make /learn endpoints under auth * refactored code * Refactor, move functions to better suited files, use temp dir for upload * Disable auth by default * Change tmpdir Co-authored-by: Joice --- config.toml | 2 +- http_handlers.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++-- server.go | 5 ++++ varnam_learn.go | 39 +++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 3 deletions(-) diff --git a/config.toml b/config.toml index 861996c..fb284b4 100644 --- a/config.toml +++ b/config.toml @@ -7,7 +7,7 @@ upstream-url = "https://api.varnamproject.com" download-enabled-schemes = "" sync-interval = "30s" - accounts-enabled = true + accounts-enabled = false [users] [users.admin] password = "pass" diff --git a/http_handlers.go b/http_handlers.go index 8bd64ca..0cf678f 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -7,7 +7,11 @@ import ( "encoding/json" "errors" "fmt" + "io" + "io/ioutil" "net/http" + "os" + "path/filepath" "strconv" "strings" "time" @@ -275,8 +279,6 @@ func handleLearn(c echo.Context) error { app = c.Get("app").(*App) ) - c.Request().Header.Set("Content-Type", "application/json") - if err := c.Bind(&a); err != nil { app.log.Printf("error in binding request details for learn, err: %s", err.Error()) return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) @@ -293,6 +295,68 @@ func handleLearn(c echo.Context) error { return c.JSON(http.StatusOK, "success") } +func handleLearnFileUpload(c echo.Context) error { + var ( + app = c.Get("app").(*App) + langCode = c.Param("langCode") + ) + + // Multipart form + form, err := c.MultipartForm() + if err != nil { + app.log.Printf("failed to read form from request, language: %s, error: %s", langCode, err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, "request data not found") + } + + files, ok := form.File["files"] + if !ok { + app.log.Printf("files not found, language: %s", langCode) + return echo.NewHTTPError(http.StatusBadRequest, "no files were uploaded") + } + + if _, ok := learnChannels[langCode]; !ok { + app.log.Printf("learn file upload error: unknown language requested to learn: %s", langCode) + return echo.NewHTTPError(http.StatusBadRequest, "unable to find language to train") + } + + // Copy files first + for _, file := range files { + // Source + src, err := file.Open() + if err != nil { + app.log.Printf("learn file upload error, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + // Destination + tempDir, err := ioutil.TempDir(os.TempDir(), "varnamd") + if err != nil { + app.log.Printf("learn file upload error, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + dst, err := os.Create(filepath.Join(tempDir, file.Filename)) + if err != nil { + app.log.Printf("learn file upload error, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + // Copy + if _, err = io.Copy(dst, src); err != nil { + app.log.Printf("learn file upload error, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + // Explicitely closing resources. + _ = dst.Close() + _ = src.Close() + + learnWordsFromFile(c, langCode, dst.Name()) + } + + return c.JSON(http.StatusOK, "success") +} + func handleTrain(c echo.Context) error { var ( targs trainArgs diff --git a/server.go b/server.go index a3b2394..a188ad0 100644 --- a/server.go +++ b/server.go @@ -50,6 +50,11 @@ func initHandlers(app *App, enableInternalApis bool) *echo.Echo { if enableInternalApis { e.POST("/sync/download/:langCode/enable", handleEnableDownload) e.POST("/sync/download/:langCode/disable", handleDisableDownload) + + e.POST("/learn", authUser(handleLearn)) + e.POST("/learn/upload/:langCode", authUser(handleLearnFileUpload)) + e.POST("/train/:langCode", authUser(handleTrain)) + e.POST("/train/bulk/:langCode", authUser(handleTrainBulk)) } e.Use(middleware.Recover()) diff --git a/varnam_learn.go b/varnam_learn.go index 5ae4ff8..52fb7cc 100644 --- a/varnam_learn.go +++ b/varnam_learn.go @@ -1,8 +1,13 @@ package main import ( + "fmt" + "net/http" + "os" "strings" + "time" + "github.com/labstack/echo/v4" "github.com/varnamproject/varnamd/libvarnam" ) @@ -45,3 +50,37 @@ func (app *App) listenForWords(lang string, handle *libvarnam.Varnam) { } } } + +func learnWordsFromFile(c echo.Context, langCode string, fileToLearn string) { + c.Response().WriteHeader(http.StatusOK) + + start := time.Now() + + sendOutput := func(msg string) { + c.Response().Write([]byte(msg)) + c.Response().Flush() + } + + sendOutput(fmt.Sprintf("Learning from %s\n", fileToLearn)) + + _, _ = getOrCreateHandler(langCode, func(handle *libvarnam.Varnam) (data interface{}, err error) { + learnStatus, err := handle.LearnFromFile(fileToLearn) + end := time.Now() + + if err != nil { + sendOutput(fmt.Sprintf("Error learning from '%s'\n", err.Error())) + } else { + sendOutput(fmt.Sprintf("Learned from '%s'. TotalWords: %d, Failed: %d. Took %s\n", fileToLearn, learnStatus.TotalWords, learnStatus.Failed, end.Sub(start))) + } + + if err = os.Remove(fileToLearn); err != nil { + sendOutput(fmt.Sprintf("Error deleting '%s'. %s\n", fileToLearn, err.Error())) + } + + return + }) +} + +type learnFile struct { + lang string +} From da3e5efa87302137f9fbaba5570f7993c8cff6e8 Mon Sep 17 00:00:00 2001 From: "Joice M. Joseph" Date: Sat, 3 Oct 2020 15:11:40 +0530 Subject: [PATCH 51/56] fixing lints --- cache.go | 3 +++ http_handlers.go | 2 +- varnam_learn.go | 6 +----- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cache.go b/cache.go index 14409f5..cd3b6b9 100644 --- a/cache.go +++ b/cache.go @@ -59,12 +59,15 @@ func (c *MemCache) Get(lang, word string) ([]string, error) { return strings.Split(string(val), wordSeparator), nil } +// Delete lang-word from cache. func (c *MemCache) Delete(lang, word string) (bool, error) { var key = fmt.Sprintf("%s-%s", lang, word) affected := c.fc.Del([]byte(key)) + return affected, nil } +// Clear everything in the cache. func (c *MemCache) Clear() { c.fc.Clear() } diff --git a/http_handlers.go b/http_handlers.go index 0cf678f..c205552 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -379,7 +379,7 @@ func handleTrain(c echo.Context) error { go func(args trainArgs) { ch <- args }(targs) - app.cache.Delete(langCode, targs.Pattern) + _, _ = app.cache.Delete(langCode, targs.Pattern) return c.JSON(200, "Word Trained") } diff --git a/varnam_learn.go b/varnam_learn.go index 52fb7cc..fc8c63c 100644 --- a/varnam_learn.go +++ b/varnam_learn.go @@ -57,7 +57,7 @@ func learnWordsFromFile(c echo.Context, langCode string, fileToLearn string) { start := time.Now() sendOutput := func(msg string) { - c.Response().Write([]byte(msg)) + _, _ = c.Response().Write([]byte(msg)) c.Response().Flush() } @@ -80,7 +80,3 @@ func learnWordsFromFile(c echo.Context, langCode string, fileToLearn string) { return }) } - -type learnFile struct { - lang string -} From b86c4d736bee4bb1d4586bebcba476fc8eb084c7 Mon Sep 17 00:00:00 2001 From: Subin Siby Date: Sat, 3 Oct 2020 13:49:09 +0000 Subject: [PATCH 52/56] Language pack (#6) * Add /packs, /packs/:langCode endpoint, basic packs structure * Funcs should return errors too * Add /packs/:langCode/:packIdentifier endpoint for downloading a pack * Add endpoint to download pack from upstream * Enable internal API by default * Add learning from pack file * Add endpoint to get a pack version info & a /download to download that pack --- config.toml | 2 +- http_handlers.go | 84 ++++++++++++++++++++++++ packs.go | 163 +++++++++++++++++++++++++++++++++++++++++++++++ server.go | 11 ++-- 4 files changed, 254 insertions(+), 6 deletions(-) create mode 100644 packs.go diff --git a/config.toml b/config.toml index fb284b4..9d90b41 100644 --- a/config.toml +++ b/config.toml @@ -1,5 +1,5 @@ [app] - enable-internal-api = false + enable-internal-api = true enable-ssl = false cert-path = "" key-file-path = "" diff --git a/http_handlers.go b/http_handlers.go index c205552..f94e2f7 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -75,6 +75,12 @@ type trainBulkArgs struct { Word string `json:"word"` } +// PackDownloadRequestArgs is the args to download pack from upstream +type PackDownloadRequestArgs struct { + LangCode string `json:"lang"` + PackVersionIdentifier string `json:"pack"` +} + func handleStatus(c echo.Context) error { uptime := time.Since(startedAt) @@ -502,3 +508,81 @@ func handleIndex(c echo.Context) error { return c.String(http.StatusOK, string(b)) } + +func handlePacks(c echo.Context) error { + var ( + langCode = c.Param("langCode") + ) + + if langCode != "" { + pack, err := getPackInfo(langCode) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + return c.JSON(http.StatusOK, pack) + } + + packs, err := getPacksInfo() + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + return c.JSON(http.StatusOK, packs) +} + +func handlePackVersionInfo(c echo.Context) error { + var ( + langCode = c.Param("langCode") + packVersionIdentifier = c.Param("packVersionIdentifier") + ) + + pack, err := getPackVersionInfo(langCode, packVersionIdentifier) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + + return c.JSON(http.StatusOK, pack) +} + +func handlePacksDownload(c echo.Context) error { + var ( + packVersionIdentifier = c.Param("packVersionIdentifier") + langCode = c.Param("langCode") + ) + + if _, err := getPackInfo(langCode); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + + packFilePath, err := getPackFilePath(langCode, packVersionIdentifier) + + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + return c.Attachment(packFilePath, packVersionIdentifier) +} + +// varnamd Admin can download packs from upstream +// This is an internal function +func handlePackDownloadRequest(c echo.Context) error { + var ( + args PackDownloadRequestArgs + app = c.Get("app").(*App) + err error + ) + + if err := c.Bind(&args); err != nil { + app.log.Printf("error reading request, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } + + packFilePath, err := downloadPackFile(args.LangCode, args.PackVersionIdentifier) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error downloading pack: %s", err.Error())) + } + + learnWordsFromFile(c, args.LangCode, packFilePath) + + return c.JSON(http.StatusOK, "success") +} diff --git a/packs.go b/packs.go new file mode 100644 index 0000000..81badfb --- /dev/null +++ b/packs.go @@ -0,0 +1,163 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path" +) + +// PackVersion Details of a pack version +type PackVersion struct { + Identifier string // Pack identifier is unique across all language packs. Example: ml-basic-1 + Version int + Description string + Size int +} + +// Pack Details of a pack +type Pack struct { + Identifier string + Name string + Description string + LangCode string + Versions []PackVersion +} + +func fileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} + +// Download pack from upstream +func downloadPackFile(langCode string, packVersionIdentifier string) (string, error) { + fileURL := fmt.Sprintf("%s/packs/%s/%s", varnamdConfig.upstream, langCode, packVersionIdentifier) + filePath := path.Join(getPacksDir(), langCode, "a"+packVersionIdentifier) + + resp, err := http.Get(fileURL) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Fatal(err) + } + + return "", errors.New(string(respData)) + } + + out, err := os.Create(filePath) + if err != nil { + return "", err + } + defer out.Close() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + + if err != nil { + return "", err + } + + return filePath, nil +} + +func getPackFilePath(langCode string, packVersionIdentifier string) (string, error) { + if _, err := getPackVersionInfo(langCode, packVersionIdentifier); err != nil { + return "", err + } + + // Example: .varnamd/ml/ml-basic-1 + packFilePath := path.Join(getPacksDir(), langCode, packVersionIdentifier) + + if !fileExists(packFilePath) { + return "", errors.New("Pack file not found") + } + + return packFilePath, nil +} + +func getPackVersionInfo(langCode string, packVersionIdentifier string) (*PackVersion, error) { + pack, err := getPackInfo(langCode) + + if err != nil { + return nil, err + } + + var packVersion *PackVersion = nil + + for _, version := range pack.Versions { + if version.Identifier == packVersionIdentifier { + packVersion = &version + break + } + } + + if packVersion == nil { + return nil, fmt.Errorf("Pack version not found") + } + + return packVersion, nil +} + +func getPackInfo(langCode string) (*Pack, error) { + packs, err := getPacksInfo() + + if err != nil { + return nil, err + } + + for _, pack := range packs { + if pack.LangCode == langCode { + return &pack, nil + } + } + + return nil, errors.New("Pack not found") +} + +func getPacksInfo() ([]Pack, error) { + if err := createPacksDir(); err != nil { + err := fmt.Errorf("Failed to create packs directory, err: %s", err.Error()) + return nil, err + } + + packsFilePath := getPacksDir() + "/packs.json" + + if !fileExists(packsFilePath) { + err := errors.New("Packs file doesn't exist") + return nil, err + } + + packsFile, _ := ioutil.ReadFile(packsFilePath) + + var packsInfo []Pack + + if err := json.Unmarshal(packsFile, &packsInfo); err != nil { + err := fmt.Errorf("Parsing packs JSON failed, err: %s", err.Error()) + return nil, err + } + + return packsInfo, nil +} + +func createPacksDir() error { + packsDir := getPacksDir() + return os.MkdirAll(packsDir, 0750) +} + +func getPacksDir() string { + configDir := getConfigDir() + return path.Join(configDir, "packs") +} diff --git a/server.go b/server.go index a188ad0..400a48a 100644 --- a/server.go +++ b/server.go @@ -36,13 +36,12 @@ func initHandlers(app *App, enableInternalApis bool) *echo.Echo { e.GET("/download/:langCode/:downloadStart", handleDownload) e.GET("/languages", handleLanguages) e.GET("/languages/:langCode/download", handleLanguageDownload) + e.GET("/packs", handlePacks) + e.GET("/packs/:langCode", handlePacks) + e.GET("/packs/:langCode/:packVersionIdentifier", handlePackVersionInfo) + e.GET("/packs/:langCode/:packVersionIdentifier/download", handlePacksDownload) e.GET("/status", handleStatus) - e.POST("/learn", handleLearn) - e.POST("/train/:langCode", authUser(handleTrain)) - e.POST("/train/bulk/:langCode", authUser(handleTrainBulk)) - e.POST("/delete", authUser(handleDelete)) - e.GET("/", handleIndex) e.GET("/*", echo.WrapHandler(app.fs.FileServer())) @@ -55,6 +54,8 @@ func initHandlers(app *App, enableInternalApis bool) *echo.Echo { e.POST("/learn/upload/:langCode", authUser(handleLearnFileUpload)) e.POST("/train/:langCode", authUser(handleTrain)) e.POST("/train/bulk/:langCode", authUser(handleTrainBulk)) + e.POST("/delete", authUser(handleDelete)) + e.POST("/packs/download", handlePackDownloadRequest) } e.Use(middleware.Recover()) From d4132c1f9f8eedb6065a73fba731e4f78a7036fd Mon Sep 17 00:00:00 2001 From: Subin Siby Date: Mon, 5 Oct 2020 17:25:55 +0000 Subject: [PATCH 53/56] Language pack download (#8) * Add different endpoints for pack & diff pack versions * Show 404 if no pack found * packs.json should be updated when a new pack is downloaded from upstream * Disallow pack download if already installed * Test upstream URL * Refactor better * Assign small-caps JSON property names --- config.toml | 1 + http_handlers.go | 77 ++++++++++++++----- packs.go | 192 +++++++++++++++++++++++++++++++++++++++-------- server.go | 5 +- varnam_learn.go | 8 +- 5 files changed, 230 insertions(+), 53 deletions(-) diff --git a/config.toml b/config.toml index 9d90b41..9c58b12 100644 --- a/config.toml +++ b/config.toml @@ -5,6 +5,7 @@ key-file-path = "" max-handles = 10 upstream-url = "https://api.varnamproject.com" + #upstream-url = "http://127.0.0.1:8124" download-enabled-schemes = "" sync-interval = "30s" accounts-enabled = false diff --git a/http_handlers.go b/http_handlers.go index f94e2f7..1ae2ba7 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -75,10 +75,11 @@ type trainBulkArgs struct { Word string `json:"word"` } -// PackDownloadRequestArgs is the args to download pack from upstream -type PackDownloadRequestArgs struct { - LangCode string `json:"lang"` - PackVersionIdentifier string `json:"pack"` +// packDownloadArgs is the args to request a pack download from upstream +type packDownloadArgs struct { + LangCode string `json:"lang"` + Identifier string `json:"pack"` + Version string `json:"version"` } func handleStatus(c echo.Context) error { @@ -357,7 +358,7 @@ func handleLearnFileUpload(c echo.Context) error { _ = dst.Close() _ = src.Close() - learnWordsFromFile(c, langCode, dst.Name()) + learnWordsFromFile(c, langCode, dst.Name(), true) } return c.JSON(http.StatusOK, "success") @@ -512,33 +513,66 @@ func handleIndex(c echo.Context) error { func handlePacks(c echo.Context) error { var ( langCode = c.Param("langCode") + app = c.Get("app").(*App) ) if langCode != "" { - pack, err := getPackInfo(langCode) + pack, err := getPacksLangInfo(langCode) if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + statusCode := http.StatusBadRequest + if err.Error() == "No packs found" { + statusCode = http.StatusNotFound + } + + app.log.Printf("error reading packs, err: %s", err.Error()) + return echo.NewHTTPError(statusCode, err.Error()) } return c.JSON(http.StatusOK, pack) } packs, err := getPacksInfo() if err != nil { + app.log.Printf("error reading packs, err: %s", err.Error()) return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) } return c.JSON(http.StatusOK, packs) } +func handlePackInfo(c echo.Context) error { + var ( + langCode = c.Param("langCode") + packIdentifier = c.Param("packIdentifier") + ) + + pack, err := getPackInfo(langCode, packIdentifier) + if err != nil { + statusCode := http.StatusBadRequest + if err.Error() == "Pack not found" { + statusCode = http.StatusNotFound + } + + return echo.NewHTTPError(statusCode, err.Error()) + } + + return c.JSON(http.StatusOK, pack) +} + func handlePackVersionInfo(c echo.Context) error { var ( langCode = c.Param("langCode") + packIdentifier = c.Param("packIdentifier") packVersionIdentifier = c.Param("packVersionIdentifier") ) - pack, err := getPackVersionInfo(langCode, packVersionIdentifier) + pack, err := getPackVersionInfo(langCode, packIdentifier, packVersionIdentifier) if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + statusCode := http.StatusBadRequest + if err.Error() == "Pack version not found" { + statusCode = http.StatusNotFound + } + + return echo.NewHTTPError(statusCode, err.Error()) } return c.JSON(http.StatusOK, pack) @@ -546,15 +580,16 @@ func handlePackVersionInfo(c echo.Context) error { func handlePacksDownload(c echo.Context) error { var ( - packVersionIdentifier = c.Param("packVersionIdentifier") langCode = c.Param("langCode") + packIdentifier = c.Param("packIdentifier") + packVersionIdentifier = c.Param("packVersionIdentifier") ) - if _, err := getPackInfo(langCode); err != nil { + if _, err := getPackVersionInfo(langCode, packIdentifier, packVersionIdentifier); err != nil { return echo.NewHTTPError(http.StatusBadRequest, err.Error()) } - packFilePath, err := getPackFilePath(langCode, packVersionIdentifier) + packFilePath, err := getPackFilePath(langCode, packIdentifier, packVersionIdentifier) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) @@ -567,9 +602,10 @@ func handlePacksDownload(c echo.Context) error { // This is an internal function func handlePackDownloadRequest(c echo.Context) error { var ( - args PackDownloadRequestArgs - app = c.Get("app").(*App) - err error + args packDownloadArgs + app = c.Get("app").(*App) + err error + downloadResult packDownload ) if err := c.Bind(&args); err != nil { @@ -577,12 +613,19 @@ func handlePackDownloadRequest(c echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } - packFilePath, err := downloadPackFile(args.LangCode, args.PackVersionIdentifier) + downloadResult, err = downloadPackFile(args.LangCode, args.Identifier, args.Version) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error downloading pack: %s", err.Error())) } - learnWordsFromFile(c, args.LangCode, packFilePath) + // Learn from pack file and don't remove it + learnWordsFromFile(c, args.LangCode, downloadResult.FilePath, false) + + // Update packs.json with our new installed pack + err = updatePacksInfo(args.LangCode, downloadResult.Pack, downloadResult.Version) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } return c.JSON(http.StatusOK, "success") } diff --git a/packs.go b/packs.go index 81badfb..17e97c2 100644 --- a/packs.go +++ b/packs.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "net/http" "os" "path" @@ -14,19 +13,25 @@ import ( // PackVersion Details of a pack version type PackVersion struct { - Identifier string // Pack identifier is unique across all language packs. Example: ml-basic-1 - Version int - Description string - Size int + Identifier string `json:"identifier"` // Pack identifier is unique across all language packs. Example: ml-basic-1 + Version int `json:"version"` + Description string `json:"description"` + Size int `json:"size"` } // Pack Details of a pack type Pack struct { - Identifier string - Name string - Description string - LangCode string - Versions []PackVersion + Identifier string `json:"identifier"` + Name string `json:"name"` + Description string `json:"description"` + LangCode string `json:"lang"` + Versions []PackVersion `json:"versions"` +} + +type packDownload struct { + Pack *Pack + Version *PackVersion + FilePath string } func fileExists(filename string) bool { @@ -37,29 +42,123 @@ func fileExists(filename string) bool { return !info.IsDir() } +// After a new pack download from upstream, update packs.json with installed packs +func updatePacksInfo(langCode string, pack *Pack, packVersion *PackVersion) error { + packs, err := getPacksInfo() + if err != nil { + return err + } + + var ( + existingPackIndex int + existingPack *Pack = nil + ) + + for index, packR := range packs { + if packR.Identifier == pack.Identifier { + existingPackIndex = index + existingPack = &packR + break + } + } + + if existingPack == nil { + // will have one element + pack.Versions = []PackVersion{*packVersion} + + // Append new pack + packs = append(packs, *pack) + } else { + // Append new pack version + existingPack.Versions = append(existingPack.Versions, *packVersion) + + packs[existingPackIndex] = *existingPack + } + + file, err := json.MarshalIndent(packs, "", " ") + if err != nil { + return err + } + + err = ioutil.WriteFile(getPacksInfoPath(), file, 0644) + if err != nil { + return err + } + + return nil +} + // Download pack from upstream -func downloadPackFile(langCode string, packVersionIdentifier string) (string, error) { - fileURL := fmt.Sprintf("%s/packs/%s/%s", varnamdConfig.upstream, langCode, packVersionIdentifier) - filePath := path.Join(getPacksDir(), langCode, "a"+packVersionIdentifier) +func downloadPackFile(langCode, packIdentifier, packVersionIdentifier string) (packDownload, error) { + var ( + pack *Pack + packVersion *PackVersion = nil + ) + + packInstalled, _ := getPackVersionInfo(langCode, packIdentifier, packVersionIdentifier) + if packInstalled != nil { + return packDownload{}, fmt.Errorf("Pack already installed") + } - resp, err := http.Get(fileURL) + packInfoURL := fmt.Sprintf("%s/packs/%s/%s", varnamdConfig.upstream, langCode, packIdentifier) + + resp, err := http.Get(packInfoURL) if err != nil { - return "", err + return packDownload{}, err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return packDownload{}, err + } + + return packDownload{}, fmt.Errorf(string(respData)) + } + + if err = json.NewDecoder(resp.Body).Decode(&pack); err != nil { + err := fmt.Errorf("Parsing packs JSON failed, err: %s", err.Error()) + return packDownload{}, err + } + + for _, version := range pack.Versions { + if version.Identifier == packVersionIdentifier { + packVersion = &version + break + } + } + + if packVersion == nil { + return packDownload{}, fmt.Errorf("Pack version not found") + } + + fileURL := fmt.Sprintf("%s/packs/%s/%s/%s/download", varnamdConfig.upstream, langCode, packIdentifier, packVersionIdentifier) + fileDir := path.Join(getPacksDir(), langCode) + filePath := path.Join(fileDir, packVersionIdentifier) + + if !fileExists(fileDir) { + os.MkdirAll(fileDir, 0755) + } + + resp, err = http.Get(fileURL) + if err != nil { + return packDownload{}, err } defer resp.Body.Close() if resp.StatusCode != 200 { respData, err := ioutil.ReadAll(resp.Body) if err != nil { - log.Fatal(err) + return packDownload{}, err } - return "", errors.New(string(respData)) + return packDownload{}, fmt.Errorf(string(respData)) } out, err := os.Create(filePath) if err != nil { - return "", err + return packDownload{}, err } defer out.Close() @@ -67,14 +166,14 @@ func downloadPackFile(langCode string, packVersionIdentifier string) (string, er _, err = io.Copy(out, resp.Body) if err != nil { - return "", err + return packDownload{}, err } - return filePath, nil + return packDownload{Pack: pack, Version: packVersion, FilePath: filePath}, nil } -func getPackFilePath(langCode string, packVersionIdentifier string) (string, error) { - if _, err := getPackVersionInfo(langCode, packVersionIdentifier); err != nil { +func getPackFilePath(langCode, packIdentifier, packVersionIdentifier string) (string, error) { + if _, err := getPackVersionInfo(langCode, packIdentifier, packVersionIdentifier); err != nil { return "", err } @@ -88,8 +187,8 @@ func getPackFilePath(langCode string, packVersionIdentifier string) (string, err return packFilePath, nil } -func getPackVersionInfo(langCode string, packVersionIdentifier string) (*PackVersion, error) { - pack, err := getPackInfo(langCode) +func getPackVersionInfo(langCode string, packIdentifier string, packVersionIdentifier string) (*PackVersion, error) { + pack, err := getPackInfo(langCode, packIdentifier) if err != nil { return nil, err @@ -111,20 +210,43 @@ func getPackVersionInfo(langCode string, packVersionIdentifier string) (*PackVer return packVersion, nil } -func getPackInfo(langCode string) (*Pack, error) { +func getPackInfo(langCode string, packIdentifier string) (*Pack, error) { + packs, err := getPacksLangInfo(langCode) + + if err != nil { + return nil, err + } + + for _, pack := range packs { + if pack.Identifier == packIdentifier { + return &pack, nil + } + } + + return nil, fmt.Errorf("Pack not found") +} + +// Get packs by language +func getPacksLangInfo(langCode string) ([]Pack, error) { packs, err := getPacksInfo() if err != nil { return nil, err } + var langPacks []Pack + for _, pack := range packs { if pack.LangCode == langCode { - return &pack, nil + langPacks = append(langPacks, pack) } } - return nil, errors.New("Pack not found") + if len(langPacks) == 0 { + return nil, fmt.Errorf("No packs found") + } + + return langPacks, nil } func getPacksInfo() ([]Pack, error) { @@ -133,11 +255,15 @@ func getPacksInfo() ([]Pack, error) { return nil, err } - packsFilePath := getPacksDir() + "/packs.json" + packsFilePath := getPacksInfoPath() if !fileExists(packsFilePath) { - err := errors.New("Packs file doesn't exist") - return nil, err + file, err := os.Create(packsFilePath) + if err != nil { + return nil, err + } + file.WriteString("[]") + defer file.Close() } packsFile, _ := ioutil.ReadFile(packsFilePath) @@ -152,9 +278,13 @@ func getPacksInfo() ([]Pack, error) { return packsInfo, nil } +func getPacksInfoPath() string { + return getPacksDir() + "/packs.json" +} + func createPacksDir() error { packsDir := getPacksDir() - return os.MkdirAll(packsDir, 0750) + return os.MkdirAll(packsDir, 0644) } func getPacksDir() string { diff --git a/server.go b/server.go index 400a48a..7e50c76 100644 --- a/server.go +++ b/server.go @@ -38,8 +38,9 @@ func initHandlers(app *App, enableInternalApis bool) *echo.Echo { e.GET("/languages/:langCode/download", handleLanguageDownload) e.GET("/packs", handlePacks) e.GET("/packs/:langCode", handlePacks) - e.GET("/packs/:langCode/:packVersionIdentifier", handlePackVersionInfo) - e.GET("/packs/:langCode/:packVersionIdentifier/download", handlePacksDownload) + e.GET("/packs/:langCode/:packIdentifier", handlePackInfo) + e.GET("/packs/:langCode/:packIdentifier/:packVersionIdentifier", handlePackVersionInfo) + e.GET("/packs/:langCode/:packIdentifier/:packVersionIdentifier/download", handlePacksDownload) e.GET("/status", handleStatus) e.GET("/", handleIndex) diff --git a/varnam_learn.go b/varnam_learn.go index fc8c63c..d4ff496 100644 --- a/varnam_learn.go +++ b/varnam_learn.go @@ -51,7 +51,7 @@ func (app *App) listenForWords(lang string, handle *libvarnam.Varnam) { } } -func learnWordsFromFile(c echo.Context, langCode string, fileToLearn string) { +func learnWordsFromFile(c echo.Context, langCode string, fileToLearn string, removeFile bool) { c.Response().WriteHeader(http.StatusOK) start := time.Now() @@ -73,8 +73,10 @@ func learnWordsFromFile(c echo.Context, langCode string, fileToLearn string) { sendOutput(fmt.Sprintf("Learned from '%s'. TotalWords: %d, Failed: %d. Took %s\n", fileToLearn, learnStatus.TotalWords, learnStatus.Failed, end.Sub(start))) } - if err = os.Remove(fileToLearn); err != nil { - sendOutput(fmt.Sprintf("Error deleting '%s'. %s\n", fileToLearn, err.Error())) + if removeFile { + if err = os.Remove(fileToLearn); err != nil { + sendOutput(fmt.Sprintf("Error deleting '%s'. %s\n", fileToLearn, err.Error())) + } } return From fceeafd97f2439a7a9e4e012800559b6cbc800d0 Mon Sep 17 00:00:00 2001 From: Subin Siby Date: Thu, 8 Oct 2020 15:55:49 +0530 Subject: [PATCH 54/56] Fix packs dir creation permission --- packs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packs.go b/packs.go index 17e97c2..9f63e5a 100644 --- a/packs.go +++ b/packs.go @@ -284,7 +284,7 @@ func getPacksInfoPath() string { func createPacksDir() error { packsDir := getPacksDir() - return os.MkdirAll(packsDir, 0644) + return os.MkdirAll(packsDir, 0750) } func getPacksDir() string { From b18192cbbcb558c0603ef0eb5e270ffe5adc575a Mon Sep 17 00:00:00 2001 From: Subin Siby Date: Sat, 7 Nov 2020 07:24:49 +0000 Subject: [PATCH 55/56] Use Trained files in Pack to import (#9) * Remove usage of packs.json and use pack.json inside different packs * Update pack.json when packs are downloaded * Packs are now trained files to import from and not files to learn from * Show error if pack import failed, don't update pack.json then * Cache packs info for faster requests and avoid recurrent disk reads * Compress pack downloads into gzip, considerable filesize decrease * Use new vpf extension for pack file * division is the correct way to convert to int --- http_handlers.go | 32 +++++++++++++-- libvarnam/libvarnam.go | 15 ++++++- main.go | 2 +- packs.go | 90 +++++++++++++++++++++++++++--------------- varnam_learn.go | 35 ++++++++++++++++ 5 files changed, 137 insertions(+), 37 deletions(-) diff --git a/http_handlers.go b/http_handlers.go index 1ae2ba7..f75b55a 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "net/http" "os" + "path" "path/filepath" "strconv" "strings" @@ -595,7 +596,28 @@ func handlePacksDownload(c echo.Context) error { return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) } - return c.Attachment(packFilePath, packVersionIdentifier) + packFileGzipPath := path.Join(packFilePath + ".gzip") + + if !fileExists(packFileGzipPath) { + // compress into gzip + packFileBytes, err := ioutil.ReadFile(packFilePath) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + var gb bytes.Buffer + w := gzip.NewWriter(&gb) + w.Write(packFileBytes) + w.Close() + + err = ioutil.WriteFile(packFileGzipPath, gb.Bytes(), 0644) + + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + } + + return c.Attachment(packFileGzipPath, packVersionIdentifier) } // varnamd Admin can download packs from upstream @@ -619,9 +641,13 @@ func handlePackDownloadRequest(c echo.Context) error { } // Learn from pack file and don't remove it - learnWordsFromFile(c, args.LangCode, downloadResult.FilePath, false) + err = importLearningsFromFile(c, args.LangCode, downloadResult.FilePath, false) + + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("Error importing from '%s'\n", err.Error())) + } - // Update packs.json with our new installed pack + // Add pack.json with the installed pack versions err = updatePacksInfo(args.LangCode, downloadResult.Pack, downloadResult.Version) if err != nil { return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) diff --git a/libvarnam/libvarnam.go b/libvarnam/libvarnam.go index 83a8eab..d81b63d 100644 --- a/libvarnam/libvarnam.go +++ b/libvarnam/libvarnam.go @@ -118,6 +118,19 @@ func (v *Varnam) LearnFromFile(filePath string) (*LearnStatus, error) { return &LearnStatus{TotalWords: int(status.total_words), Failed: int(status.failed)}, nil } +// ImportFromFile Import learnigns from file (varnam exported file) +func (v *Varnam) ImportFromFile(filePath string) error { + rc := C.varnam_import_learnings_from_file(v.handle, C.CString(filePath)) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return nil +} + // Init initializes varnam bindings. func Init(schemeIdentifier string) (*Varnam, error) { var ( @@ -180,7 +193,7 @@ func (v *Varnam) Learn(text string) error { return nil } -// Learn from given input text. +// DeleteWord from given input text. func (v *Varnam) DeleteWord(text string) error { rc := C.varnam_delete_word(v.handle, C.CString(text)) diff --git a/main.go b/main.go index c1745c6..e39a302 100644 --- a/main.go +++ b/main.go @@ -116,7 +116,7 @@ func syncRequired() bool { // Starts the sync process only if it is not running func startSyncDispatcher() { if syncRequired() && !syncDispatcherRunning { - sync := newSyncDispatcher(varnamdConfig.syncInterval * time.Second) + sync := newSyncDispatcher(varnamdConfig.syncInterval / time.Second) sync.start() sync.runNow() // run one round of sync immediatly rather than waiting for the next interval to occur diff --git a/packs.go b/packs.go index 9f63e5a..57e8fbe 100644 --- a/packs.go +++ b/packs.go @@ -1,6 +1,7 @@ package main import ( + "compress/gzip" "encoding/json" "errors" "fmt" @@ -34,6 +35,8 @@ type packDownload struct { FilePath string } +var packsInfoCached []Pack + func fileExists(filename string) bool { info, err := os.Stat(filename) if os.IsNotExist(err) { @@ -50,13 +53,11 @@ func updatePacksInfo(langCode string, pack *Pack, packVersion *PackVersion) erro } var ( - existingPackIndex int - existingPack *Pack = nil + existingPack *Pack = nil ) - for index, packR := range packs { + for _, packR := range packs { if packR.Identifier == pack.Identifier { - existingPackIndex = index existingPack = &packR break } @@ -65,26 +66,27 @@ func updatePacksInfo(langCode string, pack *Pack, packVersion *PackVersion) erro if existingPack == nil { // will have one element pack.Versions = []PackVersion{*packVersion} - - // Append new pack - packs = append(packs, *pack) } else { // Append new pack version existingPack.Versions = append(existingPack.Versions, *packVersion) - packs[existingPackIndex] = *existingPack + pack = existingPack } - file, err := json.MarshalIndent(packs, "", " ") + // Save pack.json + packInfoPath := path.Join(getPacksDir(), pack.LangCode, pack.Identifier, "pack.json") + file, err := json.MarshalIndent(pack, "", " ") if err != nil { return err } - err = ioutil.WriteFile(getPacksInfoPath(), file, 0644) + err = ioutil.WriteFile(packInfoPath, file, 0644) if err != nil { return err } + packsInfoCached = nil + return nil } @@ -134,7 +136,7 @@ func downloadPackFile(langCode, packIdentifier, packVersionIdentifier string) (p } fileURL := fmt.Sprintf("%s/packs/%s/%s/%s/download", varnamdConfig.upstream, langCode, packIdentifier, packVersionIdentifier) - fileDir := path.Join(getPacksDir(), langCode) + fileDir := path.Join(getPacksDir(), langCode, packIdentifier) filePath := path.Join(fileDir, packVersionIdentifier) if !fileExists(fileDir) { @@ -156,14 +158,20 @@ func downloadPackFile(langCode, packIdentifier, packVersionIdentifier string) (p return packDownload{}, fmt.Errorf(string(respData)) } + fz, err := gzip.NewReader(resp.Body) + if err != nil { + return packDownload{}, err + } + defer fz.Close() + out, err := os.Create(filePath) if err != nil { return packDownload{}, err } defer out.Close() - // Write the body to file - _, err = io.Copy(out, resp.Body) + // Write the gzip decoded content to file + _, err = io.Copy(out, fz) if err != nil { return packDownload{}, err @@ -177,8 +185,8 @@ func getPackFilePath(langCode, packIdentifier, packVersionIdentifier string) (st return "", err } - // Example: .varnamd/ml/ml-basic-1 - packFilePath := path.Join(getPacksDir(), langCode, packVersionIdentifier) + // Example: .varnamd/ml/ml-basic/ml-basic-1.vpf + packFilePath := path.Join(getPacksDir(), langCode, packIdentifier, packVersionIdentifier) + ".vpf" if !fileExists(packFilePath) { return "", errors.New("Pack file not found") @@ -255,31 +263,49 @@ func getPacksInfo() ([]Pack, error) { return nil, err } - packsFilePath := getPacksInfoPath() - - if !fileExists(packsFilePath) { - file, err := os.Create(packsFilePath) - if err != nil { - return nil, err - } - file.WriteString("[]") - defer file.Close() + if packsInfoCached != nil { + return packsInfoCached, nil } - packsFile, _ := ioutil.ReadFile(packsFilePath) - var packsInfo []Pack - if err := json.Unmarshal(packsFile, &packsInfo); err != nil { - err := fmt.Errorf("Parsing packs JSON failed, err: %s", err.Error()) + files, err := ioutil.ReadDir(getPacksDir()) + if err != nil { return nil, err } - return packsInfo, nil -} + for _, langFolder := range files { + langFolderPath := path.Join(getPacksDir(), langFolder.Name()) + if langFolder.IsDir() { + // inside ml + langFolderFiles, err := ioutil.ReadDir(langFolderPath) + + if err != nil { + return nil, err + } + + for _, packFolder := range langFolderFiles { + if packFolder.IsDir() { + packInfoPath := path.Join(langFolderPath, packFolder.Name(), "pack.json") + if fileExists(packInfoPath) { + var packInfo Pack + packsFile, _ := ioutil.ReadFile(packInfoPath) + + if err := json.Unmarshal(packsFile, &packInfo); err != nil { + err := fmt.Errorf("Parsing packs JSON failed, err: %s", err.Error()) + return nil, err + } + + packsInfo = append(packsInfo, packInfo) + } + } + } + } + } -func getPacksInfoPath() string { - return getPacksDir() + "/packs.json" + packsInfoCached = packsInfo + + return packsInfo, nil } func createPacksDir() error { diff --git a/varnam_learn.go b/varnam_learn.go index d4ff496..22e4225 100644 --- a/varnam_learn.go +++ b/varnam_learn.go @@ -82,3 +82,38 @@ func learnWordsFromFile(c echo.Context, langCode string, fileToLearn string, rem return }) } + +func importLearningsFromFile(c echo.Context, langCode string, fileToLearn string, removeFile bool) error { + c.Response().WriteHeader(http.StatusOK) + + start := time.Now() + + sendOutput := func(msg string) { + _, _ = c.Response().Write([]byte(msg)) + c.Response().Flush() + } + + sendOutput(fmt.Sprintf("Importing from %s\n", fileToLearn)) + + var importError error + + _, _ = getOrCreateHandler(langCode, func(handle *libvarnam.Varnam) (data interface{}, err error) { + err = handle.ImportFromFile(fileToLearn) + end := time.Now() + + if err != nil { + importError = err + } else { + sendOutput(fmt.Sprintf("Import completed. Took %s\n", end.Sub(start))) + } + + if removeFile { + if err = os.Remove(fileToLearn); err != nil { + sendOutput(fmt.Sprintf("Error deleting '%s'. %s\n", fileToLearn, err.Error())) + } + } + return + }) + + return importError +} From 27570aac9c9aa5054967b5b241a92cef2fbb34bd Mon Sep 17 00:00:00 2001 From: Subin Siby Date: Sat, 28 Nov 2020 16:40:28 +0530 Subject: [PATCH 56/56] .vlf - Varnam Learnings File --- packs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packs.go b/packs.go index 57e8fbe..a59de2a 100644 --- a/packs.go +++ b/packs.go @@ -185,8 +185,8 @@ func getPackFilePath(langCode, packIdentifier, packVersionIdentifier string) (st return "", err } - // Example: .varnamd/ml/ml-basic/ml-basic-1.vpf - packFilePath := path.Join(getPacksDir(), langCode, packIdentifier, packVersionIdentifier) + ".vpf" + // Example: .varnamd/ml/ml-basic/ml-basic-1.vlf + packFilePath := path.Join(getPacksDir(), langCode, packIdentifier, packVersionIdentifier) + ".vlf" if !fileExists(packFilePath) { return "", errors.New("Pack file not found")