diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bbd179b --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +BIN := varnam.bin +HASH := $(shell git rev-parse HEAD | cut -c 1-8) +COMMIT_DATE := $(shell git show -s --format=%ci ${HASH}) +BUILD_DATE := $(shell date '+%Y-%m-%d %H:%M:%S') +VERSION := ${HASH} (${COMMIT_DATE}) +STATIC := ui:/ + +deps: + go get -u github.com/knadh/stuffbin/... + +.PHONY: build +build: ## Build the binary (default) + go build -o ${BIN} -ldflags="-X 'main.buildVersion=${VERSION}' -X 'main.buildDate=${BUILD_DATE}' -s -w" + stuffbin -a stuff -in ${BIN} -out ${BIN} ${STATIC} + +.PHONY: run +run: build + ./${BIN} + +.PHONY: clean +clean: ## Remove temporary files and the binary + go clean + +# Absolutely awesome: http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html +.PHONY: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.DEFAULT_GOAL := build \ No newline at end of file diff --git a/cache.go b/cache.go new file mode 100644 index 0000000..cd3b6b9 --- /dev/null +++ b/cache.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "strings" + "time" + + "github.com/coocood/freecache" +) + +const ( + defaultCacheSize = 100 << 20 // 100 MB cache + defaultExpiry = time.Hour * 24 // Cache for 24 hours + wordSeparator = "<>" // Assuming the separator wont be used in any case. +) + +// Cache objects. +type Cache interface { + Set(lang, word string, val ...string) error + Get(lang, word string) ([]string, error) + Delete(lang, word string) (bool, error) + Clear() +} + +// MemCache impliments Cache interface. +type MemCache struct { + fc *freecache.Cache +} + +// NewMemCache will create object of new Cache impliemtation. +func NewMemCache() *MemCache { + return newCacheWithSize(defaultCacheSize) +} + +func newCacheWithSize(size int) *MemCache { + return &MemCache{fc: freecache.NewCache(size)} +} + +// Set lang-word as val to cache. +func (c *MemCache) Set(lang, word string, val ...string) error { + var value = strings.Join(val, wordSeparator) + return c.setWithExpiry(lang, word, []byte(value), int(defaultExpiry)) +} + +func (c *MemCache) setWithExpiry(lang, word string, val []byte, expiry int) error { + var key = fmt.Sprintf("%s-%s", lang, word) + return c.fc.Set([]byte(key), val, expiry) +} + +// Get lang-word from cache. +func (c *MemCache) Get(lang, word string) ([]string, error) { + var key = fmt.Sprintf("%s-%s", lang, word) + + val, err := c.fc.Get([]byte(key)) + if err != nil { + return nil, err + } + + return strings.Split(string(val), wordSeparator), nil +} + +// Delete lang-word from cache. +func (c *MemCache) Delete(lang, word string) (bool, error) { + var key = fmt.Sprintf("%s-%s", lang, word) + affected := c.fc.Del([]byte(key)) + + return affected, nil +} + +// Clear everything in the cache. +func (c *MemCache) Clear() { + c.fc.Clear() +} diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..9c58b12 --- /dev/null +++ b/config.toml @@ -0,0 +1,14 @@ +[app] + enable-internal-api = true + enable-ssl = false + cert-path = "" + key-file-path = "" + max-handles = 10 + upstream-url = "https://api.varnamproject.com" + #upstream-url = "http://127.0.0.1:8124" + download-enabled-schemes = "" + sync-interval = "30s" + accounts-enabled = false +[users] + [users.admin] + password = "pass" diff --git a/defs.go b/defs.go deleted file mode 100644 index f3cbb5d..0000000 --- a/defs.go +++ /dev/null @@ -1,4 +0,0 @@ -package main - -const varnamdVersion = "0.0.8" -const downloadPageSize = 100 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..4938e4b --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module github.com/varnamproject/varnamd + +go 1.14 + +require ( + github.com/coocood/freecache v1.1.1 + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e + github.com/golang/protobuf v1.4.2 // indirect + github.com/knadh/koanf v0.12.1 + github.com/knadh/stuffbin v1.1.0 + github.com/labstack/echo/v4 v4.1.16 + github.com/mattn/go-colorable v0.1.7 // indirect + github.com/mattn/go-sqlite3 v1.14.0 + github.com/spf13/pflag v1.0.5 + github.com/valyala/fasttemplate v1.2.0 // indirect + golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect + golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect + golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect + golang.org/x/text v0.3.3 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..da866de --- /dev/null +++ b/go.sum @@ -0,0 +1,122 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc= +github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/knadh/koanf v0.12.1 h1:9N0asqPUvZ0jL9thTbMZ0FAT3VrGfR+aYm5JTInT+Gs= +github.com/knadh/koanf v0.12.1/go.mod h1:31bzRSM7vS5Vm9LNLo7B2Re1zhLOZT6EQKeodixBikE= +github.com/knadh/stuffbin v1.1.0 h1:f5S5BHzZALjuJEgTIOMC9NidEnBJM7Ze6Lu1GHR/lwU= +github.com/knadh/stuffbin v1.1.0/go.mod h1:yVCFaWaKPubSNibBsTAJ939q2ABHudJQxRWZWV5yh+4= +github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz0o= +github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= +github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= +github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rhnvrm/simples3 v0.5.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.0 h1:y3yXRCoDvC2HTtIHvL2cc7Zd+bqA+zqDO6oQzsJO07E= +github.com/valyala/fasttemplate v1.2.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/http_handlers.go b/http_handlers.go index 19e448e..f75b55a 100644 --- a/http_handlers.go +++ b/http_handlers.go @@ -3,19 +3,23 @@ package main import ( "bytes" "compress/gzip" + "context" "encoding/json" "errors" "fmt" - "log" + "io" + "io/ioutil" "net/http" - "runtime" + "os" + "path" + "path/filepath" "strconv" "strings" "time" "github.com/golang/groupcache" - "github.com/gorilla/mux" - "github.com/varnamproject/libvarnam-golang" + "github.com/labstack/echo/v4" + "github.com/varnamproject/varnamd/libvarnam" ) var errCacheSkipped = errors.New("cache skipped") @@ -24,6 +28,7 @@ var errCacheSkipped = errors.New("cache skipped") // Data will be set if the cache returns CacheSkipped type varnamCacheContext struct { Data []byte + context.Context } type standardResponse struct { @@ -32,13 +37,8 @@ type standardResponse struct { At string `json:"at"` } -func newStandardResponse(err string) standardResponse { - s := standardResponse{Success: true, Error: "", At: time.Now().UTC().String()} - if err != "" { - s.Error = err - s.Success = false - } - return s +func newStandardResponse() standardResponse { + return standardResponse{Success: true, At: time.Now().UTC().String()} } type transliterationResponse struct { @@ -58,158 +58,152 @@ type downloadResponse struct { standardResponse } -type requestParams struct { - langCode string - word string - downloadStart int +// Args to read. +type args struct { + LangCode string `json:"lang"` + Text string `json:"text"` } -func parseParams(r *http.Request) *requestParams { - params := mux.Vars(r) - downloadStart, _ := strconv.Atoi(params["downloadStart"]) - return &requestParams{langCode: params["langCode"], word: params["word"], - downloadStart: downloadStart} +//TrainArgs read the incoming data +type trainArgs struct { + Pattern string `json:"pattern"` + Word string `json:"word"` } -func corsHandler(h http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Access-Control-Allow-Origin", "*") - if r.Method == "OPTIONS" { - w.WriteHeader(http.StatusOK) - } else { - h.ServeHTTP(w, r) - } - } +//TrainBulkArgs read the incoming data for bulk training. +type trainBulkArgs struct { + Pattern []string `json:"pattern"` + Word string `json:"word"` } -func recoverHandler(next http.Handler) http.Handler { - fn := func(w http.ResponseWriter, r *http.Request) { - defer func() { - if err := recover(); err != nil { - stack := make([]byte, 1024) - stack = stack[:runtime.Stack(stack, false)] - log.Printf("panic: %s\n%s", err, stack) - http.Error(w, http.StatusText(500), 500) - } - }() - next.ServeHTTP(w, r) - } - return http.HandlerFunc(fn) +// packDownloadArgs is the args to request a pack download from upstream +type packDownloadArgs struct { + LangCode string `json:"lang"` + Identifier string `json:"pack"` + Version string `json:"version"` } -func renderError(w http.ResponseWriter, err error) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - if err != nil { - w.WriteHeader(http.StatusBadRequest) - errorData := newStandardResponse(err.Error()) - json.NewEncoder(w).Encode(errorData) - } -} - -func renderGzippedJSON(w http.ResponseWriter, data []byte) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.Header().Set("Content-Encoding", "gzip") - w.Write(data) -} +func handleStatus(c echo.Context) error { + uptime := time.Since(startedAt) -func renderJSON(w http.ResponseWriter, data interface{}) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(data) -} - -func getLanguageAndWord(r *http.Request) (langCode string, word string) { - params := mux.Vars(r) - langCode = params["langCode"] - word = params["word"] - return -} - -func getLangCode(r *http.Request) string { - params := mux.Vars(r) - return params["langCode"] -} - -func getWord(r *http.Request) string { - params := mux.Vars(r) - return params["word"] -} - -func statusHandler(w http.ResponseWriter, r *http.Request) { - uptime := time.Now().Sub(startedAt) resp := struct { Version string `json:"version"` Uptime string `json:"uptime"` standardResponse }{ - varnamdVersion, + buildVersion + "-" + buildDate, uptime.String(), - newStandardResponse(""), + newStandardResponse(), } - renderJSON(w, resp) + return c.JSON(http.StatusOK, resp) } -func transliterationHandler(w http.ResponseWriter, r *http.Request) { - langCode, word := getLanguageAndWord(r) - words, err := transliterate(langCode, word) +func handleTransliteration(c echo.Context) error { + var ( + langCode = c.Param("langCode") + word = c.Param("word") + app = c.Get("app").(*App) + ) + + words, err := app.cache.Get(langCode, word) if err != nil { - renderError(w, err) - } else { - renderJSON(w, - transliterationResponse{standardResponse: newStandardResponse(""), Result: words.([]string), Input: word}) + w, err := transliterate(langCode, word) + if err != nil { + app.log.Printf("error in transliterationg, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) + } + + words, _ = w.([]string) + _ = app.cache.Set(langCode, word, words...) } + + return c.JSON(http.StatusOK, transliterationResponse{standardResponse: newStandardResponse(), Result: words, Input: word}) } -func reverseTransliterationHandler(w http.ResponseWriter, r *http.Request) { - langCode, word := getLanguageAndWord(r) - result, err := reveseTransliterate(langCode, word) +func handleReverseTransliteration(c echo.Context) error { + var ( + langCode = c.Param("langCode") + word = c.Param("word") + app = c.Get("app").(*App) + ) + + result, err := app.cache.Get(langCode, word) if err != nil { - renderError(w, err) - } else { - response := struct { - standardResponse - Result string `json:"result"` - }{ - newStandardResponse(""), - result.(string), + res, err := reveseTransliterate(langCode, word) + if err != nil { + app.log.Printf("error in reverse transliterationg, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error transliterating given string. message: %s", err.Error())) } - renderJSON(w, response) + + result = []string{res.(string)} + _ = app.cache.Set(langCode, word, res.(string)) + } + + if len(result) <= 0 { + app.log.Printf("no reverse transliteration found for lang: %s word: %s", langCode, word) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("no transliteration found for lanugage: %s, word: %s", langCode, word)) + } + + response := struct { + standardResponse + Result string `json:"result"` + }{ + newStandardResponse(), + result[0], } + + return c.JSON(http.StatusOK, response) } -func metadataHandler(w http.ResponseWriter, r *http.Request) { - schemeIdentifier, _ := getLanguageAndWord(r) - getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { +func handleMetadata(c echo.Context) error { + var ( + schemeIdentifier = c.Param("langCode") + app = c.Get("app").(*App) + ) + + data, err := getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { details, err := handle.GetCorpusDetails() if err != nil { - renderError(w, err) - return + return nil, err } - renderJSON(w, &metaResponse{Result: details, standardResponse: newStandardResponse("")}) - return + + return &metaResponse{Result: details, standardResponse: newStandardResponse()}, nil }) + if err != nil { + app.log.Printf("error in getting corpus details for: %s, err: %s", schemeIdentifier, err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } + + return c.JSON(http.StatusOK, data) } -func downloadHandler(w http.ResponseWriter, r *http.Request) { - params := parseParams(r) - if params.downloadStart < 0 { - renderError(w, errors.New("Invalid parameters")) - return +func handleDownload(c echo.Context) error { + var ( + langCode = c.Param("langCode") + start, _ = strconv.Atoi(c.Param("downloadStart")) + + app = c.Get("app").(*App) + ) + + if start < 0 { + return echo.NewHTTPError(http.StatusBadRequest, "invalid parameter") } - fillCache := func(ctx groupcache.Context, key string, dest groupcache.Sink) error { + fillCache := func(ctx context.Context, key string, dest groupcache.Sink) error { // cache miss, fetch from DB // key is in the form + parts := strings.Split(key, "+") - schemeId := parts[0] + schemeID := parts[0] downloadStart, _ := strconv.Atoi(parts[1]) - words, err := getWords(schemeId, downloadStart) + + words, err := getWords(schemeID, downloadStart) if err != nil { return err } - response := downloadResponse{Count: len(words), Words: words, standardResponse: newStandardResponse("")} + response := downloadResponse{Count: len(words), Words: words, standardResponse: newStandardResponse()} + b, err := json.Marshal(response) if err != nil { return err @@ -218,17 +212,21 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) { // gzipping the response so that it can be served directly var gb bytes.Buffer gWriter := gzip.NewWriter(&gb) - defer gWriter.Close() - gWriter.Write(b) - gWriter.Flush() + + defer func() { _ = gWriter.Close() }() + + _, _ = gWriter.Write(b) + _ = gWriter.Flush() if len(words) < downloadPageSize { - varnamCtx := ctx.(*varnamCacheContext) + varnamCtx, _ := ctx.(*varnamCacheContext) varnamCtx.Data = gb.Bytes() + return errCacheSkipped } - dest.SetBytes(gb.Bytes()) + _ = dest.SetBytes(gb.Bytes()) + return nil } @@ -245,57 +243,415 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) { } }) - cacheGroup := cacheGroups[params.langCode] + cacheGroup := cacheGroups[langCode] ctx := varnamCacheContext{} + var data []byte - if err := cacheGroup.Get(&ctx, fmt.Sprintf("%s+%d", params.langCode, params.downloadStart), groupcache.AllocatingByteSliceSink(&data)); err != nil { + if err := cacheGroup.Get(&ctx, fmt.Sprintf("%s+%d", langCode, start), groupcache.AllocatingByteSliceSink(&data)); err != nil { if err == errCacheSkipped { - renderGzippedJSON(w, ctx.Data) - return + c.Response().Header().Set("Content-Encoding", "gzip") + return c.Blob(http.StatusOK, "application/json; charset=utf-8", ctx.Data) } - renderError(w, err) - return + app.log.Printf("error in fetching deta from cache: %s, err: %s", langCode, err.Error()) + + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } - renderGzippedJSON(w, data) + c.Response().Header().Set("Content-Encoding", "gzip") + + return c.Blob(http.StatusOK, "application/json; charset=utf-8", data) } -func languagesHandler(w http.ResponseWriter, r *http.Request) { - renderJSON(w, schemeDetails) +func handleLanguages(c echo.Context) error { + return c.JSON(http.StatusOK, schemeDetails) } -func learnHandler(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) - var args Args - if e := decoder.Decode(&args); e != nil { - renderError(w, e) - return +func handleLanguageDownload(c echo.Context) error { + var ( + langCode = c.Param("langCode") + ) + + filepath, err := getSchemeFilePath(langCode) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error: %s", err.Error())) + } + + return c.Attachment(filepath.(string), langCode+".vst") +} + +func handleLearn(c echo.Context) error { + var ( + a args + + app = c.Get("app").(*App) + ) + + if err := c.Bind(&a); err != nil { + app.log.Printf("error in binding request details for learn, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) } - ch, ok := learnChannels[args.LangCode] + ch, ok := learnChannels[a.LangCode] if !ok { - renderError(w, errors.New("Unable to find language")) - return + app.log.Printf("unknown language requested to learn: %s", a.LangCode) + return echo.NewHTTPError(http.StatusBadRequest, "unable to find language") } - go func(word string) { ch <- word }(args.Word) - renderJSON(w, "success") + + go func(word string) { ch <- word }(a.Text) + + return c.JSON(http.StatusOK, "success") } -func toggleDownloadEnabledStatus(w http.ResponseWriter, r *http.Request, status bool) { - params := parseParams(r) - err := varnamdConfig.setDownloadStatus(params.langCode, status) +func handleLearnFileUpload(c echo.Context) error { + var ( + app = c.Get("app").(*App) + langCode = c.Param("langCode") + ) + + // Multipart form + form, err := c.MultipartForm() if err != nil { - renderError(w, err) - } else { - renderJSON(w, newStandardResponse("")) + app.log.Printf("failed to read form from request, language: %s, error: %s", langCode, err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, "request data not found") + } + + files, ok := form.File["files"] + if !ok { + app.log.Printf("files not found, language: %s", langCode) + return echo.NewHTTPError(http.StatusBadRequest, "no files were uploaded") + } + + if _, ok := learnChannels[langCode]; !ok { + app.log.Printf("learn file upload error: unknown language requested to learn: %s", langCode) + return echo.NewHTTPError(http.StatusBadRequest, "unable to find language to train") + } + + // Copy files first + for _, file := range files { + // Source + src, err := file.Open() + if err != nil { + app.log.Printf("learn file upload error, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + // Destination + tempDir, err := ioutil.TempDir(os.TempDir(), "varnamd") + if err != nil { + app.log.Printf("learn file upload error, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + dst, err := os.Create(filepath.Join(tempDir, file.Filename)) + if err != nil { + app.log.Printf("learn file upload error, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + // Copy + if _, err = io.Copy(dst, src); err != nil { + app.log.Printf("learn file upload error, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + // Explicitely closing resources. + _ = dst.Close() + _ = src.Close() + + learnWordsFromFile(c, langCode, dst.Name(), true) } + + return c.JSON(http.StatusOK, "success") } -func enableDownload(w http.ResponseWriter, r *http.Request) { - toggleDownloadEnabledStatus(w, r, true) +func handleTrain(c echo.Context) error { + var ( + targs trainArgs + app = c.Get("app").(*App) + langCode = c.Param("langCode") + ) + + c.Request().Header.Set("Content-Type", "application/json") + + if err := c.Bind(&targs); err != nil { + app.log.Printf("error reading request, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } + + ch, ok := trainChannel[langCode] + if !ok { + app.log.Printf("unknown language requested to learn: %s", langCode) + return echo.NewHTTPError(http.StatusBadRequest, "unable to find language to train") + } + + go func(args trainArgs) { ch <- args }(targs) + + _, _ = app.cache.Delete(langCode, targs.Pattern) + + return c.JSON(200, "Word Trained") } -func disableDownload(w http.ResponseWriter, r *http.Request) { - toggleDownloadEnabledStatus(w, r, false) +// handleTrainBulk is an endpoint for training words in the following format. +// {[ +// {word, patterns: []}, +// {word, patterns: []}, +// {word, patterns: []}, +// {word, patterns: []} +// ]} +// It will covert each bulk arg to trainArg and will send to train channel. +// Training is happened at listenForWords method. +func handleTrainBulk(c echo.Context) error { + var ( + bulkArgs []trainBulkArgs + app = c.Get("app").(*App) + langCode = c.Param("langCode") + ) + + if err := c.Bind(&bulkArgs); err != nil { + app.log.Printf("error reading request, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } + + ch, ok := trainChannel[langCode] + if !ok { + app.log.Printf("unknown language requested to learn: %s", langCode) + return echo.NewHTTPError(http.StatusBadRequest, "unable to find language to train") + } + + for _, v := range bulkArgs { + for _, p := range v.Pattern { + go func(args trainArgs) { + ch <- args + }(trainArgs{ + Pattern: p, + Word: v.Word, + }) + } + } + + return c.JSON(200, "Words Trained") +} + +// Delete a word +func handleDelete(c echo.Context) error { + var ( + a args + + app = c.Get("app").(*App) + ) + + c.Request().Header.Set("Content-Type", "application/json") + + if err := c.Bind(&a); err != nil { + app.log.Printf("error in binding request details for delete, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } + + if _, err := deleteWord(a.LangCode, a.Text); err != nil { + app.log.Printf("error deleting word, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error: %s", err.Error())) + } + + app.cache.Clear() + + return c.JSON(http.StatusOK, "success") +} + +func toggleDownloadEnabledStatus(langCode string, status bool) (interface{}, error) { + if err := varnamdConfig.setDownloadStatus(langCode, status); err != nil { + return nil, err + } + + return newStandardResponse(), nil +} + +func handleEnableDownload(c echo.Context) error { + var ( + langCode = c.Param("langCode") + + app = c.Get("app").(*App) + ) + + data, err := toggleDownloadEnabledStatus(langCode, true) + if err != nil { + app.log.Printf("failed to toggle download enable, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } + + return c.JSON(http.StatusOK, data) +} + +func handleDisableDownload(c echo.Context) error { + var ( + langCode = c.Param("langCode") + app = c.Get("app").(*App) + ) + + data, err := toggleDownloadEnabledStatus(langCode, false) + if err != nil { + app.log.Printf("failed to disable download, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } + + return c.JSON(http.StatusOK, data) +} + +// handleIndex is the root handler that renders the Javascript frontend. +func handleIndex(c echo.Context) error { + app, _ := c.Get("app").(*App) + + b, err := app.fs.Read("/index.html") + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + c.Response().Header().Set("Content-Type", "text/html") + + return c.String(http.StatusOK, string(b)) +} + +func handlePacks(c echo.Context) error { + var ( + langCode = c.Param("langCode") + app = c.Get("app").(*App) + ) + + if langCode != "" { + pack, err := getPacksLangInfo(langCode) + if err != nil { + statusCode := http.StatusBadRequest + if err.Error() == "No packs found" { + statusCode = http.StatusNotFound + } + + app.log.Printf("error reading packs, err: %s", err.Error()) + return echo.NewHTTPError(statusCode, err.Error()) + } + return c.JSON(http.StatusOK, pack) + } + + packs, err := getPacksInfo() + if err != nil { + app.log.Printf("error reading packs, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + return c.JSON(http.StatusOK, packs) +} + +func handlePackInfo(c echo.Context) error { + var ( + langCode = c.Param("langCode") + packIdentifier = c.Param("packIdentifier") + ) + + pack, err := getPackInfo(langCode, packIdentifier) + if err != nil { + statusCode := http.StatusBadRequest + if err.Error() == "Pack not found" { + statusCode = http.StatusNotFound + } + + return echo.NewHTTPError(statusCode, err.Error()) + } + + return c.JSON(http.StatusOK, pack) +} + +func handlePackVersionInfo(c echo.Context) error { + var ( + langCode = c.Param("langCode") + packIdentifier = c.Param("packIdentifier") + packVersionIdentifier = c.Param("packVersionIdentifier") + ) + + pack, err := getPackVersionInfo(langCode, packIdentifier, packVersionIdentifier) + if err != nil { + statusCode := http.StatusBadRequest + if err.Error() == "Pack version not found" { + statusCode = http.StatusNotFound + } + + return echo.NewHTTPError(statusCode, err.Error()) + } + + return c.JSON(http.StatusOK, pack) +} + +func handlePacksDownload(c echo.Context) error { + var ( + langCode = c.Param("langCode") + packIdentifier = c.Param("packIdentifier") + packVersionIdentifier = c.Param("packVersionIdentifier") + ) + + if _, err := getPackVersionInfo(langCode, packIdentifier, packVersionIdentifier); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + + packFilePath, err := getPackFilePath(langCode, packIdentifier, packVersionIdentifier) + + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + packFileGzipPath := path.Join(packFilePath + ".gzip") + + if !fileExists(packFileGzipPath) { + // compress into gzip + packFileBytes, err := ioutil.ReadFile(packFilePath) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + var gb bytes.Buffer + w := gzip.NewWriter(&gb) + w.Write(packFileBytes) + w.Close() + + err = ioutil.WriteFile(packFileGzipPath, gb.Bytes(), 0644) + + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + } + + return c.Attachment(packFileGzipPath, packVersionIdentifier) +} + +// varnamd Admin can download packs from upstream +// This is an internal function +func handlePackDownloadRequest(c echo.Context) error { + var ( + args packDownloadArgs + app = c.Get("app").(*App) + err error + downloadResult packDownload + ) + + if err := c.Bind(&args); err != nil { + app.log.Printf("error reading request, err: %s", err.Error()) + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error getting metadata. message: %s", err.Error())) + } + + downloadResult, err = downloadPackFile(args.LangCode, args.Identifier, args.Version) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("error downloading pack: %s", err.Error())) + } + + // Learn from pack file and don't remove it + err = importLearningsFromFile(c, args.LangCode, downloadResult.FilePath, false) + + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("Error importing from '%s'\n", err.Error())) + } + + // Add pack.json with the installed pack versions + err = updatePacksInfo(args.LangCode, downloadResult.Pack, downloadResult.Version) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + return c.JSON(http.StatusOK, "success") } diff --git a/init.go b/init.go new file mode 100644 index 0000000..0028f3c --- /dev/null +++ b/init.go @@ -0,0 +1,110 @@ +package main + +import ( + "fmt" + "log" + "os" + "path" + "runtime" + "strings" + "time" + + "github.com/knadh/stuffbin" +) + +func initConfig(cfg appConfig) *config { + toDownload := make(map[string]bool) + schemes := strings.Split(cfg.DownloadEnabledSchemes, ",") + + for _, scheme := range schemes { + s := strings.TrimSpace(scheme) + + if s != "" { + if !isValidSchemeIdentifier(s) { + panic(fmt.Sprintf("%s is not a valid libvarnam supported scheme", s)) + } + + toDownload[s] = true + } + } + + return &config{upstream: cfg.UpstreamURL, schemesToDownload: toDownload, + syncInterval: time.Duration(cfg.SyncInterval)} +} + +func (c *config) setDownloadStatus(langCode string, status bool) error { + if !isValidSchemeIdentifier(langCode) { + return fmt.Errorf("%s is not a valid libvarnam supported scheme", langCode) + } + + c.schemesToDownload[langCode] = status + + if status { + // when varnamd was started without any langcodes to sync, the dispatcher won't be running + // in that case, we need to start the dispatcher since we have a new lang code to download now + startSyncDispatcher() + } + + return nil +} + +func getConfigDir() string { + if runtime.GOOS == "windows" { + return path.Join(os.Getenv("localappdata"), ".varnamd") + } + + return path.Join(os.Getenv("HOME"), ".varnamd") +} + +// initVFS initializes the stuffbin virtual FileSystem to provide +// access to bunded static assets to the app. +func initVFS() (stuffbin.FileSystem, error) { + files := []string{ + "ui:/", + } + + binPath, err := os.Executable() + if err != nil { + return nil, err + } + + fs, err := stuffbin.UnStuff(binPath) + if err != nil { + log.Printf("unable to initialize embedded filesystem: %v", err) + log.Printf("using local filesystem") + + fs, err = stuffbin.NewLocalFS("/", files...) + if err != nil { + return nil, err + } + } + + return fs, nil +} + +func initAppConfig() (appConfig, error) { + var config appConfig + // Read configuration using Koanf. + if err := kf.Unmarshal("app", &config); err != nil { + return config, err + } + + // If address is empty run on localhost port 8080. + if config.Address == "" { + config.Address = fmt.Sprintf("%s:%d", kf.String("host"), kf.Int("p")) + } + + if config.EnableSSL && (config.CertFilePath == "" || config.KeyFilePath == "") { + config.EnableSSL = false + } + + if config.SyncInterval < 1*time.Second { + config.SyncInterval = 30 * time.Second + } + + if config.UpstreamURL == "" { + config.UpstreamURL = "https://api.varnamproject.com" + } + + return config, nil +} diff --git a/libvarnam/libvarnam.go b/libvarnam/libvarnam.go new file mode 100644 index 0000000..d81b63d --- /dev/null +++ b/libvarnam/libvarnam.go @@ -0,0 +1,232 @@ +package libvarnam + +// #cgo pkg-config: varnam +// #include +import "C" +import "fmt" + +// Varnam app binding. +type Varnam struct { + handle *C.varnam +} + +// VarnamError satisfies error interface. +type VarnamError struct { + errorCode int + message string +} + +// SchemeDetails returns language and other changes. +type SchemeDetails struct { + LangCode string + Identifier string + DisplayName string + Author string + CompiledDate string + IsStable bool +} + +// CorpusDetails returns corpus details. +type CorpusDetails struct { + WordsCount int `json:"wordsCount"` +} + +// LearnStatus . +type LearnStatus struct { + TotalWords int + Failed int +} + +func (e *VarnamError) Error() string { + return e.message +} + +// GetSuggestionsFilePath returns suggestions. +func (v *Varnam) GetSuggestionsFilePath() string { + return C.GoString(C.varnam_get_suggestions_file(v.handle)) +} + +// GetSchemeFilePath returns the scheme file (.vst) +func (v *Varnam) GetSchemeFilePath() string { + return C.GoString(C.varnam_get_scheme_file(v.handle)) +} + +// GetCorpusDetails will return corpus details. +func (v *Varnam) GetCorpusDetails() (*CorpusDetails, error) { + var details *C.vcorpus_details + + rc := C.varnam_get_corpus_details(v.handle, &details) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return nil, &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return &CorpusDetails{WordsCount: int(details.wordsCount)}, nil +} + +// Transliterate given string to corresponding language. +func (v *Varnam) Transliterate(text string) ([]string, error) { + var va *C.varray + rc := C.varnam_transliterate(v.handle, C.CString(text), &va) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return nil, &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + var ( + i C.int + words []string + ) + + for i = 0; i < C.varray_length(va); i++ { + word := (*C.vword)(C.varray_get(va, i)) + words = append(words, C.GoString(word.text)) + } + + return words, nil +} + +// ReverseTransliterate given string. +func (v *Varnam) ReverseTransliterate(text string) (string, error) { + var output *C.char + rc := C.varnam_reverse_transliterate(v.handle, C.CString(text), &output) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return "", &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return C.GoString(output), nil +} + +// LearnFromFile learns from file from the given filepath. +func (v *Varnam) LearnFromFile(filePath string) (*LearnStatus, error) { + var status C.vlearn_status + rc := C.varnam_learn_from_file(v.handle, C.CString(filePath), &status, nil, nil) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return nil, &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return &LearnStatus{TotalWords: int(status.total_words), Failed: int(status.failed)}, nil +} + +// ImportFromFile Import learnigns from file (varnam exported file) +func (v *Varnam) ImportFromFile(filePath string) error { + rc := C.varnam_import_learnings_from_file(v.handle, C.CString(filePath)) + + if rc != C.VARNAM_SUCCESS { + errorCode := (int)(rc) + + return &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return nil +} + +// Init initializes varnam bindings. +func Init(schemeIdentifier string) (*Varnam, error) { + var ( + v *C.varnam + msg *C.char + ) + + rc := C.varnam_init_from_id(C.CString(schemeIdentifier), &v, &msg) + + if rc != C.VARNAM_SUCCESS { + return nil, &VarnamError{errorCode: (int)(rc), message: C.GoString(msg)} + } + + return &Varnam{handle: v}, nil +} + +// GetAllSchemeDetails returns all scheme related details. +func GetAllSchemeDetails() []*SchemeDetails { + allHandles := C.varnam_get_all_handles() + + if allHandles == nil { + return []*SchemeDetails{} + } + + var schemeDetails []*SchemeDetails + + length := int(C.varray_length(allHandles)) + + for i := 0; i < length; i++ { + var detail *C.vscheme_details + + handle := (*C.varnam)(C.varray_get(allHandles, C.int(i))) + rc := C.varnam_get_scheme_details(handle, &detail) + + if rc != C.VARNAM_SUCCESS { + return []*SchemeDetails{} + } + + schemeDetails = append(schemeDetails, &SchemeDetails{ + LangCode: C.GoString(detail.langCode), Identifier: C.GoString(detail.identifier), + DisplayName: C.GoString(detail.displayName), Author: C.GoString(detail.author), + CompiledDate: C.GoString(detail.compiledDate), IsStable: detail.isStable > 0}) + + C.varnam_destroy(handle) + } + + return schemeDetails +} + +// Learn from given input text. +func (v *Varnam) Learn(text string) error { + rc := C.varnam_learn(v.handle, C.CString(text)) + + if rc != 0 { + errorCode := (int)(rc) + + return &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return nil +} + +// DeleteWord from given input text. +func (v *Varnam) DeleteWord(text string) error { + rc := C.varnam_delete_word(v.handle, C.CString(text)) + + if rc != 0 { + errorCode := (int)(rc) + + return &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return nil +} + +func (v *Varnam) getVarnamError(errorCode int) string { + errormessage := C.varnam_get_last_error(v.handle) + varnamErrorMsg := C.GoString(errormessage) + + return fmt.Sprintf("%d:%s", errorCode, varnamErrorMsg) +} + +// Destroy closes handle. +func (v *Varnam) Destroy() { + C.varnam_destroy(v.handle) +} + +// Train methods trains with the word and pattern,eg: pattern=chrome,word=ക്രോം +func (v *Varnam) Train(pattern, word string) error { + rc := C.varnam_train(v.handle, C.CString(pattern), C.CString(word)) + + if rc != 0 { + errorCode := (int)(rc) + + return &VarnamError{errorCode: errorCode, message: v.getVarnamError(errorCode)} + } + + return nil +} diff --git a/main.go b/main.go index 9318a68..e39a302 100644 --- a/main.go +++ b/main.go @@ -1,119 +1,112 @@ package main import ( - "errors" - "flag" "fmt" "log" "os" - "path" "runtime" - "strings" "time" + + flag "github.com/spf13/pflag" + + "github.com/knadh/koanf" + "github.com/knadh/koanf/parsers/toml" + "github.com/knadh/koanf/providers/file" + "github.com/knadh/koanf/providers/posflag" + "github.com/knadh/stuffbin" +) + +type userConfig map[string]string + +const ( + downloadPageSize = 100 ) var ( - port int - version bool - maxHandleCount int - host string - uiDir string - enableInternalApis bool // internal APIs are not exposed to public - enableSSL bool - certFilePath string - keyFilePath string - logToFile bool // logs will be written to file when true - varnamdConfig *config // config instance used across the application - startedAt time.Time - downloadEnabledSchemes string // comma separated list of scheme identifier for which download will be performed - syncIntervalInSecs int - upstreamURL string - syncDispatcherRunning bool + kf = koanf.New(".") + + varnamdConfig *config // config instance used across the application + syncDispatcherRunning bool + startedAt time.Time + buildVersion string + buildDate string + maxHandleCount int + authEnabled bool + + // User accounts are stored here. + users map[string]userConfig ) +type appConfig struct { + Address string + + EnableInternalApis bool `koanf:"enable-internal-api"` // internal APIs are not exposed to public + EnableSSL bool `koanf:"enable-ssl"` + CertFilePath string `koanf:"cert-path"` + KeyFilePath string `koanf:"key-file-path"` + + DownloadEnabledSchemes string `koanf:"download-enabled-schemes"` + SyncInterval time.Duration `koanf:"sync-interval"` + UpstreamURL string `koanf:"upstream-url"` +} + +// App is a singleton to share across handlers. +type App struct { + cache Cache + log *log.Logger + fs stuffbin.FileSystem +} + // varnamd configurations // this is populated from various command line flags type config struct { - upstream string - schemesToDownload map[string]bool - syncIntervalInSecs time.Duration + upstream string + schemesToDownload map[string]bool + syncInterval time.Duration } -func initConfig() *config { - toDownload := make(map[string]bool) - schemes := strings.Split(downloadEnabledSchemes, ",") - for _, scheme := range schemes { - s := strings.TrimSpace(scheme) - if s != "" { - if !isValidSchemeIdentifier(s) { - panic(fmt.Sprintf("%s is not a valid libvarnam supported scheme", s)) - } - toDownload[s] = true - } - } - - return &config{upstream: upstreamURL, schemesToDownload: toDownload, - syncIntervalInSecs: time.Duration(syncIntervalInSecs)} -} +func init() { + // Set max processors to number of CPUs to maximize performance. + runtime.GOMAXPROCS(runtime.NumCPU()) -func (c *config) setDownloadStatus(langCode string, status bool) error { - if !isValidSchemeIdentifier(langCode) { - return errors.New(fmt.Sprintf("%s is not a valid libvarnam supported scheme", langCode)) + // Setup flags to read from user to start the application. + // Initialize 'config' flagset. + flagSet := flag.NewFlagSet("config", flag.ContinueOnError) + flagSet.Usage = func() { + log.Fatal(flagSet.FlagUsages()) } - c.schemesToDownload[langCode] = status - if status { - // when varnamd was started without any langcodes to sync, the dispatcher won't be running - // in that case, we need to start the dispatcher since we have a new lang code to download now - startSyncDispatcher() - } + // Create config flag to read 'config.toml' from user. + flagSet.String("config", "config.toml", "Path to the TOML configuration file") - return nil -} + flagSet.Int("p", 8080, "Run daemon in specified port") + flagSet.String("host", "", "Host for the varnam daemon server") -func getConfigDir() string { - if runtime.GOOS == "windows" { - return path.Join(os.Getenv("localappdata"), ".varnamd") - } else { - return path.Join(os.Getenv("HOME"), ".varnamd") - } -} + // Create flag for version check. + flagSet.Bool("version", false, "Current version of the build") -func getLogsDir() string { - d := getConfigDir() - logsDir := path.Join(d, "logs") - err := os.MkdirAll(logsDir, 0777) + err := flagSet.Parse(os.Args[1:]) if err != nil { - panic(err) + log.Fatalf("error parsing flags: %v", err) } - return logsDir -} + // Load commandline params user given. + if err = kf.Load(posflag.Provider(flagSet, ".", kf), nil); err != nil { + log.Fatal(err.Error()) + } -func redirectLogToFile() { - year, month, day := time.Now().Date() - logfile := path.Join(getLogsDir(), fmt.Sprintf("%d-%d-%d.log", year, month, day)) - f, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - panic(err) + // Handle --version flag. Print build version, build date and die. + if kf.Bool("version") { + fmt.Printf("Commit: %v\nBuild: %v\n", buildVersion, buildDate) + os.Exit(0) } - log.SetOutput(f) -} -func init() { - flag.IntVar(&port, "p", 8080, "Run daemon in specified port") - flag.IntVar(&maxHandleCount, "max-handle-count", 10, "Maximum number of handles can be opened for each language") - flag.StringVar(&host, "host", "", "Host for the varnam daemon server") - flag.StringVar(&uiDir, "ui", "", "UI directory path") - flag.BoolVar(&enableInternalApis, "enable-internal-apis", false, "Enable internal APIs") - flag.BoolVar(&enableSSL, "enable-ssl", false, "Enables SSL") - flag.StringVar(&certFilePath, "cert-file-path", "", "Certificate file path") - flag.StringVar(&keyFilePath, "key-file-path", "", "Key file path") - flag.StringVar(&upstreamURL, "upstream", "https://api.varnamproject.com", "Provide an upstream server") - flag.StringVar(&downloadEnabledSchemes, "enable-download", "", "Comma separated language identifier for which varnamd will download words from upstream") - flag.IntVar(&syncIntervalInSecs, "sync-interval", 30, "Download interval in seconds") - flag.BoolVar(&logToFile, "log-to-file", true, "If true, logs will be written to a file") - flag.BoolVar(&version, "version", false, "Print the version and exit") + // Load the config file. + log.Printf("reading config: %s", kf.String("config")) + + if err = kf.Load(file.Provider(kf.String("config")), toml.Parser()); err != nil { + log.Printf("error reading config: %v", err) + } } func syncRequired() bool { @@ -123,28 +116,48 @@ func syncRequired() bool { // Starts the sync process only if it is not running func startSyncDispatcher() { if syncRequired() && !syncDispatcherRunning { - sync := newSyncDispatcher(varnamdConfig.syncIntervalInSecs * time.Second) + sync := newSyncDispatcher(varnamdConfig.syncInterval / time.Second) sync.start() sync.runNow() // run one round of sync immediatly rather than waiting for the next interval to occur + syncDispatcherRunning = true } } func main() { - runtime.GOMAXPROCS(runtime.NumCPU()) - flag.Parse() - varnamdConfig = initConfig() - startedAt = time.Now() - if version { - fmt.Println(varnamdVersion) - os.Exit(0) + config, err := initAppConfig() + if err != nil { + log.Fatal(err.Error()) } - if logToFile { - redirectLogToFile() + + maxHandleCount = kf.Int("app.max-handles") + if maxHandleCount <= 0 { + maxHandleCount = 10 + } + + authEnabled = kf.Bool("app.accounts-enabled") + if authEnabled { + if err = kf.Unmarshal("users", &users); err != nil { + log.Fatal(err.Error()) + } } - log.Printf("varnamd %s", varnamdVersion) + varnamdConfig = initConfig(config) + startedAt = time.Now() + + log.Printf("varnamd %s-%s", buildVersion, buildDate) + + fs, err := initVFS() + if err != nil { + log.Fatal(err.Error()) + } + + app := &App{ + cache: NewMemCache(), + log: log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile), + fs: fs, + } startSyncDispatcher() - startDaemon() + startDaemon(app, config) } diff --git a/packs.go b/packs.go new file mode 100644 index 0000000..a59de2a --- /dev/null +++ b/packs.go @@ -0,0 +1,319 @@ +package main + +import ( + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path" +) + +// PackVersion Details of a pack version +type PackVersion struct { + Identifier string `json:"identifier"` // Pack identifier is unique across all language packs. Example: ml-basic-1 + Version int `json:"version"` + Description string `json:"description"` + Size int `json:"size"` +} + +// Pack Details of a pack +type Pack struct { + Identifier string `json:"identifier"` + Name string `json:"name"` + Description string `json:"description"` + LangCode string `json:"lang"` + Versions []PackVersion `json:"versions"` +} + +type packDownload struct { + Pack *Pack + Version *PackVersion + FilePath string +} + +var packsInfoCached []Pack + +func fileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} + +// After a new pack download from upstream, update packs.json with installed packs +func updatePacksInfo(langCode string, pack *Pack, packVersion *PackVersion) error { + packs, err := getPacksInfo() + if err != nil { + return err + } + + var ( + existingPack *Pack = nil + ) + + for _, packR := range packs { + if packR.Identifier == pack.Identifier { + existingPack = &packR + break + } + } + + if existingPack == nil { + // will have one element + pack.Versions = []PackVersion{*packVersion} + } else { + // Append new pack version + existingPack.Versions = append(existingPack.Versions, *packVersion) + + pack = existingPack + } + + // Save pack.json + packInfoPath := path.Join(getPacksDir(), pack.LangCode, pack.Identifier, "pack.json") + file, err := json.MarshalIndent(pack, "", " ") + if err != nil { + return err + } + + err = ioutil.WriteFile(packInfoPath, file, 0644) + if err != nil { + return err + } + + packsInfoCached = nil + + return nil +} + +// Download pack from upstream +func downloadPackFile(langCode, packIdentifier, packVersionIdentifier string) (packDownload, error) { + var ( + pack *Pack + packVersion *PackVersion = nil + ) + + packInstalled, _ := getPackVersionInfo(langCode, packIdentifier, packVersionIdentifier) + if packInstalled != nil { + return packDownload{}, fmt.Errorf("Pack already installed") + } + + packInfoURL := fmt.Sprintf("%s/packs/%s/%s", varnamdConfig.upstream, langCode, packIdentifier) + + resp, err := http.Get(packInfoURL) + if err != nil { + return packDownload{}, err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return packDownload{}, err + } + + return packDownload{}, fmt.Errorf(string(respData)) + } + + if err = json.NewDecoder(resp.Body).Decode(&pack); err != nil { + err := fmt.Errorf("Parsing packs JSON failed, err: %s", err.Error()) + return packDownload{}, err + } + + for _, version := range pack.Versions { + if version.Identifier == packVersionIdentifier { + packVersion = &version + break + } + } + + if packVersion == nil { + return packDownload{}, fmt.Errorf("Pack version not found") + } + + fileURL := fmt.Sprintf("%s/packs/%s/%s/%s/download", varnamdConfig.upstream, langCode, packIdentifier, packVersionIdentifier) + fileDir := path.Join(getPacksDir(), langCode, packIdentifier) + filePath := path.Join(fileDir, packVersionIdentifier) + + if !fileExists(fileDir) { + os.MkdirAll(fileDir, 0755) + } + + resp, err = http.Get(fileURL) + if err != nil { + return packDownload{}, err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return packDownload{}, err + } + + return packDownload{}, fmt.Errorf(string(respData)) + } + + fz, err := gzip.NewReader(resp.Body) + if err != nil { + return packDownload{}, err + } + defer fz.Close() + + out, err := os.Create(filePath) + if err != nil { + return packDownload{}, err + } + defer out.Close() + + // Write the gzip decoded content to file + _, err = io.Copy(out, fz) + + if err != nil { + return packDownload{}, err + } + + return packDownload{Pack: pack, Version: packVersion, FilePath: filePath}, nil +} + +func getPackFilePath(langCode, packIdentifier, packVersionIdentifier string) (string, error) { + if _, err := getPackVersionInfo(langCode, packIdentifier, packVersionIdentifier); err != nil { + return "", err + } + + // Example: .varnamd/ml/ml-basic/ml-basic-1.vlf + packFilePath := path.Join(getPacksDir(), langCode, packIdentifier, packVersionIdentifier) + ".vlf" + + if !fileExists(packFilePath) { + return "", errors.New("Pack file not found") + } + + return packFilePath, nil +} + +func getPackVersionInfo(langCode string, packIdentifier string, packVersionIdentifier string) (*PackVersion, error) { + pack, err := getPackInfo(langCode, packIdentifier) + + if err != nil { + return nil, err + } + + var packVersion *PackVersion = nil + + for _, version := range pack.Versions { + if version.Identifier == packVersionIdentifier { + packVersion = &version + break + } + } + + if packVersion == nil { + return nil, fmt.Errorf("Pack version not found") + } + + return packVersion, nil +} + +func getPackInfo(langCode string, packIdentifier string) (*Pack, error) { + packs, err := getPacksLangInfo(langCode) + + if err != nil { + return nil, err + } + + for _, pack := range packs { + if pack.Identifier == packIdentifier { + return &pack, nil + } + } + + return nil, fmt.Errorf("Pack not found") +} + +// Get packs by language +func getPacksLangInfo(langCode string) ([]Pack, error) { + packs, err := getPacksInfo() + + if err != nil { + return nil, err + } + + var langPacks []Pack + + for _, pack := range packs { + if pack.LangCode == langCode { + langPacks = append(langPacks, pack) + } + } + + if len(langPacks) == 0 { + return nil, fmt.Errorf("No packs found") + } + + return langPacks, nil +} + +func getPacksInfo() ([]Pack, error) { + if err := createPacksDir(); err != nil { + err := fmt.Errorf("Failed to create packs directory, err: %s", err.Error()) + return nil, err + } + + if packsInfoCached != nil { + return packsInfoCached, nil + } + + var packsInfo []Pack + + files, err := ioutil.ReadDir(getPacksDir()) + if err != nil { + return nil, err + } + + for _, langFolder := range files { + langFolderPath := path.Join(getPacksDir(), langFolder.Name()) + if langFolder.IsDir() { + // inside ml + langFolderFiles, err := ioutil.ReadDir(langFolderPath) + + if err != nil { + return nil, err + } + + for _, packFolder := range langFolderFiles { + if packFolder.IsDir() { + packInfoPath := path.Join(langFolderPath, packFolder.Name(), "pack.json") + if fileExists(packInfoPath) { + var packInfo Pack + packsFile, _ := ioutil.ReadFile(packInfoPath) + + if err := json.Unmarshal(packsFile, &packInfo); err != nil { + err := fmt.Errorf("Parsing packs JSON failed, err: %s", err.Error()) + return nil, err + } + + packsInfo = append(packsInfo, packInfo) + } + } + } + } + } + + packsInfoCached = packsInfo + + return packsInfo, nil +} + +func createPacksDir() error { + packsDir := getPacksDir() + return os.MkdirAll(packsDir, 0750) +} + +func getPacksDir() string { + configDir := getConfigDir() + return path.Join(configDir, "packs") +} diff --git a/server.go b/server.go index 6156b6e..7e50c76 100644 --- a/server.go +++ b/server.go @@ -1,53 +1,128 @@ package main import ( - "fmt" - "log" + "encoding/base64" "net/http" - "os" + "strings" - "github.com/gorilla/mux" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" ) -func startDaemon() { +func startDaemon(app *App, cfg appConfig) { initLanguageChannels() - initLearnChannels() - r := mux.NewRouter() - r.HandleFunc("/tl/{langCode}/{word}", transliterationHandler).Methods("GET") - r.HandleFunc("/rtl/{langCode}/{word}", reverseTransliterationHandler).Methods("GET") - r.HandleFunc("/meta/{langCode}", metadataHandler).Methods("GET") - r.HandleFunc("/download/{langCode}/{downloadStart}", downloadHandler).Methods("GET") - r.HandleFunc("/learn", learnHandler).Methods("POST") - r.HandleFunc("/languages", languagesHandler).Methods("GET") - r.HandleFunc("/status", statusHandler).Methods("GET") - if enableInternalApis { - r.HandleFunc("/sync/download/{langCode}/enable", enableDownload).Methods("POST") - r.HandleFunc("/sync/download/{langCode}/disable", disableDownload).Methods("POST") - } + app.initChannels() + + e := initHandlers(app, cfg.EnableInternalApis) - addUI(r) + app.log.Printf("🚀 starting varnamd\nListening on %s", cfg.Address) - address := fmt.Sprintf("%s:%d", host, port) - log.Printf("Listening on %s", address) - if enableSSL { - if err := http.ListenAndServeTLS(address, certFilePath, keyFilePath, recoverHandler(corsHandler(r))); err != nil { - log.Fatalln(err) + if cfg.EnableSSL { + if err := e.StartTLS(cfg.Address, cfg.CertFilePath, cfg.KeyFilePath); err != nil { + app.log.Fatal(err) } } else { - if err := http.ListenAndServe(address, recoverHandler(corsHandler(r))); err != nil { - log.Fatalln(err) + if err := e.Start(cfg.Address); err != nil { + app.log.Fatal(err) } } } -func addUI(r *mux.Router) { - if uiDir == "" { - return - } +func initHandlers(app *App, enableInternalApis bool) *echo.Echo { + e := echo.New() + e.GET("/tl/:langCode/:word", handleTransliteration) + e.GET("/rtl/:langCode/:word", handleReverseTransliteration) + e.GET("/meta/:langCode:", handleMetadata) + e.GET("/download/:langCode/:downloadStart", handleDownload) + e.GET("/languages", handleLanguages) + e.GET("/languages/:langCode/download", handleLanguageDownload) + e.GET("/packs", handlePacks) + e.GET("/packs/:langCode", handlePacks) + e.GET("/packs/:langCode/:packIdentifier", handlePackInfo) + e.GET("/packs/:langCode/:packIdentifier/:packVersionIdentifier", handlePackVersionInfo) + e.GET("/packs/:langCode/:packIdentifier/:packVersionIdentifier/download", handlePacksDownload) + e.GET("/status", handleStatus) + + e.GET("/", handleIndex) + + e.GET("/*", echo.WrapHandler(app.fs.FileServer())) - if _, err := os.Stat(uiDir); err != nil { - log.Fatalln("UI path doesnot exist", err) + if enableInternalApis { + e.POST("/sync/download/:langCode/enable", handleEnableDownload) + e.POST("/sync/download/:langCode/disable", handleDisableDownload) + + e.POST("/learn", authUser(handleLearn)) + e.POST("/learn/upload/:langCode", authUser(handleLearnFileUpload)) + e.POST("/train/:langCode", authUser(handleTrain)) + e.POST("/train/bulk/:langCode", authUser(handleTrainBulk)) + e.POST("/delete", authUser(handleDelete)) + e.POST("/packs/download", handlePackDownloadRequest) } - r.PathPrefix("/").Handler(http.FileServer(http.Dir(uiDir))) + e.Use(middleware.Recover()) + e.Use(middleware.Logger()) + + // Custom middleware to set sigleton to app's context. + e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + c.Set("app", app) + return next(c) + } + }) + + e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ + AllowOrigins: []string{"*"}, + AllowMethods: []string{http.MethodOptions}, + })) + + e.Use(middleware.SecureWithConfig(middleware.SecureConfig{ + XSSProtection: "", + ContentTypeNosniff: "", + XFrameOptions: "", + HSTSMaxAge: 3600, + // ContentSecurityPolicy: "default-src 'self'", + })) + + return e +} + +// authUser as a separate method to apply this middleware only for selected endpoints. +func authUser(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + var app = c.Get("app").(*App) + + if authEnabled { + auth := strings.Split(c.Request().Header.Get("Authorization"), " ") + if len(auth) < 2 { + app.log.Printf("authorization header not found") + return echo.NewHTTPError(http.StatusUnauthorized, "authorization header not found") + } + + if strings.ToLower(auth[0]) != "basic" { + app.log.Printf("authorization header not found") + return echo.NewHTTPError(http.StatusUnauthorized, "authorization details not found") + } + + creds, err := base64.StdEncoding.DecodeString(auth[1]) + if err != nil { + app.log.Printf("error decoding auth headers, error: %s", err.Error()) + return echo.NewHTTPError(http.StatusUnauthorized, "authorization failed, failed to decode authstring") + } + + authCreds := strings.Split(string(creds), ":") + + user, ok := users[strings.TrimSpace(authCreds[0])] + if !ok { + app.log.Printf("user not found") + return echo.NewHTTPError(http.StatusUnauthorized, "authorization failed, user not found") + } + + if user["password"] != strings.TrimSpace(authCreds[1]) { + app.log.Printf("password mismatch") + return echo.NewHTTPError(http.StatusUnauthorized, "authorization failed, password mismatch") + } + } + + return next(c) + } } diff --git a/sync.go b/sync.go index 52f0b53..5b9c4ae 100644 --- a/sync.go +++ b/sync.go @@ -8,11 +8,12 @@ import ( "net/http" "os" "path" + "path/filepath" "strconv" "strings" "time" - "github.com/varnamproject/libvarnam-golang" + "github.com/varnamproject/varnamd/libvarnam" ) type syncDispatcher struct { @@ -21,21 +22,20 @@ type syncDispatcher struct { ticker *time.Ticker } -func newSyncDispatcher(intervalInSeconds time.Duration) *syncDispatcher { - return &syncDispatcher{ticker: time.NewTicker(intervalInSeconds), force: make(chan bool), quit: make(chan struct{})} +func newSyncDispatcher(interval time.Duration) *syncDispatcher { + return &syncDispatcher{ticker: time.NewTicker(interval), force: make(chan bool), quit: make(chan struct{})} } func (s *syncDispatcher) start() { - err := createSyncMetadataDir() - if err != nil { + if err := createSyncMetadataDir(); err != nil { fmt.Printf("Failed to create sync metadata directory. Sync will be disabled.\nActual error: %s\n", err.Error()) return } + for s := range varnamdConfig.schemesToDownload { // download cache directory for each of the languages - err = createLearnQueueDir(s) - if err != nil { - fmt.Printf("Failed to create learn queue directory for '%s'. Sync will be disabled.\nActual error: %s\n", err.Error()) + if err := createLearnQueueDir(s); err != nil { + fmt.Printf("Failed to create learn queue directory for '%s'. Sync will be disabled.\nActual error: %s\n", s, err.Error()) return } } @@ -55,10 +55,6 @@ func (s *syncDispatcher) start() { }() } -func (s *syncDispatcher) stop() { - close(s.quit) -} - func (s *syncDispatcher) runNow() { s.force <- true } @@ -106,12 +102,14 @@ func syncWordsFromUpstreamFor(langCode string) { func addFilesFromLocalLearnQueue(langCode string, files []string, filesToLearn chan string) { if files != nil { log.Printf("Adding %d files to learn from local learn queue\n", len(files)) + for _, f := range files { filesToLearn <- f } } else { log.Printf("Local learn queue for '%s' is empty", langCode) } + close(filesToLearn) } @@ -119,16 +117,21 @@ func downloadAllWords(langCode string, corpusSize int, output chan string) { for { offset := getDownloadOffset(langCode) log.Printf("Offset: %d\n", offset) + if offset >= corpusSize { break } + filePath, err := downloadWordsAndUpdateOffset(langCode, offset) if err != nil { break } + output <- filePath } + log.Println("Local copy is upto date. No need to download from upstream") + close(output) } @@ -139,9 +142,11 @@ func learnAll(langCode string, filesToLearn chan string) { } func learnFromFile(langCode, fileToLearn string) { - log.Printf("Learning from %s\n", fileToLearn) start := time.Now() - getOrCreateHandler(langCode, func(handle *libvarnam.Varnam) (data interface{}, err error) { + + log.Printf("Learning from %s\n", fileToLearn) + + _, _ = getOrCreateHandler(langCode, func(handle *libvarnam.Varnam) (data interface{}, err error) { learnStatus, err := handle.LearnFromFile(fileToLearn) end := time.Now() if err != nil { @@ -150,8 +155,7 @@ func learnFromFile(langCode, fileToLearn string) { log.Printf("Learned from '%s'. TotalWords: %d, Failed: %d. Took %s\n", fileToLearn, learnStatus.TotalWords, learnStatus.Failed, end.Sub(start)) } - err = os.Remove(fileToLearn) - if err != nil { + if err = os.Remove(fileToLearn); err != nil { log.Printf("Error deleting '%s'. %s\n", fileToLearn, err.Error()) } @@ -166,8 +170,7 @@ func downloadWordsAndUpdateOffset(langCode string, offset int) (string, error) { return "", err } - err = setDownloadOffset(langCode, offset+count) - if err != nil { + if err = setDownloadOffset(langCode, offset+count); err != nil { log.Printf("Error setting download offset for '%s'. %s\n", langCode, err.Error()) return "", err } @@ -176,26 +179,30 @@ func downloadWordsAndUpdateOffset(langCode string, offset int) (string, error) { } func getCorpusDetails(langCode string) (*libvarnam.CorpusDetails, error) { + var m metaResponse + url := fmt.Sprintf("%s/meta/%s", varnamdConfig.upstream, langCode) log.Printf("Fetching corpus details for '%s'\n", langCode) - var m metaResponse - err := getJSONResponse(url, &m) - if err != nil { + + if err := getJSONResponse(url, &m); err != nil { return nil, err } + log.Printf("Corpus size: %d\n", m.Result.WordsCount) + return m.Result, nil } // Downloads words from upstream starting from the specified offset and stores it locally in the learn queue // Returns the number of words downloaded, local file path and error if any func downloadWords(langCode string, offset int) (totalWordsDownloaded int, downloadedFilePath string, err error) { - url := fmt.Sprintf("%s/download/%s/%d", varnamdConfig.upstream, langCode, offset) var response downloadResponse - err = getJSONResponse(url, &response) - if err != nil { + + url := fmt.Sprintf("%s/download/%s/%d", varnamdConfig.upstream, langCode, offset) + if err = getJSONResponse(url, &response); err != nil { return 0, "", err } + downloadedFilePath, err = transformAndPersistWords(langCode, offset, &response) if err != nil { log.Printf("Download was successful, but failed to persist to local learn queue. %s\n", err.Error()) @@ -207,24 +214,28 @@ func downloadWords(langCode string, offset int) (totalWordsDownloaded int, downl func transformAndPersistWords(langCode string, offset int, dresp *downloadResponse) (string, error) { learnQueueDir := getLearnQueueDir(langCode) + targetFile, err := os.Create(path.Join(learnQueueDir, fmt.Sprintf("%s.%d", langCode, offset))) if err != nil { return "", err } - defer targetFile.Close() + + defer func() { _ = targetFile.Close() }() for _, word := range dresp.Words { - _, err = targetFile.WriteString(fmt.Sprintf("%s %d\n", word.Word, word.Confidence)) - if err != nil { + if _, err = targetFile.WriteString(fmt.Sprintf("%s %d\n", word.Word, word.Confidence)); err != nil { return "", err } } + return targetFile.Name(), nil } func getFilesFromLearnQueue(langCode string) []string { var files []string + learnQueueDir := getLearnQueueDir(langCode) + queueContents, err := ioutil.ReadDir(learnQueueDir) if err != nil { return nil @@ -241,22 +252,23 @@ func getFilesFromLearnQueue(langCode string) []string { func getJSONResponse(url string, output interface{}) error { log.Printf("GET: '%s'\n", url) - resp, err := http.Get(url) + + resp, err := http.Get(url) // #nosec G107 if err != nil { return err } - defer resp.Body.Close() + + defer func() { _ = resp.Body.Close() }() + jsonDecoder := json.NewDecoder(resp.Body) - err = jsonDecoder.Decode(output) - if err != nil { - return err - } - return nil + + return jsonDecoder.Decode(output) } func getDownloadOffset(langCode string) int { filePath := getDownloadOffsetMetadataFile(langCode) - content, err := ioutil.ReadFile(filePath) + + content, err := ioutil.ReadFile(filepath.Clean(filePath)) if err != nil { return 0 } @@ -281,30 +293,22 @@ func getDownloadOffsetMetadataFile(langCode string) string { func createLearnQueueDir(langCode string) error { queueDir := getLearnQueueDir(langCode) - err := os.MkdirAll(queueDir, 0777) - if err != nil { - return err - } - return nil + return os.MkdirAll(queueDir, 0750) } func getLearnQueueDir(langCode string) string { syncDir := getSyncMetadataDir() queueDir := path.Join(syncDir, fmt.Sprintf("%s.learn.queue", langCode)) + return queueDir } func createSyncMetadataDir() error { syncDir := getSyncMetadataDir() - err := os.MkdirAll(syncDir, 0777) - if err != nil { - return err - } - return nil + return os.MkdirAll(syncDir, 0750) } func getSyncMetadataDir() string { configDir := getConfigDir() - syncDir := path.Join(configDir, "sync") - return syncDir + return path.Join(configDir, "sync") } diff --git a/ui/index.html b/ui/index.html index e4be1b9..26235f1 100644 --- a/ui/index.html +++ b/ui/index.html @@ -1,163 +1,76 @@ - - - - - - - - - - - - - Varnam - Type in Indian languages - - - + + + + + + - -
- Your Internet connection might have interrupted - -
-
-
+ Varnam - Type in Indian languages + -
-
- - English - - - -
-
- -
-
- - -
+ +
+ Your Internet connection might have interrupted + +
+
+
+
+
+ + English + + +
-
-
-
- +
+
-
- +
+ +
-
- - - - - - +
+
+ +
+
+ +
+ +
+
+ + + + + + diff --git a/ui/javascripts/copycss.js b/ui/javascripts/copycss.js index 3e092fb..2a62c9c 100644 --- a/ui/javascripts/copycss.js +++ b/ui/javascripts/copycss.js @@ -1,6 +1,6 @@ -(function($) { +(function ($) { - $.fn.getStyles = function(only, except) { + $.fn.getStyles = function (only, except) { // the map to return with requested styles and values as KVP var product = {}; @@ -30,12 +30,12 @@ // convenience methods to turn css case ('background-image') to camel ('backgroundImage') var pattern = /\-([a-z])/g; - var uc = function(a, b) { - return b.toUpperCase(); - }; - var camelize = function(string) { - return string.replace(pattern, uc); - }; + var uc = function (a, b) { + return b.toUpperCase(); + }; + var camelize = function (string) { + return string.replace(pattern, uc); + }; // make sure we're getting a good reference if (style = window.getComputedStyle(dom, null)) { @@ -88,7 +88,7 @@ }; // sugar - source is the selector, dom element or jQuery instance to copy from - only and except are optional - $.fn.copyCSS = function(source, only, except) { + $.fn.copyCSS = function (source, only, except) { var styles = $(source).getStyles(only, except); this.css(styles); }; diff --git a/ui/stylesheets/addon.css b/ui/stylesheets/addon.css index 730ded8..4f0b70f 100644 --- a/ui/stylesheets/addon.css +++ b/ui/stylesheets/addon.css @@ -1,17 +1,17 @@ .CodeMirror-completions { - position: absolute; - z-index: 10; - overflow: hidden; - -webkit-box-shadow: 2px 3px 5px rgba(0,0,0,.2); - -moz-box-shadow: 2px 3px 5px rgba(0,0,0,.2); - box-shadow: 2px 3px 5px rgba(0,0,0,.2); + position: absolute; + z-index: 10; + overflow: hidden; + -webkit-box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2); + -moz-box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2); + box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2); } .CodeMirror-completions select { - background: #fafafa; - outline: none; - border: none; - padding: 0; - margin: 0; - font-family: monospace; -} \ No newline at end of file + background: #fafafa; + outline: none; + border: none; + padding: 0; + margin: 0; + font-family: monospace; +} diff --git a/ui/stylesheets/ambiance.css b/ui/stylesheets/ambiance.css index 3b93f9c..6e020c3 100644 --- a/ui/stylesheets/ambiance.css +++ b/ui/stylesheets/ambiance.css @@ -2,47 +2,101 @@ /* Color scheme */ -.cm-s-ambiance .cm-keyword { color: #cda869; } -.cm-s-ambiance .cm-atom { color: #CF7EA9; } -.cm-s-ambiance .cm-number { color: #78CF8A; } -.cm-s-ambiance .cm-def { color: #aac6e3; } -.cm-s-ambiance .cm-variable { color: #ffb795; } -.cm-s-ambiance .cm-variable-2 { color: #eed1b3; } -.cm-s-ambiance .cm-variable-3 { color: #faded3; } -.cm-s-ambiance .cm-property { color: #eed1b3; } -.cm-s-ambiance .cm-operator {color: #fa8d6a;} -.cm-s-ambiance .cm-comment { color: #555; font-style:italic; } -.cm-s-ambiance .cm-string { color: #8f9d6a; } -.cm-s-ambiance .cm-string-2 { color: #9d937c; } -.cm-s-ambiance .cm-meta { color: #D2A8A1; } -.cm-s-ambiance .cm-error { color: #AF2018; } -.cm-s-ambiance .cm-qualifier { color: yellow; } -.cm-s-ambiance .cm-builtin { color: #9999cc; } -.cm-s-ambiance .cm-bracket { color: #24C2C7; } -.cm-s-ambiance .cm-tag { color: #fee4ff } -.cm-s-ambiance .cm-attribute { color: #9B859D; } -.cm-s-ambiance .cm-header {color: blue;} -.cm-s-ambiance .cm-quote { color: #24C2C7; } -.cm-s-ambiance .cm-hr { color: pink; } -.cm-s-ambiance .cm-link { color: #F4C20B; } -.cm-s-ambiance .cm-special { color: #FF9D00; } +.cm-s-ambiance .cm-keyword { + color: #cda869; +} +.cm-s-ambiance .cm-atom { + color: #cf7ea9; +} +.cm-s-ambiance .cm-number { + color: #78cf8a; +} +.cm-s-ambiance .cm-def { + color: #aac6e3; +} +.cm-s-ambiance .cm-variable { + color: #ffb795; +} +.cm-s-ambiance .cm-variable-2 { + color: #eed1b3; +} +.cm-s-ambiance .cm-variable-3 { + color: #faded3; +} +.cm-s-ambiance .cm-property { + color: #eed1b3; +} +.cm-s-ambiance .cm-operator { + color: #fa8d6a; +} +.cm-s-ambiance .cm-comment { + color: #555; + font-style: italic; +} +.cm-s-ambiance .cm-string { + color: #8f9d6a; +} +.cm-s-ambiance .cm-string-2 { + color: #9d937c; +} +.cm-s-ambiance .cm-meta { + color: #d2a8a1; +} +.cm-s-ambiance .cm-error { + color: #af2018; +} +.cm-s-ambiance .cm-qualifier { + color: yellow; +} +.cm-s-ambiance .cm-builtin { + color: #9999cc; +} +.cm-s-ambiance .cm-bracket { + color: #24c2c7; +} +.cm-s-ambiance .cm-tag { + color: #fee4ff; +} +.cm-s-ambiance .cm-attribute { + color: #9b859d; +} +.cm-s-ambiance .cm-header { + color: blue; +} +.cm-s-ambiance .cm-quote { + color: #24c2c7; +} +.cm-s-ambiance .cm-hr { + color: pink; +} +.cm-s-ambiance .cm-link { + color: #f4c20b; +} +.cm-s-ambiance .cm-special { + color: #ff9d00; +} -.cm-s-ambiance .CodeMirror-matchingbracket { color: #0f0; } -.cm-s-ambiance .CodeMirror-nonmatchingbracket { color: #f22; } +.cm-s-ambiance .CodeMirror-matchingbracket { + color: #0f0; +} +.cm-s-ambiance .CodeMirror-nonmatchingbracket { + color: #f22; +} .cm-s-ambiance .CodeMirror-selected { background: rgba(255, 255, 255, 0.15); } .CodeMirror-focused .cm-s-ambiance .CodeMirror-selected { - background: rgba(255, 255, 255, 0.10); + background: rgba(255, 255, 255, 0.1); } /* Editor styling */ .cm-s-ambiance { - line-height: 1.40em; - font-family: Monaco, Menlo,"Andale Mono","lucida console","Courier New",monospace !important; - color: #E6E1DC; + line-height: 1.4em; + font-family: Monaco, Menlo, "Andale Mono", "lucida console", "Courier New", + monospace !important; + color: #e6e1dc; background-color: #202020; -webkit-box-shadow: inset 0 0 10px black; -moz-box-shadow: inset 0 0 10px black; @@ -51,10 +105,10 @@ } .cm-s-ambiance .CodeMirror-gutter { - background: #3D3D3D; + background: #3d3d3d; padding: 0 5px; text-shadow: #333 1px 1px; - border-right: 1px solid #4D4D4D; + border-right: 1px solid #4d4d4d; box-shadow: 0 10px 20px black; } @@ -64,11 +118,10 @@ } .cm-s-ambiance .CodeMirror-lines { - } .cm-s-ambiance .CodeMirror-lines .CodeMirror-cursor { - border-left: 1px solid #7991E8; + border-left: 1px solid #7991e8; } .cm-s-ambiance .activeline { diff --git a/ui/stylesheets/bootstrap-responsive.css b/ui/stylesheets/bootstrap-responsive.css index 06e55c0..4429722 100644 --- a/ui/stylesheets/bootstrap-responsive.css +++ b/ui/stylesheets/bootstrap-responsive.css @@ -35,9 +35,9 @@ width: 100%; min-height: 28px; -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - -ms-box-sizing: border-box; - box-sizing: border-box; + -moz-box-sizing: border-box; + -ms-box-sizing: border-box; + box-sizing: border-box; } .hidden { @@ -185,9 +185,9 @@ width: 100%; min-height: 28px; -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - -ms-box-sizing: border-box; - box-sizing: border-box; + -moz-box-sizing: border-box; + -ms-box-sizing: border-box; + box-sizing: border-box; } .input-prepend input, .input-append input, @@ -312,9 +312,9 @@ margin-left: 2.762430939%; *margin-left: 2.709239449638298%; -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - -ms-box-sizing: border-box; - box-sizing: border-box; + -moz-box-sizing: border-box; + -ms-box-sizing: border-box; + box-sizing: border-box; } .row-fluid [class*="span"]:first-child { margin-left: 0; @@ -548,9 +548,9 @@ margin-left: 2.564102564%; *margin-left: 2.510911074638298%; -webkit-box-sizing: border-box; - -moz-box-sizing: border-box; - -ms-box-sizing: border-box; - box-sizing: border-box; + -moz-box-sizing: border-box; + -ms-box-sizing: border-box; + box-sizing: border-box; } .row-fluid [class*="span"]:first-child { margin-left: 0; @@ -732,15 +732,15 @@ font-weight: bold; color: #999999; -webkit-border-radius: 3px; - -moz-border-radius: 3px; - border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; } .nav-collapse .btn { padding: 4px 10px 4px; font-weight: normal; -webkit-border-radius: 4px; - -moz-border-radius: 4px; - border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; } .nav-collapse .dropdown-menu li + li a { margin-bottom: 2px; @@ -765,11 +765,11 @@ background-color: transparent; border: none; -webkit-border-radius: 0; - -moz-border-radius: 0; - border-radius: 0; + -moz-border-radius: 0; + border-radius: 0; -webkit-box-shadow: none; - -moz-box-shadow: none; - box-shadow: none; + -moz-box-shadow: none; + box-shadow: none; } .nav-collapse .dropdown-menu:before, .nav-collapse .dropdown-menu:after { @@ -785,9 +785,12 @@ margin: 9px 0; border-top: 1px solid #222222; border-bottom: 1px solid #222222; - -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); - -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); - box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); + -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), + 0 1px 0 rgba(255, 255, 255, 0.1); + -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), + 0 1px 0 rgba(255, 255, 255, 0.1); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), + 0 1px 0 rgba(255, 255, 255, 0.1); } .navbar .nav-collapse .nav.pull-right { float: none; diff --git a/ui/stylesheets/codemirror.css b/ui/stylesheets/codemirror.css index 05ad0ed..524c749 100644 --- a/ui/stylesheets/codemirror.css +++ b/ui/stylesheets/codemirror.css @@ -20,7 +20,8 @@ /* Vertical scrollbar */ .CodeMirror-scrollbar { position: absolute; - right: 0; top: 0; + right: 0; + top: 0; overflow-x: hidden; overflow-y: scroll; z-index: 5; @@ -47,7 +48,9 @@ } .CodeMirror-gutter { - position: absolute; left: 0; top: 0; + position: absolute; + left: 0; + top: 0; z-index: 10; background-color: #f7f7f7; border-right: 1px solid #eee; @@ -57,12 +60,12 @@ .CodeMirror-gutter-text { color: #aaa; text-align: right; - padding: .4em .2em .4em .4em; + padding: 0.4em 0.2em 0.4em 0.4em; white-space: pre !important; cursor: default; } .CodeMirror-lines { - padding: .4em; + padding: 0.4em; white-space: pre; cursor: text; } @@ -72,10 +75,14 @@ -webkit-border-radius: 0; -o-border-radius: 0; border-radius: 0; - border-width: 0; margin: 0; padding: 0; background: transparent; + border-width: 0; + margin: 0; + padding: 0; + background: transparent; font-family: inherit; font-size: inherit; - padding: 0; margin: 0; + padding: 0; + margin: 0; white-space: pre; word-wrap: normal; line-height: inherit; @@ -107,67 +114,132 @@ width: auto; border: 0; background: transparent; - background: rgba(0, 200, 0, .4); + background: rgba(0, 200, 0, 0.4); filter: progid:DXImageTransform.Microsoft.gradient(startColorstr=#6600c800, endColorstr=#4c00c800); } /* Kludge to turn off filter in ie9+, which also accepts rgba */ .cm-keymap-fat-cursor pre.CodeMirror-cursor:not(#nonsense_id) { filter: progid:DXImageTransform.Microsoft.gradient(enabled=false); } -.CodeMirror pre.CodeMirror-cursor.CodeMirror-overwrite {} +.CodeMirror pre.CodeMirror-cursor.CodeMirror-overwrite { +} .CodeMirror-focused pre.CodeMirror-cursor { visibility: visible; } -div.CodeMirror-selected { background: #d9d9d9; } -.CodeMirror-focused div.CodeMirror-selected { background: #d7d4f0; } +div.CodeMirror-selected { + background: #d9d9d9; +} +.CodeMirror-focused div.CodeMirror-selected { + background: #d7d4f0; +} .CodeMirror-searching { background: #ffa; - background: rgba(255, 255, 0, .4); + background: rgba(255, 255, 0, 0.4); } /* Default theme */ -.cm-s-default span.cm-keyword {color: #708;} -.cm-s-default span.cm-atom {color: #219;} -.cm-s-default span.cm-number {color: #164;} -.cm-s-default span.cm-def {color: #00f;} -.cm-s-default span.cm-variable {color: black;} -.cm-s-default span.cm-variable-2 {color: #05a;} -.cm-s-default span.cm-variable-3 {color: #085;} -.cm-s-default span.cm-property {color: black;} -.cm-s-default span.cm-operator {color: black;} -.cm-s-default span.cm-comment {color: #a50;} -.cm-s-default span.cm-string {color: #a11;} -.cm-s-default span.cm-string-2 {color: #f50;} -.cm-s-default span.cm-meta {color: #555;} -.cm-s-default span.cm-error {color: #f00;} -.cm-s-default span.cm-qualifier {color: #555;} -.cm-s-default span.cm-builtin {color: #30a;} -.cm-s-default span.cm-bracket {color: #997;} -.cm-s-default span.cm-tag {color: #170;} -.cm-s-default span.cm-attribute {color: #00c;} -.cm-s-default span.cm-header {color: blue;} -.cm-s-default span.cm-quote {color: #090;} -.cm-s-default span.cm-hr {color: #999;} -.cm-s-default span.cm-link {color: #00c;} - -span.cm-header, span.cm-strong {font-weight: bold;} -span.cm-em {font-style: italic;} -span.cm-emstrong {font-style: italic; font-weight: bold;} -span.cm-link {text-decoration: underline;} - -span.cm-invalidchar {color: #f00;} - -div.CodeMirror span.CodeMirror-matchingbracket {color: #0f0;} -div.CodeMirror span.CodeMirror-nonmatchingbracket {color: #f22;} +.cm-s-default span.cm-keyword { + color: #708; +} +.cm-s-default span.cm-atom { + color: #219; +} +.cm-s-default span.cm-number { + color: #164; +} +.cm-s-default span.cm-def { + color: #00f; +} +.cm-s-default span.cm-variable { + color: black; +} +.cm-s-default span.cm-variable-2 { + color: #05a; +} +.cm-s-default span.cm-variable-3 { + color: #085; +} +.cm-s-default span.cm-property { + color: black; +} +.cm-s-default span.cm-operator { + color: black; +} +.cm-s-default span.cm-comment { + color: #a50; +} +.cm-s-default span.cm-string { + color: #a11; +} +.cm-s-default span.cm-string-2 { + color: #f50; +} +.cm-s-default span.cm-meta { + color: #555; +} +.cm-s-default span.cm-error { + color: #f00; +} +.cm-s-default span.cm-qualifier { + color: #555; +} +.cm-s-default span.cm-builtin { + color: #30a; +} +.cm-s-default span.cm-bracket { + color: #997; +} +.cm-s-default span.cm-tag { + color: #170; +} +.cm-s-default span.cm-attribute { + color: #00c; +} +.cm-s-default span.cm-header { + color: blue; +} +.cm-s-default span.cm-quote { + color: #090; +} +.cm-s-default span.cm-hr { + color: #999; +} +.cm-s-default span.cm-link { + color: #00c; +} -@media print { +span.cm-header, +span.cm-strong { + font-weight: bold; +} +span.cm-em { + font-style: italic; +} +span.cm-emstrong { + font-style: italic; + font-weight: bold; +} +span.cm-link { + text-decoration: underline; +} + +span.cm-invalidchar { + color: #f00; +} +div.CodeMirror span.CodeMirror-matchingbracket { + color: #0f0; +} +div.CodeMirror span.CodeMirror-nonmatchingbracket { + color: #f22; +} + +@media print { /* Hide the cursor when printing */ .CodeMirror pre.CodeMirror-cursor { visibility: hidden; } - } diff --git a/ui/stylesheets/print.css b/ui/stylesheets/print.css index 280e58d..256e962 100644 --- a/ui/stylesheets/print.css +++ b/ui/stylesheets/print.css @@ -1,16 +1,19 @@ -div.toolbar, #editor_div, header, .navbar{ - display:none; +div.toolbar, +#editor_div, +header, +.navbar { + display: none; } -#preview_div{ - display:block !important; - width:100%; +#preview_div { + display: block !important; + width: 100%; margin: 0; border: 0; padding: 0; } -#preview{ +#preview { border: 0; margin: 0; box-shadow: none; diff --git a/ui/stylesheets/styles.css b/ui/stylesheets/styles.css index a5a7e3d..eca7b67 100644 --- a/ui/stylesheets/styles.css +++ b/ui/stylesheets/styles.css @@ -1,146 +1,257 @@ body { - background: url("../images/stressed_linen.png") repeat scroll 0 0 #151515; - /*color: #EAEAEA;*/ - color: #B3B3B3; - line-height: 1.5; - margin: 0 0 10px; - padding: 0; - font-size: 16px; - font-weight: normal; + background: url("../images/stressed_linen.png") repeat scroll 0 0 #151515; + /*color: #EAEAEA;*/ + color: #b3b3b3; + line-height: 1.5; + margin: 0 0 10px; + padding: 0; + font-size: 16px; + font-weight: normal; } code { - border: 0px; - background-color: rgba(186, 196, 204, 0.12); + border: 0px; + background-color: rgba(186, 196, 204, 0.12); } .container { - max-width: 800px; - margin: 0 auto; - width: 90%; + max-width: 800px; + margin: 0 auto; + width: 90%; } header h1:before { - content: "./ "; - font-size: 24px; + content: "./ "; + font-size: 24px; } .main-content { - background: none repeat scroll 0 0 rgba(186, 196, 204, 0.12); - border-radius: 10px 10px 10px 10px; - margin-top: 15px; - padding-left: 30px; - padding-right: 30px; + background: none repeat scroll 0 0 rgba(186, 196, 204, 0.12); + border-radius: 10px 10px 10px 10px; + margin-top: 15px; + padding-left: 30px; + padding-right: 30px; } .footer { - font-size: 12px; - text-align: right; - padding-left: 50%; + font-size: 12px; + text-align: right; + padding-left: 50%; } h1 { - font-size: 30px; + font-size: 30px; } h1 a { - color: inherit; - text-decoration: none; + color: inherit; + text-decoration: none; } h1 a:hover { - color: inherit; - text-decoration: none; + color: inherit; + text-decoration: none; } -h2, h3 { - font-size: 18px; +h2, +h3 { + font-size: 18px; } -h1, h2, h3 { - /*color: #0088CC;*/ - color: #B5E853; +h1, +h2, +h3 { + /*color: #0088CC;*/ + color: #b5e853; } .main-content h3 { - font-size: 18px; + font-size: 18px; } .logo { - font-family: Monaco,"Bitstream Vera Sans Mono","Lucida Console",Terminal,monospace; + font-family: Monaco, "Bitstream Vera Sans Mono", "Lucida Console", Terminal, + monospace; } -header .logo h2{ - color: #666666; - font-weight: 300; - font-size:18px; - letter-spacing: -0.03em; - text-shadow: 0 1px 1px rgba(0, 0, 0, 0.1), 0 0 5px rgba(181, 232, 83, 0.1), 0 0 10px rgba(181, 232, 83, 0.1); - line-height: 30px; +header .logo h2 { + color: #666666; + font-weight: 300; + font-size: 18px; + letter-spacing: -0.03em; + text-shadow: 0 1px 1px rgba(0, 0, 0, 0.1), 0 0 5px rgba(181, 232, 83, 0.1), + 0 0 10px rgba(181, 232, 83, 0.1); + line-height: 30px; } #mainlinks a { - border-color: #000000 rgba(0, 0, 0, 0.7) rgba(0, 0, 0, 0.7); - border-image: none; - border-left: 2px solid rgba(0, 0, 0, 0.7); - border-radius: 50px 50px 50px 50px; - border-right: 2px solid rgba(0, 0, 0, 0.7); - border-style: solid; - border-width: 2px; - box-shadow: 0 1px 0 rgba(255, 255, 255, 0.05) inset; - color: #B3B3B3; - display: inline-block; - font-family: Helvetica,Arial,sans-serif; - font-size: 15px; - padding: 8px 18px; - text-decoration: none; - text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.75); - background-color: #1E1C1C; + border-color: #000000 rgba(0, 0, 0, 0.7) rgba(0, 0, 0, 0.7); + border-image: none; + border-left: 2px solid rgba(0, 0, 0, 0.7); + border-radius: 50px 50px 50px 50px; + border-right: 2px solid rgba(0, 0, 0, 0.7); + border-style: solid; + border-width: 2px; + box-shadow: 0 1px 0 rgba(255, 255, 255, 0.05) inset; + color: #b3b3b3; + display: inline-block; + font-family: Helvetica, Arial, sans-serif; + font-size: 15px; + padding: 8px 18px; + text-decoration: none; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.75); + background-color: #1e1c1c; } p { - margin: 0 0 1.5em; + margin: 0 0 1.5em; } .navigation { - float: right; + float: right; } .editor { - clear: both; - margin-right: 10px; + clear: both; + margin-right: 10px; } .repolinks { - list-style: none; + list-style: none; } .repolinks li:before { - content: "- "; - color: red; - font-weight: bold; - font-size: smaller; + content: "- "; + color: red; + font-weight: bold; + font-size: smaller; } -nav ul{list-style-type:none;float:right;} -nav ul li {display:inline-block; } -.headlines {margin-top:200px;} -.headlines p{font-size:25px; color:#fff;} -.main-container{margin-top:40px;} -.banner { background:#ccc;} +nav ul { + list-style-type: none; + float: right; +} +nav ul li { + display: inline-block; +} +.headlines { + margin-top: 200px; +} +.headlines p { + font-size: 25px; + color: #fff; +} +.main-container { + margin-top: 40px; +} +.banner { + background: #ccc; +} -header.inner {height:100px;background: url("../images/bg.jpg") no-repeat scroll left -165px transparent;} -header.inner .logo{height:100px;} -header.inner .logo h1{line-height:100px;} +header.inner { + height: 100px; + background: url("../images/bg.jpg") no-repeat scroll left -165px transparent; +} +header.inner .logo { + height: 100px; +} +header.inner .logo h1 { + line-height: 100px; +} pre { - background-color: black; - color: #666; + background-color: black; + color: #666; } .main-intro { - text-align: center; - padding-top: 10px; - padding-bottom: 0px; + text-align: center; + padding-top: 10px; + padding-bottom: 0px; +} + +.CodeMirror-completions { + position: absolute; + z-index: 10; + overflow: hidden; + -webkit-box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2); + -moz-box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2); + box-shadow: 2px 3px 5px rgba(0, 0, 0, 0.2); +} + +.CodeMirror-completions select { + background: #fafafa; + outline: none; + border: none; + padding: 0; + margin: 0; + font-family: monospace; +} + +.CodeMirror { + background: #ededed; + border: #333 solid 1px; +} + +iframe { + /* width: 100%;*/ + display: block; + width: 100%; + height: 400px; + background-color: #fff; + /*padding: 5px;*/ + overflow: auto; + /*margin-right:5px;*/ + border: 1px solid black; + box-shadow: rgba(0, 0, 0, 0.75) 0 2px 5px; + border-radius: 3px; +} + +.CodeMirror pre { + line-height: inherit; + color: #b3b3b3; +} + +.CodeMirror-wrap pre { + word-break: normal; +} + +.CodeMirror-gutter { + background: none repeat scroll 0 0 rgba(186, 196, 204, 0.12); +} + +.navbar h1 { + color: #000; + /* margin-top: 5px; */ + display: inline; +} + +.toolbar { + position: relative; + padding-right: 10px; +} + +.caption { + position: absolute; + bottom: 17px; + left: 155px; +} + +.navbar .navbar-inner { + background: none repeat scroll 0 0 #ffffff; + border-radius: 0 0 0 0; + box-shadow: none; + color: #000000; + padding-top: 10px; + text-shadow: none; } +.networkerror { + z-index: 9; + position: fixed; + top: 0px; + left: 18%; + display: none; + margin: 0 auto; + width: 60%; +} diff --git a/varnam_handlers.go b/varnam_handlers.go index 3bdd173..29e8474 100644 --- a/varnam_handlers.go +++ b/varnam_handlers.go @@ -9,11 +9,11 @@ import ( "github.com/golang/groupcache" _ "github.com/mattn/go-sqlite3" - "github.com/varnamproject/libvarnam-golang" + "github.com/varnamproject/varnamd/libvarnam" ) type word struct { - Id int `json:"id"` + ID int `json:"id"` Confidence int `json:"confidence"` Word string `json:"word"` } @@ -24,8 +24,8 @@ var ( mutex *sync.Mutex once sync.Once schemeDetails = libvarnam.GetAllSchemeDetails() - peers = groupcache.NewHTTPPool("http://localhost") cacheGroups = make(map[string]*groupcache.Group) + // peers = groupcache.NewHTTPPool("http://localhost") ) func isValidSchemeIdentifier(id string) bool { @@ -34,6 +34,7 @@ func isValidSchemeIdentifier(id string) bool { return true } } + return false } @@ -41,14 +42,17 @@ func initLanguageChannels() { languageChannels = make(map[string]chan *libvarnam.Varnam) channelsCount = make(map[string]int) mutex = &sync.Mutex{} + for _, scheme := range schemeDetails { languageChannels[scheme.Identifier] = make(chan *libvarnam.Varnam, maxHandleCount) channelsCount[scheme.Identifier] = maxHandleCount + for i := 0; i < maxHandleCount; i++ { handle, err := libvarnam.Init(scheme.Identifier) if err != nil { - panic("Unable to init varnam for language " + scheme.LangCode + "." + err.Error()) + panic("Unable to init varnam for language" + scheme.LangCode + ". " + err.Error()) } + languageChannels[scheme.Identifier] <- handle } } @@ -57,32 +61,47 @@ func initLanguageChannels() { func getOrCreateHandler(schemeIdentifier string, f func(handle *libvarnam.Varnam) (data interface{}, err error)) (data interface{}, err error) { ch, ok := languageChannels[schemeIdentifier] if !ok { - return nil, errors.New("Invalid scheme identifier") + return nil, errors.New("invalid scheme identifier") } + select { case handle := <-ch: data, err = f(handle) + go func() { ch <- handle }() case <-time.After(800 * time.Millisecond): var handle *libvarnam.Varnam + handle, err = libvarnam.Init(schemeIdentifier) if err != nil { log.Println(err) - return nil, errors.New("Unable to initialize varnam handle") + return nil, errors.New("unable to initialize varnam handle") } + data, err = f(handle) + if err != nil { + log.Println(err) + return nil, errors.New("unable to complete varnam handle") + } + go sendHandlerToChannel(schemeIdentifier, handle, ch) } + return } -func transliterate(schemeIdentifier string, word string) (data interface{}, err error) { +func transliterate(schemeIdentifier string, word string) (interface{}, error) { return getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { - data, err = handle.Transliterate(word) - return + return handle.Transliterate(word) }) } +// func trainwords(schemeIdentifier, word, pattern string) (interface{}, error) { +// return getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { +// return handle.Train(pattern,word) +// }) +// } + func getWords(schemeIdentifier string, downloadStart int) ([]*word, error) { filepath, _ := getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { return handle.GetSuggestionsFilePath(), nil @@ -92,37 +111,47 @@ func getWords(schemeIdentifier string, downloadStart int) ([]*word, error) { if err != nil { return nil, err } - defer db.Close() + + defer func() { _ = db.Close() }() // Making an index for all learned words so that download is faster // this needs to be removed when there is more clarity on how learned words needs to be handled - _, err = db.Exec("create index if not exists varnamd_download_only_learned on patterns_content (learned) where learned = 1;") - if err != nil { + if _, err = db.Exec("create index if not exists varnamd_download_only_learned on patterns_content (learned) where learned = 1;"); err != nil { return nil, err } - q := "select id, word, confidence from words where id in (select distinct(word_id) from patterns_content where learned = 1) order by id asc limit ? offset ?;" + q := `select id, word, confidence from words where id in (select distinct(word_id) from patterns_content where learned = 1) order by id asc limit ? offset ?;` + rows, err := db.Query(q, downloadPageSize, downloadStart) if err != nil { return nil, err } - defer rows.Close() + + if rows.Err() != nil { + return nil, rows.Err() + } + + defer func() { _ = rows.Close() }() var words []*word + for rows.Next() { - var id, confidence int - var _word string - rows.Scan(&id, &_word, &confidence) - words = append(words, &word{Id: id, Confidence: confidence, Word: _word}) + var ( + id, confidence int + _word string + ) + + _ = rows.Scan(&id, &_word, &confidence) + + words = append(words, &word{ID: id, Confidence: confidence, Word: _word}) } return words, nil } -func reveseTransliterate(schemeIdentifier string, word string) (data interface{}, err error) { +func reveseTransliterate(schemeIdentifier string, word string) (interface{}, error) { return getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { - data, err = handle.ReverseTransliterate(word) - return + return handle.ReverseTransliterate(word) }) } @@ -130,11 +159,14 @@ func sendHandlerToChannel(schemeIdentifier string, handle *libvarnam.Varnam, ch mutex.Lock() count := channelsCount[schemeIdentifier] mutex.Unlock() + if count == maxHandleCount { log.Printf("Throw away handle") handle.Destroy() + return } + select { case ch <- handle: mutex.Lock() @@ -144,3 +176,15 @@ func sendHandlerToChannel(schemeIdentifier string, handle *libvarnam.Varnam, ch default: } } + +func getSchemeFilePath(schemeIdentifier string) (interface{}, error) { + return getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { + return handle.GetSchemeFilePath(), nil + }) +} + +func deleteWord(schemeIdentifier string, word string) (interface{}, error) { + return getOrCreateHandler(schemeIdentifier, func(handle *libvarnam.Varnam) (data interface{}, err error) { + return nil, handle.DeleteWord(word) + }) +} diff --git a/varnam_learn.go b/varnam_learn.go index 6d23eb6..22e4225 100644 --- a/varnam_learn.go +++ b/varnam_learn.go @@ -1,37 +1,119 @@ package main import ( - "log" + "fmt" + "net/http" + "os" + "strings" + "time" - "github.com/varnamproject/libvarnam-golang" + "github.com/labstack/echo/v4" + "github.com/varnamproject/varnamd/libvarnam" ) -type Args struct { - LangCode string `json:"lang"` - Word string `json:"word"` -} +const defaultChanSize = 1000 var ( learnChannels map[string]chan string + trainChannel map[string]chan trainArgs ) -func initLearnChannels() { +// initChannels method will initialize learn and train channels. +func (app *App) initChannels() { learnChannels = make(map[string]chan string) + trainChannel = make(map[string]chan trainArgs) + for _, scheme := range schemeDetails { - learnChannels[scheme.Identifier] = make(chan string, 100) + learnChannels[scheme.Identifier] = make(chan string, defaultChanSize) + trainChannel[scheme.Identifier] = make(chan trainArgs, defaultChanSize) + handle, err := libvarnam.Init(scheme.Identifier) if err != nil { - log.Fatal("Unable to initialize varnam for lang", scheme.LangCode) + app.log.Fatal("Unable to initialize varnam for lang", scheme.LangCode) + } + + go app.listenForWords(scheme.Identifier, handle) + } +} + +func (app *App) listenForWords(lang string, handle *libvarnam.Varnam) { + for { + select { + case word := <-learnChannels[lang]: + if err := handle.Learn(strings.TrimSpace(word)); err != nil { + app.log.Printf("Failed to learn %s. %s\n", word, err.Error()) + } + case args := <-trainChannel[lang]: + if err := handle.Train(strings.TrimSpace(args.Pattern), strings.TrimSpace(args.Word)); err != nil { + app.log.Printf("error training word: %s, pattern: %s, err:%s", args.Word, args.Pattern, err.Error()) + } } - go listenForWords(scheme.Identifier, handle) } } -func listenForWords(lang string, handle *libvarnam.Varnam) { - for word := range learnChannels[lang] { - err := handle.Learn(word) +func learnWordsFromFile(c echo.Context, langCode string, fileToLearn string, removeFile bool) { + c.Response().WriteHeader(http.StatusOK) + + start := time.Now() + + sendOutput := func(msg string) { + _, _ = c.Response().Write([]byte(msg)) + c.Response().Flush() + } + + sendOutput(fmt.Sprintf("Learning from %s\n", fileToLearn)) + + _, _ = getOrCreateHandler(langCode, func(handle *libvarnam.Varnam) (data interface{}, err error) { + learnStatus, err := handle.LearnFromFile(fileToLearn) + end := time.Now() + if err != nil { - log.Printf("Failed to learn %s. %s\n", word, err.Error()) + sendOutput(fmt.Sprintf("Error learning from '%s'\n", err.Error())) + } else { + sendOutput(fmt.Sprintf("Learned from '%s'. TotalWords: %d, Failed: %d. Took %s\n", fileToLearn, learnStatus.TotalWords, learnStatus.Failed, end.Sub(start))) + } + + if removeFile { + if err = os.Remove(fileToLearn); err != nil { + sendOutput(fmt.Sprintf("Error deleting '%s'. %s\n", fileToLearn, err.Error())) + } } + + return + }) +} + +func importLearningsFromFile(c echo.Context, langCode string, fileToLearn string, removeFile bool) error { + c.Response().WriteHeader(http.StatusOK) + + start := time.Now() + + sendOutput := func(msg string) { + _, _ = c.Response().Write([]byte(msg)) + c.Response().Flush() } + + sendOutput(fmt.Sprintf("Importing from %s\n", fileToLearn)) + + var importError error + + _, _ = getOrCreateHandler(langCode, func(handle *libvarnam.Varnam) (data interface{}, err error) { + err = handle.ImportFromFile(fileToLearn) + end := time.Now() + + if err != nil { + importError = err + } else { + sendOutput(fmt.Sprintf("Import completed. Took %s\n", end.Sub(start))) + } + + if removeFile { + if err = os.Remove(fileToLearn); err != nil { + sendOutput(fmt.Sprintf("Error deleting '%s'. %s\n", fileToLearn, err.Error())) + } + } + return + }) + + return importError }