diff --git a/chart-repo b/chart-repo new file mode 100755 index 000000000..ceb3058e6 Binary files /dev/null and b/chart-repo differ diff --git a/chart/monocular/local-helm/fdbdoclayer/.helmignore b/chart/monocular/local-helm/fdbdoclayer/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/chart/monocular/local-helm/fdbdoclayer/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/chart/monocular/local-helm/fdbdoclayer/Chart.yaml b/chart/monocular/local-helm/fdbdoclayer/Chart.yaml new file mode 100644 index 000000000..e19e1e279 --- /dev/null +++ b/chart/monocular/local-helm/fdbdoclayer/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: fdbdoclayer +version: 0.1.0 diff --git a/chart/monocular/local-helm/fdbdoclayer/templates/_helpers.tpl b/chart/monocular/local-helm/fdbdoclayer/templates/_helpers.tpl new file mode 100644 index 000000000..a61ed5fe2 --- /dev/null +++ b/chart/monocular/local-helm/fdbdoclayer/templates/_helpers.tpl @@ -0,0 +1,33 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name for the document layer. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "doclayer.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s-%s" .Release.Name $name "fdbdoclayer" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Render image reference +*/}} +{{- define "fdb.image" -}} +{{ .registry }}/{{ .repository }}:{{ .tag }} +{{- end -}} diff --git a/chart/monocular/local-helm/fdbdoclayer/templates/fdb-doc-layer-deployment.yaml b/chart/monocular/local-helm/fdbdoclayer/templates/fdb-doc-layer-deployment.yaml new file mode 100644 index 000000000..48fcd6269 --- /dev/null +++ b/chart/monocular/local-helm/fdbdoclayer/templates/fdb-doc-layer-deployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ template "fullname" . }}-fdbdoclayer + labels: + app: {{ template "fullname" . }}-fdbdoclayer + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + replicas: {{ .Values.fdbdoclayer.replicas }} + selector: + matchLabels: + app: {{ template "fullname" . }}-fdbdoclayer + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "fullname" . }}-fdbdoclayer + release: {{ .Release.Name }} + spec: +{{- with .Values.securityContext }} + securityContext: +{{ toYaml . | indent 8 }} +{{- end }} + containers: + - name: fdbdoclayer + image: {{ template "fdb.image" .Values.fdbdoclayer.image }} + env: + - name: FDB_COORDINATOR + value: {{ template "fullname" . }}-fdbserver + - name: CLUSTER_ID + value: chartdb:erlklkg + ports: + - containerPort: {{ .Values.fdbdoclayer.service.port }} + livenessProbe: + tcpSocket: + port: {{ .Values.fdbdoclayer.service.port }} + initialDelaySeconds: 3 + periodSeconds: 20 + readinessProbe: + tcpSocket: + port: {{ .Values.fdbdoclayer.service.port }} + initialDelaySeconds: 3 + periodSeconds: 10 \ No newline at end of file diff --git a/chart/monocular/local-helm/fdbdoclayer/templates/fdb-doc-layer-networkpolicy.yaml b/chart/monocular/local-helm/fdbdoclayer/templates/fdb-doc-layer-networkpolicy.yaml new file mode 100644 index 000000000..f3ddc168a --- /dev/null +++ b/chart/monocular/local-helm/fdbdoclayer/templates/fdb-doc-layer-networkpolicy.yaml @@ -0,0 +1,20 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "fullname" . }}-fdbdoclayer + labels: + app: {{ template "fullname" . }}-fdbdoclayer + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + podSelector: + matchLabels: + app: {{ template "fullname" . }}-fdbdoclayer + release: {{ .Release.Name }} + ingress: + ports: + port: {{ .Values.fdbdoclayer.service.port }} + protocol: TCP +{{- end }} diff --git a/chart/monocular/local-helm/fdbdoclayer/templates/fdb-doc-layer-service.yaml b/chart/monocular/local-helm/fdbdoclayer/templates/fdb-doc-layer-service.yaml new file mode 100644 index 000000000..d10f6e545 --- /dev/null +++ b/chart/monocular/local-helm/fdbdoclayer/templates/fdb-doc-layer-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "doclayer.fullname" . }} + labels: + app: {{ template "fullname" . }}-fdbdoclayer + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + type: ClusterIP + ports: + - port: {{ .Values.fdbdoclayer.service.port }} + targetPort: {{ .Values.fdbdoclayer.service.port }} + protocol: TCP + name: {{ .Values.fdbdoclayer.service.name }} + selector: + app: {{ template "fullname" . }}-fdbdoclayer + release: {{ .Release.Name }} \ No newline at end of file diff --git a/chart/monocular/local-helm/fdbdoclayer/templates/fdbserver-deployment.yaml b/chart/monocular/local-helm/fdbdoclayer/templates/fdbserver-deployment.yaml new file mode 100644 index 000000000..d5dd4d729 --- /dev/null +++ b/chart/monocular/local-helm/fdbdoclayer/templates/fdbserver-deployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ template "fullname" . }}-fdbserver + labels: + app: {{ template "fullname" . }}-fdbserver + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + replicas: {{ .Values.fdbserver.replicas }} + selector: + matchLabels: + app: {{ template "fullname" . }}-fdbserver + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "fullname" . }}-fdbserver + release: {{ .Release.Name }} + spec: +{{- with .Values.securityContext }} + securityContext: +{{ toYaml . | indent 8 }} +{{- end }} + containers: + - name: fdbserver + image: {{ template "fdb.image" .Values.fdbserver.image }} + env: + - name: FDB_COORDINATOR + value: {{ template "fullname" . }}-fdbserver.monocular.svc.cluster.local + - name: CLUSTER_ID + value: chartdb:erlklkg + ports: + - containerPort: {{ .Values.fdbserver.service.port }} + livenessProbe: + tcpSocket: + port: {{ .Values.fdbserver.service.port }} + initialDelaySeconds: 3 + periodSeconds: 20 + readinessProbe: + tcpSocket: + port: {{ .Values.fdbserver.service.port }} + initialDelaySeconds: 3 + periodSeconds: 10 \ No newline at end of file diff --git a/chart/monocular/local-helm/fdbdoclayer/templates/fdbserver-networkpolicy.yaml b/chart/monocular/local-helm/fdbdoclayer/templates/fdbserver-networkpolicy.yaml new file mode 100644 index 000000000..50201611b --- /dev/null +++ b/chart/monocular/local-helm/fdbdoclayer/templates/fdbserver-networkpolicy.yaml @@ -0,0 +1,20 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "fullname" . }}-fdbserver + labels: + app: {{ template "fullname" . }}-fdbserver + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + podSelector: + matchLabels: + app: {{ template "fullname" . }}-fdbserver + release: {{ .Release.Name }} + ingress: + - ports: + - port: {{ .Values.fdbserver.service.port }} + protocol: TCP +{{- end }} diff --git a/chart/monocular/local-helm/fdbdoclayer/templates/fdbserver-service.yaml b/chart/monocular/local-helm/fdbdoclayer/templates/fdbserver-service.yaml new file mode 100644 index 000000000..3b258e5c9 --- /dev/null +++ b/chart/monocular/local-helm/fdbdoclayer/templates/fdbserver-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "fullname" . }}-fdbserver + labels: + app: {{ template "fullname" . }}-fdbserver + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + type: ClusterIP + ports: + - port: {{ .Values.fdbserver.service.port }} + targetPort: {{ .Values.fdbserver.service.port }} + protocol: TCP + name: {{ .Values.fdbserver.service.name }} + selector: + app: {{ template "fullname" . }}-fdbserver + release: {{ .Release.Name }} \ No newline at end of file diff --git a/chart/monocular/local-helm/fdbdoclayer/values.yaml b/chart/monocular/local-helm/fdbdoclayer/values.yaml new file mode 100644 index 000000000..de07ac2ea --- /dev/null +++ b/chart/monocular/local-helm/fdbdoclayer/values.yaml @@ -0,0 +1,26 @@ +fdbserver: + replicas: 1 + image: + registry: docker.io + repository: kreinecke/foundationdb + tag: 6.0.18-k8s + pullPolicy: Always + service: + name: fdbserver + port: 4500 + +fdbdoclayer: + replicas: 1 + image: + registry: docker.io + repository: kreinecke/fdb-document-layer + tag: 1.6.3-k8s + pullPolicy: Always + service: + name: fdbdoclayer + port: 27016 + +# Installs networkPolicy for the services. This will allow communication betweeen the services in +# environments where the default policy is deny +networkPolicy: + enabled: false \ No newline at end of file diff --git a/chart/monocular/requirements.lock b/chart/monocular/requirements.lock index c38c11992..7317e7025 100644 --- a/chart/monocular/requirements.lock +++ b/chart/monocular/requirements.lock @@ -2,5 +2,8 @@ dependencies: - name: mongodb repository: https://kubernetes-charts.storage.googleapis.com version: 7.2.10 -digest: sha256:314dea8ef057af9180610aeec4be4491ab6e8779be41076411a93360e92a3f40 -generated: "2019-09-28T21:49:40.948575+01:00" +- name: fdbdoclayer + repository: file://./local-helm/fdbdoclayer/ + version: 0.1.0 +digest: sha256:1d62539fb9fab5c5d069d9d33e456ff46286c2d8630f05018683df180f8b31cd +generated: "2019-12-03T08:55:49.128951283Z" diff --git a/chart/monocular/requirements.yaml b/chart/monocular/requirements.yaml index afc958b02..ab4ee64c5 100644 --- a/chart/monocular/requirements.yaml +++ b/chart/monocular/requirements.yaml @@ -3,3 +3,7 @@ dependencies: version: 7.2.10 repository: https://kubernetes-charts.storage.googleapis.com condition: mongodb.enabled +- name: fdbdoclayer + version: 0.1.0 + repository: file://./local-helm/fdbdoclayer/ + condition: fdbserver.enabled \ No newline at end of file diff --git a/chart/monocular/templates/_helpers.tpl b/chart/monocular/templates/_helpers.tpl index 248a20676..f13f1b85f 100644 --- a/chart/monocular/templates/_helpers.tpl +++ b/chart/monocular/templates/_helpers.tpl @@ -30,6 +30,14 @@ Render image reference {{ .registry }}/{{ .repository }}:{{ .tag }} {{- end -}} +{{/* +Create a default fully qualified app name for the document layer. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "doclayer.fullname" -}} +{{- printf "%s-%s" .Release.Name "fdbdoclayer" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Sync job pod template */}} @@ -48,18 +56,24 @@ spec: image: {{ template "monocular.image" $global.Values.sync.image }} args: - sync + - --debug={{ default false $global.Values.debug }} - --user-agent-comment=monocular/{{ $global.Chart.AppVersion }} - {{- if and $global.Values.global.mongoUrl (not $global.Values.mongodb.enabled) }} + {{- if and $global.Values.global.mongoUrl (and (not $global.Values.mongodb.enabled) (not $global.Values.fdbserver.enabled))}} - --mongo-url={{ $global.Values.global.mongoUrl }} + - --db-type=mongodb + {{- else if $global.Values.fdbserver.enabled}} + - --doclayer-url=mongodb://{{ template "doclayer.fullname" $global }}:27016 + - --db-type=fdb {{- else }} - --mongo-url={{ template "mongodb.fullname" $global }} - --mongo-user=root + - --db-type=mongodb {{- end }} - {{ $repo.name }} - {{ $repo.url }} command: - /chart-repo - {{- if $global.Values.mongodb.enabled }} + {{- if or $global.Values.mongodb.enabled $global.Values.fdbserver.enabled}} env: - name: HTTP_PROXY value: {{ $global.Values.sync.httpProxy }} @@ -67,6 +81,8 @@ spec: value: {{ $global.Values.sync.httpsProxy }} - name: NO_PROXY value: {{ $global.Values.sync.noProxy }} + {{- end }} + {{- if not $global.Values.fdbserver.enabled }} - name: MONGO_PASSWORD valueFrom: secretKeyRef: diff --git a/chart/monocular/templates/chartsvc-deployment.yaml b/chart/monocular/templates/chartsvc-deployment.yaml index 60106330b..68d20b5be 100644 --- a/chart/monocular/templates/chartsvc-deployment.yaml +++ b/chart/monocular/templates/chartsvc-deployment.yaml @@ -29,17 +29,25 @@ spec: command: - /chartsvc args: - {{- if and .Values.global.mongoUrl (not .Values.mongodb.enabled) }} + - --debug={{ default false .Values.dbDebug }} + {{- if and .Values.global.mongoUrl (and (not .Values.mongodb.enabled) (not .Values.fdbserver.enabled))}} - --mongo-url={{ .Values.global.mongoUrl }} + - --db-type=mongodb + {{- else if .Values.fdbserver.enabled}} + - --doclayer-url=mongodb://{{ template "doclayer.fullname" . }}:27016 + - --db-type=fdb {{- else }} - - --mongo-user=root - --mongo-url={{ template "mongodb.fullname" . }} - env: + - --mongo-user=root + - --db-type=mongodb + {{- end }} + {{- if not .Values.fdbserver.enabled }} + env: - name: MONGO_PASSWORD valueFrom: secretKeyRef: - name: {{ template "mongodb.fullname" . }} key: mongodb-root-password + name: {{ template "mongodb.fullname" . }} {{- end }} ports: - name: http diff --git a/chart/monocular/values.yaml b/chart/monocular/values.yaml index bbb43e41a..9f84f08e9 100644 --- a/chart/monocular/values.yaml +++ b/chart/monocular/values.yaml @@ -1,9 +1,10 @@ sync: # Image used to perform chart repository syncs image: - registry: quay.io - repository: helmpack/chart-repo - tag: v1.9.0 + registry: docker.io + repository: kreinecke/chart-repo + tag: latest + pullPolicy: Always repos: # Official repositories - name: stable @@ -35,9 +36,10 @@ sync: # Chartsvc is used to serve chart metadata over a REST API. chartsvc: image: - registry: quay.io - repository: helmpack/chartsvc - tag: v1.9.0 + registry: docker.io + repository: kreinecke/chartsvc + tag: latest + pullPolicy: Always service: port: 8080 replicas: 3 @@ -148,8 +150,11 @@ mongodb: persistence: enabled: false +fdbserver: + enabled: false + # External MongoDB connection URL. -# This must be set if mongodb.enabled is set to false, following the pattern: +# This must be set if mongodb.enabled and foundationdb are set to false, following the pattern: # `mongodb://${MONGODB_USER}:${MONGODB_ROOT_PASSWORD}@${MONGODB_DNS}:${MONGODB_PORT}/${MONGODB_DATABASE}` # ref: https://docs.mongodb.com/manual/reference/connection-string/ global: @@ -165,3 +170,5 @@ securityContext: {} networkPolicy: enabled: false +# Enable debug output in chartsvc and cmd deployments +debug: true \ No newline at end of file diff --git a/cmd/chart-repo/Dockerfile b/cmd/chart-repo/Dockerfile index d1e7cc61a..cbbb6a7d6 100644 --- a/cmd/chart-repo/Dockerfile +++ b/cmd/chart-repo/Dockerfile @@ -1,10 +1,8 @@ FROM golang:1.12 as builder COPY . /go/src/github.com/helm/monocular WORKDIR /go/src/github.com/helm/monocular - ARG VERSION RUN GO111MODULE=on GOPROXY=https://gocenter.io CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-X main.version=$VERSION" ./cmd/chart-repo - FROM scratch COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /go/src/github.com/helm/monocular/chart-repo /chart-repo diff --git a/cmd/chart-repo/Makefile b/cmd/chart-repo/Makefile index 34ec0f9d1..727e1f4ca 100644 --- a/cmd/chart-repo/Makefile +++ b/cmd/chart-repo/Makefile @@ -1,4 +1,4 @@ -IMAGE_REPO ?= quay.io/helmpack/chart-repo +IMAGE_REPO ?= docker.io/kreinecke/chart-repo IMAGE_TAG ?= latest # Version of the binary to be produced VERSION ?= $$(git rev-parse HEAD) diff --git a/cmd/chart-repo/chart_repo.go b/cmd/chart-repo/chart_repo.go index a61b31784..6e57e8fc7 100644 --- a/cmd/chart-repo/chart_repo.go +++ b/cmd/chart-repo/chart_repo.go @@ -19,6 +19,8 @@ package main import ( "os" + "github.com/helm/monocular/cmd/chart-repo/utils" + "github.com/spf13/cobra" ) @@ -38,16 +40,27 @@ func main() { } func init() { - cmds := []*cobra.Command{syncCmd, deleteCmd} + + cmds := []*cobra.Command{SyncCmd, DeleteCmd} for _, cmd := range cmds { rootCmd.AddCommand(cmd) + + //Flag to configure running sync with either MongoDB or FoundationDB + cmd.Flags().String("db-type", "mongodb", "Database backend. Defaults to MongoDB if not specified.") + + //Flags for default mongoDB backend cmd.Flags().String("mongo-url", "localhost", "MongoDB URL (see https://godoc.org/github.com/globalsign/mgo#Dial for format)") cmd.Flags().String("mongo-database", "charts", "MongoDB database") cmd.Flags().String("mongo-user", "", "MongoDB user") + + //Flags for optional FoundationDB + Document Layer backend + cmd.Flags().String("doclayer-url", "mongodb://dev-fdbdoclayer/27016", "FoundationDB Document Layer URL") + cmd.Flags().String("doclayer-database", "charts", "FoundationDB Document-Layer database") + // see version.go - cmd.Flags().StringVarP(&userAgentComment, "user-agent-comment", "", "", "UserAgent comment used during outbound requests") + cmd.Flags().StringVarP(&utils.UserAgentComment, "user-agent-comment", "", "", "UserAgent comment used during outbound requests") cmd.Flags().Bool("debug", false, "verbose logging") } - rootCmd.AddCommand(versionCmd) + rootCmd.AddCommand(utils.VersionCmd) } diff --git a/cmd/chart-repo/common/chart_utils.go b/cmd/chart-repo/common/chart_utils.go new file mode 100644 index 000000000..163b88de1 --- /dev/null +++ b/cmd/chart-repo/common/chart_utils.go @@ -0,0 +1,204 @@ +/* +Copyright (c) 2019 The Helm Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "archive/tar" + "bytes" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "io/ioutil" + "github.com/helm/monocular/cmd/chart-repo/utils" + "net/http" + "net/url" + "os" + "path" + "strings" + "time" + + "github.com/ghodss/yaml" + "github.com/jinzhu/copier" + log "github.com/sirupsen/logrus" + helmrepo "k8s.io/helm/pkg/repo" +) + +//ParseRepoURL parses a repo URL string into a URL +func ParseRepoURL(repoURL string) (*url.URL, error) { + repoURL = strings.TrimSpace(repoURL) + return url.ParseRequestURI(repoURL) +} + +//FetchRepoIndex fetches the index.yaml for a repository +func FetchRepoIndex(r Repo, netClient HTTPClient) ([]byte, error) { + indexURL, err := ParseRepoURL(r.URL) + if err != nil { + log.WithFields(log.Fields{"url": r.URL}).WithError(err).Error("failed to parse URL") + return nil, err + } + indexURL.Path = path.Join(indexURL.Path, "index.yaml") + req, err := http.NewRequest("GET", indexURL.String(), nil) + if err != nil { + log.WithFields(log.Fields{"url": req.URL.String()}).WithError(err).Error("could not build repo index request") + return nil, err + } + + req.Header.Set("User-Agent", utils.UserAgent()) + if len(r.AuthorizationHeader) > 0 { + req.Header.Set("Authorization", r.AuthorizationHeader) + } + + res, err := netClient.Do(req) + if res != nil { + defer res.Body.Close() + } + if err != nil { + log.WithFields(log.Fields{"url": req.URL.String()}).WithError(err).Error("error requesting repo index") + return nil, err + } + + if res.StatusCode != http.StatusOK { + log.WithFields(log.Fields{"url": req.URL.String(), "status": res.StatusCode}).Error("error requesting repo index, are you sure this is a chart repository?") + return nil, errors.New("repo index request failed") + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + return body, nil +} + +//ParseRepoIndex parses a repository index into an IndexFile +func ParseRepoIndex(body []byte) (*helmrepo.IndexFile, error) { + var index helmrepo.IndexFile + err := yaml.Unmarshal(body, &index) + if err != nil { + return nil, err + } + index.SortEntries() + return &index, nil +} + +//ChartsFromIndex creates an array of Charts from a Repository IndexFile +//Deprecated charts are skipped +func ChartsFromIndex(index *helmrepo.IndexFile, r Repo) []Chart { + var charts []Chart + for _, entry := range index.Entries { + if entry[0].GetDeprecated() { + log.WithFields(log.Fields{"name": entry[0].GetName()}).Info("skipping deprecated chart") + continue + } + charts = append(charts, NewChart(entry, r)) + } + return charts +} + +// NewChart Takes an entry from the index and constructs a database representation of the +// object. +func NewChart(entry helmrepo.ChartVersions, r Repo) Chart { + var c Chart + copier.Copy(&c, entry[0]) + copier.Copy(&c.ChartVersions, entry) + c.Repo = r + c.ID = fmt.Sprintf("%s/%s", r.Name, c.Name) + return c +} + +//ExtractFilesFromTarball extracts the specified files from a tarball into a map +func ExtractFilesFromTarball(filenames []string, tarf *tar.Reader) (map[string]string, error) { + ret := make(map[string]string) + for { + header, err := tarf.Next() + if err == io.EOF { + break + } + if err != nil { + return ret, err + } + + for _, f := range filenames { + if strings.EqualFold(header.Name, f) { + var b bytes.Buffer + io.Copy(&b, tarf) + ret[f] = string(b.Bytes()) + break + } + } + } + return ret, nil +} + +//ChartTarballURL returns the URL for a given chart version +func ChartTarballURL(r Repo, cv ChartVersion) string { + source := cv.URLs[0] + if _, err := ParseRepoURL(source); err != nil { + // If the chart URL is not absolute, join with repo URL. It's fine if the + // URL we build here is invalid as we can catch this error when actually + // making the request + u, _ := url.Parse(r.URL) + u.Path = path.Join(u.Path, source) + return u.String() + } + return source +} + +// InitNetClient configures an HTTP client for making requests to repositories +func InitNetClient(additionalCA string, timeoutSeconds time.Duration) (*http.Client, error) { + // Get the SystemCertPool, continue with an empty pool on error + caCertPool, _ := x509.SystemCertPool() + if caCertPool == nil { + caCertPool = x509.NewCertPool() + } + + // If additionalCA exists, load it + if _, err := os.Stat(additionalCA); !os.IsNotExist(err) { + certs, err := ioutil.ReadFile(additionalCA) + if err != nil { + return nil, fmt.Errorf("Failed to append %s to RootCAs: %v", additionalCA, err) + } + + // Append our cert to the system pool + if ok := caCertPool.AppendCertsFromPEM(certs); !ok { + return nil, fmt.Errorf("Failed to append %s to RootCAs", additionalCA) + } + } + + // Return Transport for testing purposes + return &http.Client{ + Timeout: time.Second * timeoutSeconds, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: caCertPool, + }, + Proxy: http.ProxyFromEnvironment, + }, + }, nil +} + +//GetSha256 generates a SHA 256 hash for a given byte array +func GetSha256(src []byte) (string, error) { + f := bytes.NewReader(src) + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} diff --git a/cmd/chart-repo/types.go b/cmd/chart-repo/common/types.go similarity index 51% rename from cmd/chart-repo/types.go rename to cmd/chart-repo/common/types.go index 46b9e2730..5de9e80dd 100644 --- a/cmd/chart-repo/types.go +++ b/cmd/chart-repo/common/types.go @@ -1,5 +1,5 @@ /* -Copyright (c) 2018 The Helm Authors +Copyright (c) 2019 The Helm Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,37 +14,42 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package common import ( + "net/http" "time" ) -type repo struct { +//Repo holds information to identify a repository +type Repo struct { Name string URL string AuthorizationHeader string `bson:"-"` } -type maintainer struct { +//Maintainer describes the maintainer of a Chart +type Maintainer struct { Name string Email string } -type chart struct { +//Chart holds full descriptor of a Helm chart +type Chart struct { ID string `bson:"_id"` Name string - Repo repo + Repo Repo Description string Home string Keywords []string - Maintainers []maintainer + Maintainers []Maintainer Sources []string Icon string - ChartVersions []chartVersion + ChartVersions []ChartVersion } -type chartVersion struct { +//ChartVersion holds version information on a Chart +type ChartVersion struct { Version string AppVersion string Created time.Time @@ -52,17 +57,33 @@ type chartVersion struct { URLs []string } -type chartFiles struct { +//ChartFiles describes the chart values, readme, schema and digest components of a chart +type ChartFiles struct { ID string `bson:"_id"` Readme string Values string Schema string - Repo repo + Repo Repo Digest string } -type repoCheck struct { +//RepoCheck describes the state of a repository in terms its current checksum and last update time. +//It is used to determine whether or not to re-sync a respository. +type RepoCheck struct { ID string `bson:"_id"` LastUpdate time.Time `bson:"last_update"` Checksum string `bson:"checksum"` } + +//ImportChartFilesJob contains the information needed by an +//ImportWorker when import a chart from a repository +type ImportChartFilesJob struct { + Name string + Repo Repo + ChartVersion ChartVersion +} + +//HTTPClient defines a behaviour for making HTTP requests +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) +} diff --git a/cmd/chart-repo/delete.go b/cmd/chart-repo/delete.go index 31f549da4..73abdefbe 100644 --- a/cmd/chart-repo/delete.go +++ b/cmd/chart-repo/delete.go @@ -17,51 +17,35 @@ limitations under the License. package main import ( - "os" + "github.com/helm/monocular/cmd/chart-repo/foundationdb" + "github.com/helm/monocular/cmd/chart-repo/mongodb" - "github.com/kubeapps/common/datastore" - "github.com/sirupsen/logrus" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) -var deleteCmd = &cobra.Command{ +//DeleteCmd Delete a chart repository from Monocular +var DeleteCmd = &cobra.Command{ Use: "delete [REPO NAME]", Short: "delete a chart repository", Run: func(cmd *cobra.Command, args []string) { if len(args) != 1 { - logrus.Info("Need exactly one argument: [REPO NAME]") + log.Info("Need exactly one argument: [REPO NAME]") cmd.Help() return } - mongoURL, err := cmd.Flags().GetString("mongo-url") + dbType, err := cmd.Flags().GetString("db-type") if err != nil { - logrus.Fatal(err) - } - mongoDB, err := cmd.Flags().GetString("mongo-database") - if err != nil { - logrus.Fatal(err) - } - mongoUser, err := cmd.Flags().GetString("mongo-user") - if err != nil { - logrus.Fatal(err) - } - mongoPW := os.Getenv("MONGO_PASSWORD") - debug, err := cmd.Flags().GetBool("debug") - if err != nil { - logrus.Fatal(err) - } - if debug { - logrus.SetLevel(logrus.DebugLevel) - } - mongoConfig := datastore.Config{URL: mongoURL, Database: mongoDB, Username: mongoUser, Password: mongoPW} - dbSession, err := datastore.NewSession(mongoConfig) - if err != nil { - logrus.Fatalf("Can't connect to mongoDB: %v", err) - } - if err = deleteRepo(dbSession, args[0]); err != nil { - logrus.Fatalf("Can't delete chart repository %s from database: %v", args[0], err) + mongodb.Delete(cmd, args) } - logrus.Infof("Successfully deleted the chart repository %s from database", args[0]) + switch dbType { + case "mongodb": + mongodb.Delete(cmd, args) + case "fdb": + foundationdb.Delete(cmd, args) + default: + log.Fatalf("Unknown database type: %v. db-type, if set, must be either 'mongodb' or 'fdb'.", dbType) + } }, } diff --git a/cmd/chart-repo/foundationdb/datastore.go b/cmd/chart-repo/foundationdb/datastore.go new file mode 100644 index 000000000..b5eaf598c --- /dev/null +++ b/cmd/chart-repo/foundationdb/datastore.go @@ -0,0 +1,121 @@ +/* +Copyright (c) 2019 The Helm Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package foundationdb + +import ( + "context" + "fmt" + "time" + + log "github.com/sirupsen/logrus" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const defaultTimeout = 30 * time.Second + +// Config configures the database connection +type Config struct { + URL string + Database string + Timeout time.Duration +} + +// Client is an interface for a MongoDB client +type Client interface { + Database(name string) (Database, func()) +} + +// Database is an interface for accessing a MongoDB database +type Database interface { + Collection(name string) Collection +} + +// Collection is an interface for accessing a MongoDB collection +type Collection interface { + BulkWrite(ctxt context.Context, operations []mongo.WriteModel, options *options.BulkWriteOptions) (*mongo.BulkWriteResult, error) + DeleteMany(ctxt context.Context, filter interface{}, options *options.DeleteOptions) (*mongo.DeleteResult, error) + FindOne(ctxt context.Context, filter interface{}, result interface{}, options *options.FindOneOptions) error + InsertOne(ctxt context.Context, document interface{}, options *options.InsertOneOptions) (*mongo.InsertOneResult, error) + UpdateOne(ctxt context.Context, filter interface{}, update interface{}, options *options.UpdateOptions) (*mongo.UpdateResult, error) +} + +// mongoDatabase wraps an mongo.Database and implements Database +type mongoDatabase struct { + Database *mongo.Database +} + +// mongoClient wraps an mongo.Database and implements Database +type mongoClient struct { + Client *mongo.Client +} + +//NewDocLayerClient creates a mongoDB client using the given options +func NewDocLayerClient(ctx context.Context, options *options.ClientOptions) (Client, error) { + client, err := mongo.Connect(ctx, options) + return &mongoClient{client}, err +} + +//Database Creates a new interface for accessing the specified FDB Document-Layer database +func (c *mongoClient) Database(dbName string) (Database, func()) { + + db := &mongoDatabase{c.Client.Database(dbName)} + + return db, func() { + err := c.Client.Disconnect(context.Background()) + + if err != nil { + log.Fatal(err) + } + fmt.Println("Connection to MongoDB closed.") + } +} + +//Collection returns a reference to a given collection in the FDB Document-layer +func (d *mongoDatabase) Collection(name string) Collection { + return &mongoCollection{d.Database.Collection(name)} +} + +// mongoCollection wraps a mongo.Collection and implements Collection +type mongoCollection struct { + Collection *mongo.Collection +} + +func (c *mongoCollection) BulkWrite(ctxt context.Context, operations []mongo.WriteModel, options *options.BulkWriteOptions) (*mongo.BulkWriteResult, error) { + res, err := c.Collection.BulkWrite(ctxt, operations, options) + return res, err +} + +func (c *mongoCollection) DeleteMany(ctxt context.Context, filter interface{}, options *options.DeleteOptions) (*mongo.DeleteResult, error) { + res, err := c.Collection.DeleteMany(ctxt, filter, options) + return res, err +} + +func (c *mongoCollection) FindOne(ctxt context.Context, filter interface{}, result interface{}, options *options.FindOneOptions) error { + res := c.Collection.FindOne(ctxt, filter, options) + return res.Decode(result) +} + +func (c *mongoCollection) InsertOne(ctxt context.Context, document interface{}, options *options.InsertOneOptions) (*mongo.InsertOneResult, error) { + res, err := c.Collection.InsertOne(ctxt, document, options) + return res, err +} + +func (c *mongoCollection) UpdateOne(ctxt context.Context, filter interface{}, document interface{}, options *options.UpdateOptions) (*mongo.UpdateResult, error) { + res, err := c.Collection.UpdateOne(ctxt, filter, document, options) + return res, err +} diff --git a/cmd/chart-repo/foundationdb/delete.go b/cmd/chart-repo/foundationdb/delete.go new file mode 100644 index 000000000..0db421964 --- /dev/null +++ b/cmd/chart-repo/foundationdb/delete.go @@ -0,0 +1,64 @@ +/* +Copyright (c) 2019 The Helm Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package foundationdb + +import ( + "context" + "time" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "go.mongodb.org/mongo-driver/mongo/options" +) + +//Delete Deletes a chart repository from FoundationDB Document-Layer +func Delete(cmd *cobra.Command, args []string) { + + fdbURL, err := cmd.Flags().GetString("doclayer-url") + if err != nil { + log.Fatal(err) + } + fDB, err := cmd.Flags().GetString("doclayer-database") + if err != nil { + log.Fatal(err) + } + debug, err := cmd.Flags().GetBool("debug") + if err != nil { + log.Fatal(err) + } + if debug { + log.SetLevel(log.DebugLevel) + } + + log.Debugf("Creating client for FDB: %v, %v, %v", fdbURL, fDB, debug) + clientOptions := options.Client().ApplyURI(fdbURL).SetMinPoolSize(10).SetMaxPoolSize(100) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + client, err := NewDocLayerClient(ctx, clientOptions) + if err != nil { + log.Fatalf("Can't create client for FoundationDB document layer: %v", err) + return + } + + log.Debugf("Client created.") + + if err = deleteRepo(client, fDB, args[0]); err != nil { + log.Fatalf("Can't delete chart repository %s from database: %v", args[0], err) + } + + log.Infof("Successfully deleted the chart repository %s from database", args[0]) +} diff --git a/cmd/chart-repo/foundationdb/mockstore.go b/cmd/chart-repo/foundationdb/mockstore.go new file mode 100644 index 000000000..eaac71db6 --- /dev/null +++ b/cmd/chart-repo/foundationdb/mockstore.go @@ -0,0 +1,82 @@ +/* +Copyright (c) 2019 The Helm Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package foundationdb + +import ( + "context" + + "github.com/stretchr/testify/mock" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +//mockDatabase acts as a mock datastore.Database +type mockDatabase struct { + *mock.Mock +} + +type mockClient struct { + *mock.Mock +} + +//NewMockClient returns a mocked Document-Layer client +func NewMockClient(m *mock.Mock) Client { + return mockClient{m} +} + +//Database returns a mocked datastore.Database and empty closer function +func (c mockClient) Database(dbName string) (Database, func()) { + + db := mockDatabase{c.Mock} + + return db, func() { + } +} + +func (d mockDatabase) Collection(name string) Collection { + return mockCollection{d.Mock} +} + +// mockCollection acts as a mock datastore.Collection +type mockCollection struct { + *mock.Mock +} + +func (c mockCollection) BulkWrite(ctxt context.Context, operations []mongo.WriteModel, options *options.BulkWriteOptions) (*mongo.BulkWriteResult, error) { + args := c.Called(ctxt, operations, options) + return args.Get(0).(*mongo.BulkWriteResult), args.Error(1) +} + +func (c mockCollection) DeleteMany(ctxt context.Context, filter interface{}, options *options.DeleteOptions) (*mongo.DeleteResult, error) { + args := c.Called(ctxt, filter, options) + return args.Get(0).(*mongo.DeleteResult), args.Error(1) +} + +func (c mockCollection) FindOne(ctxt context.Context, filter interface{}, result interface{}, options *options.FindOneOptions) error { + args := c.Called(ctxt, filter, result, options) + return args.Error(0) +} + +func (c mockCollection) InsertOne(ctxt context.Context, document interface{}, options *options.InsertOneOptions) (*mongo.InsertOneResult, error) { + args := c.Called(ctxt, document, options) + return args.Get(0).(*mongo.InsertOneResult), args.Error(1) +} + +func (c mockCollection) UpdateOne(ctxt context.Context, filter interface{}, document interface{}, options *options.UpdateOptions) (*mongo.UpdateResult, error) { + args := c.Called(ctxt, filter, document, options) + return args.Get(0).(*mongo.UpdateResult), args.Error(1) +} diff --git a/cmd/chart-repo/foundationdb/sync.go b/cmd/chart-repo/foundationdb/sync.go new file mode 100644 index 000000000..5f5feab17 --- /dev/null +++ b/cmd/chart-repo/foundationdb/sync.go @@ -0,0 +1,68 @@ +/* +Copyright (c) 2018 The Helm Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package foundationdb + +import ( + "context" + "os" + "time" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "go.mongodb.org/mongo-driver/mongo/options" +) + +//Sync Add a new chart repository to FoundationDB Document-Layer and periodically sync it +func Sync(cmd *cobra.Command, args []string) { + + fdbURL, err := cmd.Flags().GetString("doclayer-url") + if err != nil { + log.Fatal(err) + } + fDB, err := cmd.Flags().GetString("doclayer-database") + if err != nil { + log.Fatal(err) + } + debug, err := cmd.Flags().GetBool("debug") + if err != nil { + log.Fatal(err) + } + if debug { + log.SetLevel(log.DebugLevel) + } + + log.Debugf("Creating client for FDB: %v, %v, %v", fdbURL, fDB, debug) + clientOptions := options.Client().ApplyURI(fdbURL).SetMinPoolSize(10).SetMaxPoolSize(100) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + client, err := NewDocLayerClient(ctx, clientOptions) + if err != nil { + log.Fatalf("Can't create client for FoundationDB document layer: %v", err) + return + } + + log.Debugf("Client created.") + + startTime := time.Now() + authorizationHeader := os.Getenv("AUTHORIZATION_HEADER") + if err = syncRepo(client, fDB, args[0], args[1], authorizationHeader); err != nil { + log.Fatalf("Can't add chart repository to database: %v", err) + return + } + timeTaken := time.Since(startTime).Seconds() + log.Infof("Successfully added the chart repository %s to database in %v seconds", args[0], timeTaken) +} diff --git a/cmd/chart-repo/foundationdb/utils.go b/cmd/chart-repo/foundationdb/utils.go new file mode 100644 index 000000000..e2a67ea38 --- /dev/null +++ b/cmd/chart-repo/foundationdb/utils.go @@ -0,0 +1,463 @@ +/* +Copyright (c) 2019 The Helm Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package foundationdb + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io/ioutil" + "net/http" + "strings" + "sync" + "time" + + "github.com/helm/monocular/cmd/chart-repo/common" + "github.com/helm/monocular/cmd/chart-repo/utils" + + "github.com/disintegration/imaging" + log "github.com/sirupsen/logrus" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const ( + chartCollection = "charts" + repositoryCollection = "repos" + chartFilesCollection = "files" + defaultTimeoutSeconds = 10 + additionalCAFile = "/usr/local/share/ca-certificates/ca.crt" +) + +var netClient common.HTTPClient = &http.Client{} + +func init() { + var err error + netClient, err = common.InitNetClient(additionalCAFile, defaultTimeoutSeconds) + if err != nil { + log.Fatal(err) + } +} + +// SyncRepo Syncing is performed in the following steps: +// 1. Update database to match chart metadata from index +// 2. Concurrently process icons for charts (concurrently) +// 3. Concurrently process the README and values.yaml for the latest chart version of each chart +// 4. Concurrently process READMEs and values.yaml for historic chart versions +// +// These steps are processed in this way to ensure relevant chart data is +// imported into the database as fast as possible. E.g. we want all icons for +// charts before fetching readmes for each chart and version pair. +func syncRepo(dbClient Client, dbName, repoName, repoURL string, authorizationHeader string) error { + + db, closer := dbClient.Database(dbName) + defer closer() + + url, err := common.ParseRepoURL(repoURL) + if err != nil { + log.WithFields(log.Fields{"url": repoURL}).WithError(err).Error("failed to parse URL") + return err + } + + log.Debugf("Checking database connection and readiness...") + collection := db.Collection("numbers") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + res, err := collection.InsertOne(ctx, bson.M{"name": "pi", "value": 3.14159}, options.InsertOne()) + if err != nil { + log.Fatalf("Database readiness test failed: %v", err) + cancel() + return err + } + id := res.InsertedID + cancel() + log.Debugf("Database connection test successful.") + log.Debugf("Inserted a test document to test collection with ID: %v", id) + + r := common.Repo{Name: repoName, URL: url.String(), AuthorizationHeader: authorizationHeader} + repoBytes, err := common.FetchRepoIndex(r, netClient) + if err != nil { + return err + } + + repoChecksum, err := common.GetSha256(repoBytes) + if err != nil { + return err + } + + // Check if the repo has been already processed + if repoAlreadyProcessed(db, repoName, repoChecksum) { + log.WithFields(log.Fields{"url": repoURL}).Info("Skipping repository since there are no updates") + return nil + } + + index, err := common.ParseRepoIndex(repoBytes) + if err != nil { + return err + } + + charts := common.ChartsFromIndex(index, r) + log.Debugf("%v Charts in index of repo: %v", len(charts), repoURL) + if len(charts) == 0 { + return errors.New("no charts in repository index") + } + + err = importCharts(db, dbName, charts) + if err != nil { + return err + } + + // Process 10 charts at a time + numWorkers := 10 + iconJobs := make(chan common.Chart, numWorkers) + chartFilesJobs := make(chan common.ImportChartFilesJob, numWorkers) + var wg sync.WaitGroup + + log.Debugf("starting %d workers", numWorkers) + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go importWorker(db, &wg, iconJobs, chartFilesJobs) + } + + // Enqueue jobs to process chart icons + for _, c := range charts { + iconJobs <- c + } + // Close the iconJobs channel to signal the worker pools to move on to the + // chart files jobs + close(iconJobs) + + // Iterate through the list of charts and enqueue the latest chart version to + // be processed. Append the rest of the chart versions to a list to be + // enqueued later + var toEnqueue []common.ImportChartFilesJob + for _, c := range charts { + chartFilesJobs <- common.ImportChartFilesJob{Name: c.Name, Repo: c.Repo, ChartVersion: c.ChartVersions[0]} + for _, cv := range c.ChartVersions[1:] { + toEnqueue = append(toEnqueue, common.ImportChartFilesJob{Name: c.Name, Repo: c.Repo, ChartVersion: cv}) + } + } + + // Enqueue all the remaining chart versions + for _, cfj := range toEnqueue { + chartFilesJobs <- cfj + } + // Close the chartFilesJobs channel to signal the worker pools that there are + // no more jobs to process + close(chartFilesJobs) + + // Wait for the worker pools to finish processing + wg.Wait() + + // Update cache in the database + if err = updateLastCheck(db, repoName, repoChecksum, time.Now()); err != nil { + return err + } + log.WithFields(log.Fields{"url": repoURL}).Info("Stored repository update in cache") + + return nil +} + +func repoAlreadyProcessed(db Database, repoName string, checksum string) bool { + lastCheck := &common.RepoCheck{} + filter := bson.M{"_id": repoName} + err := db.Collection(repositoryCollection).FindOne(context.Background(), filter, lastCheck, options.FindOne()) + return err == nil && checksum == lastCheck.Checksum +} + +func updateLastCheck(db Database, repoName string, checksum string, now time.Time) error { + selector := bson.M{"_id": repoName} + update := bson.M{"$set": bson.M{"last_update": now, "checksum": checksum}} + _, err := db.Collection(repositoryCollection).UpdateOne(context.Background(), selector, update, options.Update()) + return err +} + +func deleteRepo(dbClient Client, dbName, repoName string) error { + db, closer := dbClient.Database(dbName) + defer closer() + collection := db.Collection(chartCollection) + filter := bson.M{ + "repo.name": repoName, + } + deleteResult, err := collection.DeleteMany(context.Background(), filter, options.Delete()) + if err != nil { + log.Debugf("Error occurred during delete repo (deleting charts from index). Err: %v, Result: %v", err, deleteResult) + return err + } + log.Debugf("Repo delete (delete charts from index) result: %v charts deleted", deleteResult.DeletedCount) + + collection = db.Collection(chartFilesCollection) + deleteResult, err = collection.DeleteMany(context.Background(), filter, options.Delete()) + if err != nil { + log.Debugf("Error occurred during delete repo (deleting chart files from index). Err: %v, Result: %v", err, deleteResult) + return err + } + log.Debugf("Repo delete (delete chart files from index) result: %v chart files deleted.", deleteResult.DeletedCount) + collection = db.Collection(repositoryCollection) + deleteResult, err = collection.DeleteMany(context.Background(), filter, options.Delete()) + if err != nil { + log.Debugf("Error occurred during delete repo (deleting repositories from index). Err: %v, Result: %v", err, deleteResult) + return err + } + log.Debugf("Repo delete (delete chart files from index) result: %v repositories deleted.", deleteResult.DeletedCount) + + return err +} + +func importCharts(db Database, dbName string, charts []common.Chart) error { + var operations []mongo.WriteModel + var chartIDs []string + for _, c := range charts { + operation := mongo.NewUpdateOneModel() + chartIDs = append(chartIDs, c.ID) + // charts to upsert - pair of filter, chart + operation.SetFilter(bson.M{ + "_id": c.ID, + }) + + chartBSON, err := bson.Marshal(&c) + var doc bson.M + bson.Unmarshal(chartBSON, &doc) + delete(doc, "_id") + + if err != nil { + log.Debugf("Error marshalling chart to BSON: %v. Skipping this chart.", err) + } else { + update := doc + operation.SetUpdate(update) + operation.SetUpsert(true) + operations = append(operations, operation) + } + log.Debugf("Adding chart insert operation for chart: %v", c.ID) + } + + //Must use bulk write for array of filters + collection := db.Collection(chartCollection) + updateResult, err := collection.BulkWrite( + context.Background(), + operations, + options.BulkWrite(), + ) + + //Set upsert flag and upsert the pairs here + //Updates our index for charts that we already have and inserts charts that are new + if err != nil { + log.Debugf("Error occurred during chart import (upsert many). Err: %v", err) + return err + } + log.Debugf("Upsert chart index success. %v documents inserted, %v documents upserted, %v documents modified", updateResult.InsertedCount, updateResult.UpsertedCount, updateResult.ModifiedCount) + + //Remove from our index, any charts that no longer exist + filter := bson.M{ + "_id": bson.M{ + "$nin": chartIDs, + }, + "repo.name": charts[0].Repo.Name, + } + deleteResult, err := collection.DeleteMany(context.Background(), filter, options.Delete()) + if err != nil { + log.Debugf("Error occurred during chart import (delete many). Err: %v", err) + return err + } + log.Debugf("Delete stale charts from index success. %v documents deleted.", deleteResult.DeletedCount) + + return err +} + +func importWorker(db Database, wg *sync.WaitGroup, icons <-chan common.Chart, chartFiles <-chan common.ImportChartFilesJob) { + defer wg.Done() + for c := range icons { + log.WithFields(log.Fields{"name": c.Name}).Debug("importing icon") + if err := fetchAndImportIcon(db, c); err != nil { + log.WithFields(log.Fields{"name": c.Name}).WithError(err).Error("failed to import icon") + } + } + for j := range chartFiles { + log.WithFields(log.Fields{"name": j.Name, "version": j.ChartVersion.Version}).Debug("importing readme and values") + if err := fetchAndImportFiles(db, j.Name, j.Repo, j.ChartVersion); err != nil { + log.WithFields(log.Fields{"name": j.Name, "version": j.ChartVersion.Version}).WithError(err).Error("failed to import files") + } + } +} + +func fetchAndImportIcon(db Database, c common.Chart) error { + if c.Icon == "" { + log.WithFields(log.Fields{"name": c.Name}).Info("icon not found") + return nil + } + + req, err := http.NewRequest("GET", c.Icon, nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", utils.UserAgent()) + if len(c.Repo.AuthorizationHeader) > 0 { + req.Header.Set("Authorization", c.Repo.AuthorizationHeader) + } + + res, err := netClient.Do(req) + if res != nil { + defer res.Body.Close() + } + if err != nil { + return err + } + + if res.StatusCode != http.StatusOK { + return fmt.Errorf("%d %s", res.StatusCode, c.Icon) + } + + b := []byte{} + contentType := "" + if strings.Contains(res.Header.Get("Content-Type"), "image/svg") { + // if the icon is a SVG file simply read it + b, err = ioutil.ReadAll(res.Body) + if err != nil { + return err + } + contentType = res.Header.Get("Content-Type") + } else { + // if the icon is in any other format try to convert it to PNG + orig, err := imaging.Decode(res.Body) + if err != nil { + log.WithFields(log.Fields{"name": c.Name}).WithError(err).Error("failed to decode icon") + return err + } + + // TODO: make this configurable? + icon := imaging.Fit(orig, 160, 160, imaging.Lanczos) + + var buf bytes.Buffer + imaging.Encode(&buf, icon, imaging.PNG) + b = buf.Bytes() + contentType = "image/png" + } + + collection := db.Collection(chartCollection) + //Update single icon + update := bson.M{"$set": bson.M{"raw_icon": b, "icon_content_type": contentType}} + filter := bson.M{"_id": c.ID} + updateResult, err := collection.UpdateOne(context.Background(), filter, update, options.Update()) + if err != nil { + log.Debugf("Error occurred during chart icon import (update one). Err: %v, Result: %v", err, updateResult) + return err + } + return err +} + +func fetchAndImportFiles(db Database, name string, r common.Repo, cv common.ChartVersion) error { + + chartFilesID := fmt.Sprintf("%s/%s-%s", r.Name, name, cv.Version) + //Check if we already have indexed files for this chart version and digest + collection := db.Collection(chartFilesCollection) + filter := bson.M{"_id": chartFilesID, "digest": cv.Digest} + findResult := collection.FindOne(context.Background(), filter, &common.ChartFiles{}, options.FindOne()) + if findResult != mongo.ErrNoDocuments { + log.WithFields(log.Fields{"name": name, "version": cv.Version}).Debug("skipping existing files") + return nil + } + log.WithFields(log.Fields{"name": name, "version": cv.Version}).Debug("fetching files") + + url := common.ChartTarballURL(r, cv) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", utils.UserAgent()) + if len(r.AuthorizationHeader) > 0 { + req.Header.Set("Authorization", r.AuthorizationHeader) + } + + res, err := netClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + + // We read the whole chart into memory, this should be okay since the chart + // tarball needs to be small enough to fit into a GRPC call (Tiller + // requirement) + gzf, err := gzip.NewReader(res.Body) + if err != nil { + return err + } + defer gzf.Close() + + tarf := tar.NewReader(gzf) + + readmeFileName := name + "/README.md" + valuesFileName := name + "/values.yaml" + schemaFileName := name + "/values.schema.json" + filenames := []string{valuesFileName, readmeFileName, schemaFileName} + + files, err := common.ExtractFilesFromTarball(filenames, tarf) + if err != nil { + return err + } + + chartFiles := common.ChartFiles{ID: chartFilesID, Repo: r, Digest: cv.Digest} + if v, ok := files[readmeFileName]; ok { + chartFiles.Readme = v + } else { + log.WithFields(log.Fields{"name": name, "version": cv.Version}).Info("README.md not found") + } + if v, ok := files[valuesFileName]; ok { + chartFiles.Values = v + } else { + log.WithFields(log.Fields{"name": name, "version": cv.Version}).Info("values.yaml not found") + } + if v, ok := files[schemaFileName]; ok { + chartFiles.Schema = v + } else { + log.WithFields(log.Fields{"name": name, "version": cv.Version}).Info("values.schema.json not found") + } + + // inserts the chart files if not already indexed, or updates the existing + // entry if digest has changed + log.Debugf("Inserting chart files %v to collection: %v....", chartFilesID, chartFilesCollection) + collection = db.Collection(chartFilesCollection) + filter = bson.M{"_id": chartFilesID} + chartBSON, err := bson.Marshal(&chartFiles) + var doc bson.M + bson.Unmarshal(chartBSON, &doc) + delete(doc, "_id") + update := bson.M{"$set": doc} + updateResult, err := collection.UpdateOne(context.Background(), filter, update, options.Update().SetUpsert(true)) + if err != nil { + log.Debugf("Error occurred during chart files import (update one). Chart files : %v doc: %v Err: %v", chartFiles, doc, err) + return err + } + log.Debugf("Chart files import success. (update one) Upserted: %v Updated: %v", updateResult.UpsertedCount, updateResult.ModifiedCount) + return nil +} + +func database(client *mongo.Client, dbName string) (*mongo.Database, func()) { + + db := client.Database(dbName) + return db, func() { + err := client.Disconnect(context.Background()) + + if err != nil { + log.Fatal(err) + } + fmt.Println("Connection to MongoDB closed.") + } +} diff --git a/cmd/chart-repo/foundationdb/utils_test.go b/cmd/chart-repo/foundationdb/utils_test.go new file mode 100644 index 000000000..d8c450b13 --- /dev/null +++ b/cmd/chart-repo/foundationdb/utils_test.go @@ -0,0 +1,690 @@ +/* +Copyright (c) 2018 The Helm Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package foundationdb + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/rand" + "fmt" + "image" + "image/color" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "path" + "strings" + "testing" + "time" + + "github.com/helm/monocular/cmd/chart-repo/common" + "github.com/helm/monocular/cmd/chart-repo/utils" + + "github.com/arschles/assert" + "github.com/disintegration/imaging" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/mock" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +var validRepoIndexYAMLBytes, _ = ioutil.ReadFile("../testdata/valid-index.yaml") +var validRepoIndexYAML = string(validRepoIndexYAMLBytes) + +var invalidRepoIndexYAML = "invalid" + +type badHTTPClient struct{} + +func (h *badHTTPClient) Do(req *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + w.WriteHeader(500) + return w.Result(), nil +} + +type goodHTTPClient struct{} + +func (h *goodHTTPClient) Do(req *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + // Don't accept trailing slashes + if strings.HasPrefix(req.URL.Path, "//") { + w.WriteHeader(500) + } + // If subpath repo URL test, check that index.yaml is correctly added to the + // subpath + if req.URL.Host == "subpath.test" && req.URL.Path != "/subpath/index.yaml" { + w.WriteHeader(500) + } + + w.Write([]byte(validRepoIndexYAML)) + return w.Result(), nil +} + +type badIconClient struct{} + +func (h *badIconClient) Do(req *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + w.Write([]byte("not-an-image")) + return w.Result(), nil +} + +type goodIconClient struct{} + +func iconBytes() []byte { + var b bytes.Buffer + img := imaging.New(1, 1, color.White) + imaging.Encode(&b, img, imaging.PNG) + return b.Bytes() +} + +func (h *goodIconClient) Do(req *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + w.Write(iconBytes()) + return w.Result(), nil +} + +type svgIconClient struct{} + +func (h *svgIconClient) Do(req *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + w.Write([]byte("foo")) + res := w.Result() + res.Header.Set("Content-Type", "image/svg") + return res, nil +} + +type goodTarballClient struct { + c common.Chart + skipReadme bool + skipValues bool + skipSchema bool +} + +var testChartReadme = "# readme for chart\n\nBest chart in town" +var testChartValues = "image: test" +var testChartSchema = `{"properties": {}}` + +func (h *goodTarballClient) Do(req *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + gzw := gzip.NewWriter(w) + files := []tarballFile{{h.c.Name + "/Chart.yaml", "should be a Chart.yaml here..."}} + if !h.skipValues { + files = append(files, tarballFile{h.c.Name + "/values.yaml", testChartValues}) + } + if !h.skipReadme { + files = append(files, tarballFile{h.c.Name + "/README.md", testChartReadme}) + } + if !h.skipSchema { + files = append(files, tarballFile{h.c.Name + "/values.schema.json", testChartSchema}) + } + createTestTarball(gzw, files) + gzw.Flush() + return w.Result(), nil +} + +type authenticatedTarballClient struct { + c common.Chart +} + +func (h *authenticatedTarballClient) Do(req *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + + // Ensure we're sending the right Authorization header + if !strings.Contains(req.Header.Get("Authorization"), "Bearer ThisSecretAccessTokenAuthenticatesTheClient") { + w.WriteHeader(500) + } else { + gzw := gzip.NewWriter(w) + files := []tarballFile{{h.c.Name + "/Chart.yaml", "should be a Chart.yaml here..."}} + files = append(files, tarballFile{h.c.Name + "/values.yaml", testChartValues}) + files = append(files, tarballFile{h.c.Name + "/README.md", testChartReadme}) + files = append(files, tarballFile{h.c.Name + "/values.schema.json", testChartSchema}) + createTestTarball(gzw, files) + gzw.Flush() + } + return w.Result(), nil +} + +func Test_syncURLInvalidity(t *testing.T) { + tests := []struct { + name string + repoURL string + }{ + {"invalid URL", "not-a-url"}, + {"invalid URL", "https//google.com"}, + } + m := mock.Mock{} + dbClient := NewMockClient(&m) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := syncRepo(dbClient, "test", "test", tt.repoURL, "") + assert.ExistsErr(t, err, tt.name) + }) + } +} + +func Test_fetchRepoIndex(t *testing.T) { + tests := []struct { + name string + r common.Repo + }{ + {"valid HTTP URL", common.Repo{URL: "http://my.examplerepo.com"}}, + {"valid HTTPS URL", common.Repo{URL: "https://my.examplerepo.com"}}, + {"valid trailing URL", common.Repo{URL: "https://my.examplerepo.com/"}}, + {"valid subpath URL", common.Repo{URL: "https://subpath.test/subpath/"}}, + {"valid URL with trailing spaces", common.Repo{URL: "https://subpath.test/subpath/ "}}, + {"valid URL with leading spaces", common.Repo{URL: " https://subpath.test/subpath/"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + netClient = &goodHTTPClient{} + _, err := common.FetchRepoIndex(tt.r, netClient) + assert.NoErr(t, err) + }) + } + + t.Run("failed request", func(t *testing.T) { + netClient = &badHTTPClient{} + _, err := common.FetchRepoIndex(common.Repo{URL: "https://my.examplerepo.com"}, netClient) + assert.ExistsErr(t, err, "failed request") + }) +} + +func Test_fetchRepoIndexUserAgent(t *testing.T) { + tests := []struct { + name string + version string + userAgentComment string + expectedUserAgent string + }{ + {"default user agent", "", "", "chart-repo/devel"}, + {"custom version no app", "1.0", "", "chart-repo/1.0"}, + {"custom version and app", "1.0", "monocular/1.2", "chart-repo/1.0 (monocular/1.2)"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Override global variables used to generate the userAgent + if tt.version != "" { + utils.Version = tt.version + } + + if tt.userAgentComment != "" { + utils.UserAgentComment = tt.userAgentComment + } + + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + assert.Equal(t, tt.expectedUserAgent, req.Header.Get("User-Agent"), "expected user agent") + rw.Write([]byte(validRepoIndexYAML)) + })) + // Close the server when test finishes + defer server.Close() + + netClient = server.Client() + + _, err := common.FetchRepoIndex(common.Repo{URL: server.URL}, netClient) + assert.NoErr(t, err) + }) + } +} + +func Test_parseRepoIndex(t *testing.T) { + tests := []struct { + name string + repoYAML string + }{ + {"invalid", "invalid"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := common.ParseRepoIndex([]byte(tt.repoYAML)) + assert.ExistsErr(t, err, tt.name) + }) + } + + t.Run("valid", func(t *testing.T) { + index, err := common.ParseRepoIndex([]byte(validRepoIndexYAML)) + assert.NoErr(t, err) + assert.Equal(t, len(index.Entries), 2, "number of charts") + assert.Equal(t, index.Entries["acs-engine-autoscaler"][0].GetName(), "acs-engine-autoscaler", "chart version populated") + }) +} + +func Test_chartsFromIndex(t *testing.T) { + r := common.Repo{Name: "test", URL: "http://testrepo.com"} + index, _ := common.ParseRepoIndex([]byte(validRepoIndexYAML)) + charts := common.ChartsFromIndex(index, r) + assert.Equal(t, len(charts), 2, "number of charts") + + indexWithDeprecated := validRepoIndexYAML + ` + deprecated-chart: + - name: deprecated-chart + deprecated: true` + index2, err := common.ParseRepoIndex([]byte(indexWithDeprecated)) + assert.NoErr(t, err) + charts = common.ChartsFromIndex(index2, r) + assert.Equal(t, len(charts), 2, "number of charts") +} + +func Test_newChart(t *testing.T) { + r := common.Repo{Name: "test", URL: "http://testrepo.com"} + index, _ := common.ParseRepoIndex([]byte(validRepoIndexYAML)) + c := common.NewChart(index.Entries["wordpress"], r) + assert.Equal(t, c.Name, "wordpress", "correctly built") + assert.Equal(t, len(c.ChartVersions), 2, "correctly built") + assert.Equal(t, c.Description, "new description!", "takes chart fields from latest entry") + assert.Equal(t, c.Repo, r, "repo set") + assert.Equal(t, c.ID, "test/wordpress", "id set") +} + +func Test_importCharts(t *testing.T) { + m := &mock.Mock{} + // Ensure Upsert func is called with some arguments + m.On("BulkWrite", mock.Anything, mock.Anything, mock.Anything).Return(&mongo.BulkWriteResult{}, nil) + m.On("DeleteMany", mock.Anything, mock.Anything, mock.Anything).Return(&mongo.DeleteResult{}, nil) + dbClient := NewMockClient(m) + db, _ := dbClient.Database("test") + index, _ := common.ParseRepoIndex([]byte(validRepoIndexYAML)) + charts := common.ChartsFromIndex(index, common.Repo{Name: "test", URL: "http://testrepo.com"}) + importCharts(db, "test", charts) + + m.AssertExpectations(t) + // The BulkWrite method takes an array of WriteModels. + // For x charts to upsert, there should be x elements. + // Each element has a selector (filter) and an "update" - the update document to apply + args := m.Calls[0].Arguments.Get(1).([]mongo.WriteModel) + assert.Equal(t, len(args), len(charts), "number of charts to upsert") + for i := 0; i < len(args); i++ { + updateModel := args[i].(*mongo.UpdateOneModel) + assert.Equal(t, updateModel.Filter, bson.M{"_id": "test/" + updateModel.Update.(bson.M)["name"].(string)}, "selector") + } +} + +func Test_DeleteRepo(t *testing.T) { + m := &mock.Mock{} + m.On("DeleteMany", mock.Anything, bson.M{ + "repo.name": "test", + }, mock.Anything).Return(&mongo.DeleteResult{}, nil) + dbClient := NewMockClient(m) + + err := deleteRepo(dbClient, "test", "test") + if err != nil { + t.Errorf("failed to delete chart repo test: %v", err) + } + m.AssertExpectations(t) +} + +func Test_fetchAndImportIcon(t *testing.T) { + t.Run("no icon", func(t *testing.T) { + m := mock.Mock{} + dbClient := NewMockClient(&m) + db, _ := dbClient.Database("test") + c := common.Chart{ID: "test/acs-engine-autoscaler"} + assert.NoErr(t, fetchAndImportIcon(db, c)) + }) + + index, _ := common.ParseRepoIndex([]byte(validRepoIndexYAML)) + charts := common.ChartsFromIndex(index, common.Repo{Name: "test", URL: "http://testrepo.com"}) + + t.Run("failed download", func(t *testing.T) { + netClient = &badHTTPClient{} + c := charts[0] + m := mock.Mock{} + dbClient := NewMockClient(&m) + db, _ := dbClient.Database("test") + assert.Err(t, fmt.Errorf("500 %s", c.Icon), fetchAndImportIcon(db, c)) + }) + + t.Run("bad icon", func(t *testing.T) { + netClient = &badIconClient{} + c := charts[0] + m := mock.Mock{} + dbClient := NewMockClient(&m) + db, _ := dbClient.Database("test") + assert.Err(t, image.ErrFormat, fetchAndImportIcon(db, c)) + }) + + t.Run("valid icon", func(t *testing.T) { + netClient = &goodIconClient{} + c := charts[0] + m := mock.Mock{} + dbClient := NewMockClient(&m) + db, _ := dbClient.Database("test") + m.On("UpdateOne", mock.Anything, bson.M{"_id": c.ID}, bson.M{"$set": bson.M{"raw_icon": iconBytes(), "icon_content_type": "image/png"}}, mock.Anything).Return(&mongo.UpdateResult{}, nil) + assert.NoErr(t, fetchAndImportIcon(db, c)) + m.AssertExpectations(t) + }) + + t.Run("valid SVG icon", func(t *testing.T) { + netClient = &svgIconClient{} + c := common.Chart{ + ID: "foo", + Icon: "https://foo/bar/logo.svg", + Repo: common.Repo{}, + } + m := mock.Mock{} + dbClient := NewMockClient(&m) + db, _ := dbClient.Database("test") + m.On("UpdateOne", mock.Anything, bson.M{"_id": c.ID}, bson.M{"$set": bson.M{"raw_icon": []byte("foo"), "icon_content_type": "image/svg"}}, mock.Anything).Return(&mongo.UpdateResult{}, nil) + assert.NoErr(t, fetchAndImportIcon(db, c)) + m.AssertExpectations(t) + }) +} + +func Test_fetchAndImportFiles(t *testing.T) { + index, _ := common.ParseRepoIndex([]byte(validRepoIndexYAML)) + charts := common.ChartsFromIndex(index, common.Repo{Name: "test", URL: "http://testrepo.com", AuthorizationHeader: "Bearer ThisSecretAccessTokenAuthenticatesTheClient1s"}) + cv := charts[0].ChartVersions[0] + + t.Run("http error", func(t *testing.T) { + m := mock.Mock{} + m.On("FindOne", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(mongo.ErrNoDocuments) + dbClient := NewMockClient(&m) + db, _ := dbClient.Database("test") + netClient = &badHTTPClient{} + assert.Err(t, io.EOF, fetchAndImportFiles(db, charts[0].Name, charts[0].Repo, cv)) + }) + + t.Run("file not found", func(t *testing.T) { + netClient = &goodTarballClient{c: charts[0], skipValues: true, skipReadme: true, skipSchema: true} + m := mock.Mock{} + m.On("FindOne", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(mongo.ErrNoDocuments) + chartFilesID := fmt.Sprintf("%s/%s-%s", charts[0].Repo.Name, charts[0].Name, cv.Version) + chartFiles := common.ChartFiles{ID: chartFilesID, Readme: "", Values: "", Schema: "", Repo: charts[0].Repo, Digest: cv.Digest} + chartBSON, _ := bson.Marshal(&chartFiles) + var doc bson.M + bson.Unmarshal(chartBSON, &doc) + delete(doc, "_id") + update := bson.M{"$set": doc} + m.On("UpdateOne", mock.Anything, bson.M{"_id": chartFilesID}, update, mock.Anything).Return(&mongo.UpdateResult{}, nil) + dbClient := NewMockClient(&m) + db, _ := dbClient.Database("test") + err := fetchAndImportFiles(db, charts[0].Name, charts[0].Repo, cv) + assert.NoErr(t, err) + m.AssertExpectations(t) + }) + + t.Run("authenticated request", func(t *testing.T) { + netClient = &authenticatedTarballClient{c: charts[0]} + m := mock.Mock{} + m.On("FindOne", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(mongo.ErrNoDocuments) + chartFilesID := fmt.Sprintf("%s/%s-%s", charts[0].Repo.Name, charts[0].Name, cv.Version) + chartFiles := common.ChartFiles{ID: chartFilesID, Readme: testChartReadme, Values: testChartValues, Schema: testChartSchema, Repo: charts[0].Repo, Digest: cv.Digest} + chartBSON, _ := bson.Marshal(&chartFiles) + var doc bson.M + bson.Unmarshal(chartBSON, &doc) + delete(doc, "_id") + update := bson.M{"$set": doc} + m.On("UpdateOne", mock.Anything, bson.M{"_id": chartFilesID}, update, mock.Anything).Return(&mongo.UpdateResult{}, nil) + dbClient := NewMockClient(&m) + db, _ := dbClient.Database("test") + err := fetchAndImportFiles(db, charts[0].Name, charts[0].Repo, cv) + assert.NoErr(t, err) + m.AssertExpectations(t) + }) + + t.Run("valid tarball", func(t *testing.T) { + netClient = &goodTarballClient{c: charts[0]} + m := mock.Mock{} + m.On("FindOne", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(mongo.ErrNoDocuments) + chartFilesID := fmt.Sprintf("%s/%s-%s", charts[0].Repo.Name, charts[0].Name, cv.Version) + chartFiles := common.ChartFiles{ID: chartFilesID, Readme: testChartReadme, Values: testChartValues, Schema: testChartSchema, Repo: charts[0].Repo, Digest: cv.Digest} + chartBSON, _ := bson.Marshal(&chartFiles) + var doc bson.M + bson.Unmarshal(chartBSON, &doc) + delete(doc, "_id") + update := bson.M{"$set": doc} + m.On("UpdateOne", mock.Anything, bson.M{"_id": chartFilesID}, update, mock.Anything).Return(&mongo.UpdateResult{}, nil) + dbClient := NewMockClient(&m) + db, _ := dbClient.Database("test") + err := fetchAndImportFiles(db, charts[0].Name, charts[0].Repo, cv) + assert.NoErr(t, err) + m.AssertExpectations(t) + }) + + t.Run("file exists", func(t *testing.T) { + m := mock.Mock{} + // don't return an error when checking if files already exists + m.On("FindOne", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + dbClient := NewMockClient(&m) + db, _ := dbClient.Database("test") + err := fetchAndImportFiles(db, charts[0].Name, charts[0].Repo, cv) + assert.NoErr(t, err) + m.AssertNotCalled(t, "UpdateOne", mock.Anything, mock.Anything, mock.Anything) + }) +} + +func Test_chartTarballURL(t *testing.T) { + r := common.Repo{Name: "test", URL: "http://testrepo.com"} + tests := []struct { + name string + cv common.ChartVersion + wanted string + }{ + {"absolute url", common.ChartVersion{URLs: []string{"http://testrepo.com/wordpress-0.1.0.tgz"}}, "http://testrepo.com/wordpress-0.1.0.tgz"}, + {"relative url", common.ChartVersion{URLs: []string{"wordpress-0.1.0.tgz"}}, "http://testrepo.com/wordpress-0.1.0.tgz"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, common.ChartTarballURL(r, tt.cv), tt.wanted, "url") + }) + } +} + +func Test_extractFilesFromTarball(t *testing.T) { + tests := []struct { + name string + files []tarballFile + filename string + want string + }{ + {"file", []tarballFile{{"file.txt", "best file ever"}}, "file.txt", "best file ever"}, + {"multiple file tarball", []tarballFile{{"file.txt", "best file ever"}, {"file2.txt", "worst file ever"}}, "file2.txt", "worst file ever"}, + {"file in dir", []tarballFile{{"file.txt", "best file ever"}, {"test/file2.txt", "worst file ever"}}, "test/file2.txt", "worst file ever"}, + {"filename ignore case", []tarballFile{{"Readme.md", "# readme for chart"}, {"values.yaml", "key: value"}}, "README.md", "# readme for chart"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var b bytes.Buffer + createTestTarball(&b, tt.files) + r := bytes.NewReader(b.Bytes()) + tarf := tar.NewReader(r) + files, err := common.ExtractFilesFromTarball([]string{tt.filename}, tarf) + assert.NoErr(t, err) + assert.Equal(t, files[tt.filename], tt.want, "file body") + }) + } + + t.Run("extract multiple files", func(t *testing.T) { + var b bytes.Buffer + tFiles := []tarballFile{{"file.txt", "best file ever"}, {"file2.txt", "worst file ever"}} + createTestTarball(&b, tFiles) + r := bytes.NewReader(b.Bytes()) + tarf := tar.NewReader(r) + files, err := common.ExtractFilesFromTarball([]string{tFiles[0].Name, tFiles[1].Name}, tarf) + assert.NoErr(t, err) + assert.Equal(t, len(files), 2, "matches") + for _, f := range tFiles { + assert.Equal(t, files[f.Name], f.Body, "file body") + } + }) + + t.Run("file not found", func(t *testing.T) { + var b bytes.Buffer + createTestTarball(&b, []tarballFile{{"file.txt", "best file ever"}}) + r := bytes.NewReader(b.Bytes()) + tarf := tar.NewReader(r) + name := "file2.txt" + files, err := common.ExtractFilesFromTarball([]string{name}, tarf) + assert.NoErr(t, err) + assert.Equal(t, files[name], "", "file body") + }) + + t.Run("not a tarball", func(t *testing.T) { + b := make([]byte, 4) + rand.Read(b) + r := bytes.NewReader(b) + tarf := tar.NewReader(r) + files, err := common.ExtractFilesFromTarball([]string{"file2.txt"}, tarf) + assert.Err(t, io.ErrUnexpectedEOF, err) + assert.Equal(t, len(files), 0, "file body") + }) +} + +type tarballFile struct { + Name, Body string +} + +func createTestTarball(w io.Writer, files []tarballFile) { + // Create a new tar archive. + tarw := tar.NewWriter(w) + + // Add files to the archive. + for _, file := range files { + hdr := &tar.Header{ + Name: file.Name, + Mode: 0600, + Size: int64(len(file.Body)), + } + if err := tarw.WriteHeader(hdr); err != nil { + log.Fatalln(err) + } + if _, err := tarw.Write([]byte(file.Body)); err != nil { + log.Fatalln(err) + } + } + // Make sure to check the error on Close. + if err := tarw.Close(); err != nil { + log.Fatal(err) + } +} + +func Test_initNetClient(t *testing.T) { + // Test env + otherDir, err := ioutil.TempDir("", "ca-registry") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(otherDir) + + // Create cert + caCert := `-----BEGIN CERTIFICATE----- +MIIC6jCCAdKgAwIBAgIUKVfzA7lfBgSYP8enCVhlm0ql5YwwDQYJKoZIhvcNAQEL +BQAwDTELMAkGA1UEAxMCQ0EwHhcNMTgxMjEyMTQxNzAwWhcNMjMxMjExMTQxNzAw +WjANMQswCQYDVQQDEwJDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +ALZU3fsAgvoUuLSHr24apslaYyuX84wGoZQmtFtQ+A3DF9KL/2nn3yZ6qJPkH0TF +sbObEQRNi+P6vQ3nI/dSNMX5PzMBP2CB6L7zEXzZQEHtAK0Bzva5CKEBGX7OfIKl +aBvs+dzKVJBdb+Oh0maacMwa4QbcD6ejzF90jUbaO65lpQpcL7KQdppKOGNclRaA +hQTV2VsxrV4hH7K9btaTTxso+8W6p8v6X9vf40Ywx72p+SKnGh+FCrOp1gYLBLwo +4SM0OUQHRvqUlj0XhZk5pW0dMRwHcoz1S2GmE5bj4edr4j+zGzGxa2wRGKvM0OCn +Do84AVszTFPmUf+mCl4pJNECAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1Ud +EwEB/wQFMAMBAf8wHQYDVR0OBBYEFI5l5k+MEhrbOQ29dOW1qJhI0yKaMA0GCSqG +SIb3DQEBCwUAA4IBAQByDebUOKzn6jfmXlW62vm09V+ipqId01wm21G9XMtMEqhc +xtun6YwQeTuGPtdepWG+NXuSsiX/HNAHeaumJaaljHhdKDisnMQ0CTnNsu8NPkAl +9iMEB3iXLWkb7+HgfPJAHZVGcMqMxNEMZYHB1Fh0G2Ne376X94+GYJ08qR2C8rUP +BShhMSktB578h4GtPIWSjPhDUWg1fGe7sewR+GPyuL9859hOD0wGm9tUixBKloCu +b90fhqZZ3FqZD7W1qJGKvz/8geqi0noip+uq/dokK1jarRkOVEJP+EvXkHo0tIuc +h251U/Daz6NiQBM9AxyAw6EHm8XAZBvCuebfzyrT +-----END CERTIFICATE-----` + otherCA := path.Join(otherDir, "ca.crt") + err = ioutil.WriteFile(otherCA, []byte(caCert), 0644) + if err != nil { + t.Error(err) + } + + _, err = common.InitNetClient(otherCA, defaultTimeoutSeconds) + if err != nil { + t.Error(err) + } +} + +var emptyRepoIndexYAMLBytes, _ = ioutil.ReadFile("testdata/empty-repo-index.yaml") +var emptyRepoIndexYAML = string(emptyRepoIndexYAMLBytes) + +type emptyChartRepoHTTPClient struct{} + +func (h *emptyChartRepoHTTPClient) Do(req *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + w.Write([]byte(emptyRepoIndexYAML)) + return w.Result(), nil +} + +func Test_emptyChartRepo(t *testing.T) { + netClient = &emptyChartRepoHTTPClient{} + m := mock.Mock{} + dbClient := NewMockClient(&m) + //Expect a call to test the DB readiness + m.On("InsertOne", mock.Anything, mock.Anything, mock.Anything).Return(&mongo.InsertOneResult{}, nil) + m.On("FindOne", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + err := syncRepo(dbClient, "test", "testRepo", "https://my.examplerepo.com", "") + assert.ExistsErr(t, err, "Failed Request") +} + +func Test_getSha256(t *testing.T) { + sha, err := common.GetSha256([]byte("this is a test")) + assert.Equal(t, err, nil, "Unable to get sha") + assert.Equal(t, sha, "2e99758548972a8e8822ad47fa1017ff72f06f3ff6a016851f45c398732bc50c", "Unable to get sha") +} + +func Test_repoAlreadyProcessed(t *testing.T) { + tests := []struct { + name string + checksum string + mockedLastCheck common.RepoCheck + processed bool + }{ + {"not processed yet", "bar", common.RepoCheck{}, false}, + {"already processed", "bar", common.RepoCheck{Checksum: "bar"}, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := mock.Mock{} + repo := &common.RepoCheck{} + m.On("FindOne", mock.Anything, mock.Anything, repo, mock.Anything).Run(func(args mock.Arguments) { + *args.Get(2).(*common.RepoCheck) = tt.mockedLastCheck + }).Return(nil) + dbClient := NewMockClient(&m) + db, _ := dbClient.Database("test") + res := repoAlreadyProcessed(db, "", tt.checksum) + if res != tt.processed { + t.Errorf("Expected alreadyProcessed to be %v got %v", tt.processed, res) + } + }) + } +} + +func Test_updateLastCheck(t *testing.T) { + m := mock.Mock{} + repoName := "foo" + checksum := "bar" + now := time.Now() + selector := bson.M{"_id": repoName} + m.On("UpdateOne", mock.Anything, selector, bson.M{"$set": bson.M{"last_update": now, "checksum": checksum}}, mock.Anything).Return(&mongo.UpdateResult{}, nil) + dbClient := NewMockClient(&m) + db, _ := dbClient.Database("test") + err := updateLastCheck(db, repoName, checksum, now) + if err != nil { + t.Errorf("Unexpected error %v", err) + } + if len(m.Calls) != 1 { + t.Errorf("Expected one call got %d", len(m.Calls)) + } +} diff --git a/cmd/chart-repo/mongodb/delete.go b/cmd/chart-repo/mongodb/delete.go new file mode 100644 index 000000000..873958819 --- /dev/null +++ b/cmd/chart-repo/mongodb/delete.go @@ -0,0 +1,59 @@ +/* +Copyright (c) 2018 The Helm Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "os" + + "github.com/kubeapps/common/datastore" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +//Delete deletes a repository from a mongoDB datastore +func Delete(cmd *cobra.Command, args []string) { + mongoURL, err := cmd.Flags().GetString("mongo-url") + if err != nil { + logrus.Fatal(err) + } + mongoDB, err := cmd.Flags().GetString("mongo-database") + if err != nil { + logrus.Fatal(err) + } + mongoUser, err := cmd.Flags().GetString("mongo-user") + if err != nil { + logrus.Fatal(err) + } + mongoPW := os.Getenv("MONGO_PASSWORD") + debug, err := cmd.Flags().GetBool("debug") + if err != nil { + logrus.Fatal(err) + } + if debug { + logrus.SetLevel(logrus.DebugLevel) + } + mongoConfig := datastore.Config{URL: mongoURL, Database: mongoDB, Username: mongoUser, Password: mongoPW} + dbSession, err := datastore.NewSession(mongoConfig) + if err != nil { + logrus.Fatalf("Can't connect to mongoDB: %v", err) + } + if err = deleteRepo(dbSession, args[0]); err != nil { + logrus.Fatalf("Can't delete chart repository %s from database: %v", args[0], err) + } + + logrus.Infof("Successfully deleted the chart repository %s from database", args[0]) +} diff --git a/cmd/chart-repo/mongodb/sync.go b/cmd/chart-repo/mongodb/sync.go new file mode 100644 index 000000000..f5032580c --- /dev/null +++ b/cmd/chart-repo/mongodb/sync.go @@ -0,0 +1,64 @@ +/* +Copyright (c) 2018 The Helm Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongodb + +import ( + "os" + + "github.com/kubeapps/common/datastore" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +//Sync Add a new chart repository to MongoDB and periodically sync it +func Sync(cmd *cobra.Command, args []string) { + logrus.Debugf("DEBUGGING:") + mongoURL, err := cmd.Flags().GetString("mongo-url") + if err != nil { + logrus.Fatal(err) + } + mongoDB, err := cmd.Flags().GetString("mongo-database") + if err != nil { + logrus.Fatal(err) + } + mongoUser, err := cmd.Flags().GetString("mongo-user") + if err != nil { + logrus.Fatal(err) + } + mongoPW := os.Getenv("MONGO_PASSWORD") + + debug, err := cmd.Flags().GetBool("debug") + if err != nil { + logrus.Fatal(err) + } + if debug { + logrus.SetLevel(logrus.DebugLevel) + } + mongoConfig := datastore.Config{URL: mongoURL, Database: mongoDB, Username: mongoUser, Password: mongoPW} + dbSession, err := datastore.NewSession(mongoConfig) + if err != nil { + logrus.Debugf("DB Config: %v", mongoConfig) + logrus.Fatalf("Can't connect to mongoDB: %v", err) + } + + authorizationHeader := os.Getenv("AUTHORIZATION_HEADER") + if err = syncRepo(dbSession, args[0], args[1], authorizationHeader); err != nil { + logrus.Fatalf("Can't add chart repository to database: %v", err) + } + + logrus.Infof("Successfully added the chart repository %s to database", args[0]) +} diff --git a/cmd/chart-repo/utils.go b/cmd/chart-repo/mongodb/utils.go similarity index 61% rename from cmd/chart-repo/utils.go rename to cmd/chart-repo/mongodb/utils.go index 79b17c6db..4adf921b1 100644 --- a/cmd/chart-repo/utils.go +++ b/cmd/chart-repo/mongodb/utils.go @@ -14,34 +14,27 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package mongodb import ( "archive/tar" "bytes" "compress/gzip" - "crypto/sha256" - "crypto/tls" - "crypto/x509" "errors" "fmt" - "io" "io/ioutil" "net/http" - "net/url" - "os" - "path" "strings" "sync" "time" + "github.com/helm/monocular/cmd/chart-repo/common" + "github.com/helm/monocular/cmd/chart-repo/utils" + "github.com/disintegration/imaging" - "github.com/ghodss/yaml" "github.com/globalsign/mgo/bson" - "github.com/jinzhu/copier" "github.com/kubeapps/common/datastore" log "github.com/sirupsen/logrus" - helmrepo "k8s.io/helm/pkg/repo" ) const ( @@ -52,26 +45,11 @@ const ( additionalCAFile = "/usr/local/share/ca-certificates/ca.crt" ) -type importChartFilesJob struct { - Name string - Repo repo - ChartVersion chartVersion -} - -type httpClient interface { - Do(req *http.Request) (*http.Response, error) -} - -var netClient httpClient = &http.Client{} - -func parseRepoURL(repoURL string) (*url.URL, error) { - repoURL = strings.TrimSpace(repoURL) - return url.ParseRequestURI(repoURL) -} +var netClient common.HTTPClient = &http.Client{} func init() { var err error - netClient, err = initNetClient(additionalCAFile) + netClient, err = common.InitNetClient(additionalCAFile, defaultTimeoutSeconds) if err != nil { log.Fatal(err) } @@ -87,19 +65,19 @@ func init() { // imported into the database as fast as possible. E.g. we want all icons for // charts before fetching readmes for each chart and version pair. func syncRepo(dbSession datastore.Session, repoName, repoURL string, authorizationHeader string) error { - url, err := parseRepoURL(repoURL) + url, err := common.ParseRepoURL(repoURL) if err != nil { log.WithFields(log.Fields{"url": repoURL}).WithError(err).Error("failed to parse URL") return err } - r := repo{Name: repoName, URL: url.String(), AuthorizationHeader: authorizationHeader} - repoBytes, err := fetchRepoIndex(r) + r := common.Repo{Name: repoName, URL: url.String(), AuthorizationHeader: authorizationHeader} + repoBytes, err := common.FetchRepoIndex(r, netClient) if err != nil { return err } - repoChecksum, err := getSha256(repoBytes) + repoChecksum, err := common.GetSha256(repoBytes) if err != nil { return err } @@ -110,12 +88,12 @@ func syncRepo(dbSession datastore.Session, repoName, repoURL string, authorizati return nil } - index, err := parseRepoIndex(repoBytes) + index, err := common.ParseRepoIndex(repoBytes) if err != nil { return err } - charts := chartsFromIndex(index, r) + charts := common.ChartsFromIndex(index, r) if len(charts) == 0 { return errors.New("no charts in repository index") } @@ -126,8 +104,8 @@ func syncRepo(dbSession datastore.Session, repoName, repoURL string, authorizati // Process 10 charts at a time numWorkers := 10 - iconJobs := make(chan chart, numWorkers) - chartFilesJobs := make(chan importChartFilesJob, numWorkers) + iconJobs := make(chan common.Chart, numWorkers) + chartFilesJobs := make(chan common.ImportChartFilesJob, numWorkers) var wg sync.WaitGroup log.Debugf("starting %d workers", numWorkers) @@ -147,11 +125,11 @@ func syncRepo(dbSession datastore.Session, repoName, repoURL string, authorizati // Iterate through the list of charts and enqueue the latest chart version to // be processed. Append the rest of the chart versions to a list to be // enqueued later - var toEnqueue []importChartFilesJob + var toEnqueue []common.ImportChartFilesJob for _, c := range charts { - chartFilesJobs <- importChartFilesJob{c.Name, c.Repo, c.ChartVersions[0]} + chartFilesJobs <- common.ImportChartFilesJob{Name: c.Name, Repo: c.Repo, ChartVersion: c.ChartVersions[0]} for _, cv := range c.ChartVersions[1:] { - toEnqueue = append(toEnqueue, importChartFilesJob{c.Name, c.Repo, cv}) + toEnqueue = append(toEnqueue, common.ImportChartFilesJob{Name: c.Name, Repo: c.Repo, ChartVersion: cv}) } } @@ -175,19 +153,10 @@ func syncRepo(dbSession datastore.Session, repoName, repoURL string, authorizati return nil } -func getSha256(src []byte) (string, error) { - f := bytes.NewReader(src) - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - return "", err - } - return fmt.Sprintf("%x", h.Sum(nil)), nil -} - func repoAlreadyProcessed(dbSession datastore.Session, repoName string, checksum string) bool { db, closer := dbSession.DB() defer closer() - lastCheck := &repoCheck{} + lastCheck := &common.RepoCheck{} err := db.C(repositoryCollection).Find(bson.M{"_id": repoName}).One(lastCheck) return err == nil && checksum == lastCheck.Checksum } @@ -222,79 +191,7 @@ func deleteRepo(dbSession datastore.Session, repoName string) error { return err } -func fetchRepoIndex(r repo) ([]byte, error) { - indexURL, err := parseRepoURL(r.URL) - if err != nil { - log.WithFields(log.Fields{"url": r.URL}).WithError(err).Error("failed to parse URL") - return nil, err - } - indexURL.Path = path.Join(indexURL.Path, "index.yaml") - req, err := http.NewRequest("GET", indexURL.String(), nil) - if err != nil { - log.WithFields(log.Fields{"url": req.URL.String()}).WithError(err).Error("could not build repo index request") - return nil, err - } - - req.Header.Set("User-Agent", userAgent()) - if len(r.AuthorizationHeader) > 0 { - req.Header.Set("Authorization", r.AuthorizationHeader) - } - - res, err := netClient.Do(req) - if res != nil { - defer res.Body.Close() - } - if err != nil { - log.WithFields(log.Fields{"url": req.URL.String()}).WithError(err).Error("error requesting repo index") - return nil, err - } - - if res.StatusCode != http.StatusOK { - log.WithFields(log.Fields{"url": req.URL.String(), "status": res.StatusCode}).Error("error requesting repo index, are you sure this is a chart repository?") - return nil, errors.New("repo index request failed") - } - - body, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - return body, nil -} - -func parseRepoIndex(body []byte) (*helmrepo.IndexFile, error) { - var index helmrepo.IndexFile - err := yaml.Unmarshal(body, &index) - if err != nil { - return nil, err - } - index.SortEntries() - return &index, nil -} - -func chartsFromIndex(index *helmrepo.IndexFile, r repo) []chart { - var charts []chart - for _, entry := range index.Entries { - if entry[0].GetDeprecated() { - log.WithFields(log.Fields{"name": entry[0].GetName()}).Info("skipping deprecated chart") - continue - } - charts = append(charts, newChart(entry, r)) - } - return charts -} - -// Takes an entry from the index and constructs a database representation of the -// object. -func newChart(entry helmrepo.ChartVersions, r repo) chart { - var c chart - copier.Copy(&c, entry[0]) - copier.Copy(&c.ChartVersions, entry) - c.Repo = r - c.ID = fmt.Sprintf("%s/%s", r.Name, c.Name) - return c -} - -func importCharts(dbSession datastore.Session, charts []chart) error { +func importCharts(dbSession datastore.Session, charts []common.Chart) error { var pairs []interface{} var chartIDs []string for _, c := range charts { @@ -322,7 +219,7 @@ func importCharts(dbSession datastore.Session, charts []chart) error { return err } -func importWorker(dbSession datastore.Session, wg *sync.WaitGroup, icons <-chan chart, chartFiles <-chan importChartFilesJob) { +func importWorker(dbSession datastore.Session, wg *sync.WaitGroup, icons <-chan common.Chart, chartFiles <-chan common.ImportChartFilesJob) { defer wg.Done() for c := range icons { log.WithFields(log.Fields{"name": c.Name}).Debug("importing icon") @@ -338,7 +235,7 @@ func importWorker(dbSession datastore.Session, wg *sync.WaitGroup, icons <-chan } } -func fetchAndImportIcon(dbSession datastore.Session, c chart) error { +func fetchAndImportIcon(dbSession datastore.Session, c common.Chart) error { if c.Icon == "" { log.WithFields(log.Fields{"name": c.Name}).Info("icon not found") return nil @@ -348,7 +245,7 @@ func fetchAndImportIcon(dbSession datastore.Session, c chart) error { if err != nil { return err } - req.Header.Set("User-Agent", userAgent()) + req.Header.Set("User-Agent", utils.UserAgent()) if len(c.Repo.AuthorizationHeader) > 0 { req.Header.Set("Authorization", c.Repo.AuthorizationHeader) } @@ -396,24 +293,24 @@ func fetchAndImportIcon(dbSession datastore.Session, c chart) error { return db.C(chartCollection).UpdateId(c.ID, bson.M{"$set": bson.M{"raw_icon": b, "icon_content_type": contentType}}) } -func fetchAndImportFiles(dbSession datastore.Session, name string, r repo, cv chartVersion) error { +func fetchAndImportFiles(dbSession datastore.Session, name string, r common.Repo, cv common.ChartVersion) error { chartFilesID := fmt.Sprintf("%s/%s-%s", r.Name, name, cv.Version) db, closer := dbSession.DB() defer closer() // Check if we already have indexed files for this chart version and digest - if err := db.C(chartFilesCollection).Find(bson.M{"_id": chartFilesID, "digest": cv.Digest}).One(&chartFiles{}); err == nil { + if err := db.C(chartFilesCollection).Find(bson.M{"_id": chartFilesID, "digest": cv.Digest}).One(&common.ChartFiles{}); err == nil { log.WithFields(log.Fields{"name": name, "version": cv.Version}).Debug("skipping existing files") return nil } log.WithFields(log.Fields{"name": name, "version": cv.Version}).Debug("fetching files") - url := chartTarballURL(r, cv) + url := common.ChartTarballURL(r, cv) req, err := http.NewRequest("GET", url, nil) if err != nil { return err } - req.Header.Set("User-Agent", userAgent()) + req.Header.Set("User-Agent", utils.UserAgent()) if len(r.AuthorizationHeader) > 0 { req.Header.Set("Authorization", r.AuthorizationHeader) } @@ -440,12 +337,12 @@ func fetchAndImportFiles(dbSession datastore.Session, name string, r repo, cv ch schemaFileName := name + "/values.schema.json" filenames := []string{valuesFileName, readmeFileName, schemaFileName} - files, err := extractFilesFromTarball(filenames, tarf) + files, err := common.ExtractFilesFromTarball(filenames, tarf) if err != nil { return err } - chartFiles := chartFiles{ID: chartFilesID, Repo: r, Digest: cv.Digest} + chartFiles := common.ChartFiles{ID: chartFilesID, Repo: r, Digest: cv.Digest} if v, ok := files[readmeFileName]; ok { chartFiles.Readme = v } else { @@ -468,71 +365,3 @@ func fetchAndImportFiles(dbSession datastore.Session, name string, r repo, cv ch return nil } - -func extractFilesFromTarball(filenames []string, tarf *tar.Reader) (map[string]string, error) { - ret := make(map[string]string) - for { - header, err := tarf.Next() - if err == io.EOF { - break - } - if err != nil { - return ret, err - } - - for _, f := range filenames { - if strings.EqualFold(header.Name, f) { - var b bytes.Buffer - io.Copy(&b, tarf) - ret[f] = string(b.Bytes()) - break - } - } - } - return ret, nil -} - -func chartTarballURL(r repo, cv chartVersion) string { - source := cv.URLs[0] - if _, err := parseRepoURL(source); err != nil { - // If the chart URL is not absolute, join with repo URL. It's fine if the - // URL we build here is invalid as we can catch this error when actually - // making the request - u, _ := url.Parse(r.URL) - u.Path = path.Join(u.Path, source) - return u.String() - } - return source -} - -func initNetClient(additionalCA string) (*http.Client, error) { - // Get the SystemCertPool, continue with an empty pool on error - caCertPool, _ := x509.SystemCertPool() - if caCertPool == nil { - caCertPool = x509.NewCertPool() - } - - // If additionalCA exists, load it - if _, err := os.Stat(additionalCA); !os.IsNotExist(err) { - certs, err := ioutil.ReadFile(additionalCA) - if err != nil { - return nil, fmt.Errorf("Failed to append %s to RootCAs: %v", additionalCA, err) - } - - // Append our cert to the system pool - if ok := caCertPool.AppendCertsFromPEM(certs); !ok { - return nil, fmt.Errorf("Failed to append %s to RootCAs", additionalCA) - } - } - - // Return Transport for testing purposes - return &http.Client{ - Timeout: time.Second * defaultTimeoutSeconds, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: caCertPool, - }, - Proxy: http.ProxyFromEnvironment, - }, - }, nil -} diff --git a/cmd/chart-repo/utils_test.go b/cmd/chart-repo/mongodb/utils_test.go similarity index 81% rename from cmd/chart-repo/utils_test.go rename to cmd/chart-repo/mongodb/utils_test.go index 0adc27918..6e6406306 100644 --- a/cmd/chart-repo/utils_test.go +++ b/cmd/chart-repo/mongodb/utils_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package mongodb import ( "archive/tar" @@ -35,6 +35,9 @@ import ( "testing" "time" + "github.com/helm/monocular/cmd/chart-repo/common" + "github.com/helm/monocular/cmd/chart-repo/utils" + "github.com/arschles/assert" "github.com/disintegration/imaging" "github.com/globalsign/mgo/bson" @@ -43,7 +46,7 @@ import ( "github.com/stretchr/testify/mock" ) -var validRepoIndexYAMLBytes, _ = ioutil.ReadFile("testdata/valid-index.yaml") +var validRepoIndexYAMLBytes, _ = ioutil.ReadFile("../testdata/valid-index.yaml") var validRepoIndexYAML = string(validRepoIndexYAMLBytes) var invalidRepoIndexYAML = "invalid" @@ -121,7 +124,7 @@ func (h *svgIconClient) Do(req *http.Request) (*http.Response, error) { } type goodTarballClient struct { - c chart + c common.Chart skipReadme bool skipValues bool skipSchema bool @@ -150,7 +153,7 @@ func (h *goodTarballClient) Do(req *http.Request) (*http.Response, error) { } type authenticatedTarballClient struct { - c chart + c common.Chart } func (h *authenticatedTarballClient) Do(req *http.Request) (*http.Response, error) { @@ -192,32 +195,32 @@ func Test_syncURLInvalidity(t *testing.T) { func Test_fetchRepoIndex(t *testing.T) { tests := []struct { name string - r repo + r common.Repo }{ - {"valid HTTP URL", repo{URL: "http://my.examplerepo.com"}}, - {"valid HTTPS URL", repo{URL: "https://my.examplerepo.com"}}, - {"valid trailing URL", repo{URL: "https://my.examplerepo.com/"}}, - {"valid subpath URL", repo{URL: "https://subpath.test/subpath/"}}, - {"valid URL with trailing spaces", repo{URL: "https://subpath.test/subpath/ "}}, - {"valid URL with leading spaces", repo{URL: " https://subpath.test/subpath/"}}, + {"valid HTTP URL", common.Repo{URL: "http://my.examplerepo.com"}}, + {"valid HTTPS URL", common.Repo{URL: "https://my.examplerepo.com"}}, + {"valid trailing URL", common.Repo{URL: "https://my.examplerepo.com/"}}, + {"valid subpath URL", common.Repo{URL: "https://subpath.test/subpath/"}}, + {"valid URL with trailing spaces", common.Repo{URL: "https://subpath.test/subpath/ "}}, + {"valid URL with leading spaces", common.Repo{URL: " https://subpath.test/subpath/"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { netClient = &goodHTTPClient{} - _, err := fetchRepoIndex(tt.r) + _, err := common.FetchRepoIndex(tt.r, netClient) assert.NoErr(t, err) }) } t.Run("authenticated request", func(t *testing.T) { netClient = &authenticatedHTTPClient{} - _, err := fetchRepoIndex(repo{URL: "https://my.examplerepo.com", AuthorizationHeader: "Bearer ThisSecretAccessTokenAuthenticatesTheClient"}) + _, err := common.FetchRepoIndex(common.Repo{URL: "https://my.examplerepo.com", AuthorizationHeader: "Bearer ThisSecretAccessTokenAuthenticatesTheClient"}, netClient) assert.NoErr(t, err) }) t.Run("failed request", func(t *testing.T) { netClient = &badHTTPClient{} - _, err := fetchRepoIndex(repo{URL: "https://my.examplerepo.com"}) + _, err := common.FetchRepoIndex(common.Repo{URL: "https://my.examplerepo.com"}, netClient) assert.ExistsErr(t, err, "failed request") }) } @@ -238,11 +241,11 @@ func Test_fetchRepoIndexUserAgent(t *testing.T) { t.Run(tt.name, func(t *testing.T) { // Override global variables used to generate the userAgent if tt.version != "" { - version = tt.version + utils.Version = tt.version } if tt.userAgentComment != "" { - userAgentComment = tt.userAgentComment + utils.UserAgentComment = tt.userAgentComment } server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { @@ -254,7 +257,7 @@ func Test_fetchRepoIndexUserAgent(t *testing.T) { netClient = server.Client() - _, err := fetchRepoIndex(repo{URL: server.URL}) + _, err := common.FetchRepoIndex(common.Repo{URL: server.URL}, netClient) assert.NoErr(t, err) }) } @@ -269,13 +272,13 @@ func Test_parseRepoIndex(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := parseRepoIndex([]byte(tt.repoYAML)) + _, err := common.ParseRepoIndex([]byte(tt.repoYAML)) assert.ExistsErr(t, err, tt.name) }) } t.Run("valid", func(t *testing.T) { - index, err := parseRepoIndex([]byte(validRepoIndexYAML)) + index, err := common.ParseRepoIndex([]byte(validRepoIndexYAML)) assert.NoErr(t, err) assert.Equal(t, len(index.Entries), 2, "number of charts") assert.Equal(t, index.Entries["acs-engine-autoscaler"][0].GetName(), "acs-engine-autoscaler", "chart version populated") @@ -283,25 +286,25 @@ func Test_parseRepoIndex(t *testing.T) { } func Test_chartsFromIndex(t *testing.T) { - r := repo{Name: "test", URL: "http://testrepo.com"} - index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) - charts := chartsFromIndex(index, r) + r := common.Repo{Name: "test", URL: "http://testrepo.com"} + index, _ := common.ParseRepoIndex([]byte(validRepoIndexYAML)) + charts := common.ChartsFromIndex(index, r) assert.Equal(t, len(charts), 2, "number of charts") indexWithDeprecated := validRepoIndexYAML + ` deprecated-chart: - name: deprecated-chart deprecated: true` - index2, err := parseRepoIndex([]byte(indexWithDeprecated)) + index2, err := common.ParseRepoIndex([]byte(indexWithDeprecated)) assert.NoErr(t, err) - charts = chartsFromIndex(index2, r) + charts = common.ChartsFromIndex(index2, r) assert.Equal(t, len(charts), 2, "number of charts") } func Test_newChart(t *testing.T) { - r := repo{Name: "test", URL: "http://testrepo.com"} - index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) - c := newChart(index.Entries["wordpress"], r) + r := common.Repo{Name: "test", URL: "http://testrepo.com"} + index, _ := common.ParseRepoIndex([]byte(validRepoIndexYAML)) + c := common.NewChart(index.Entries["wordpress"], r) assert.Equal(t, c.Name, "wordpress", "correctly built") assert.Equal(t, len(c.ChartVersions), 2, "correctly built") assert.Equal(t, c.Description, "new description!", "takes chart fields from latest entry") @@ -315,8 +318,8 @@ func Test_importCharts(t *testing.T) { m.On("Upsert", mock.Anything) m.On("RemoveAll", mock.Anything) dbSession := mockstore.NewMockSession(m) - index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) - charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com"}) + index, _ := common.ParseRepoIndex([]byte(validRepoIndexYAML)) + charts := common.ChartsFromIndex(index, common.Repo{Name: "test", URL: "http://testrepo.com"}) importCharts(dbSession, charts) m.AssertExpectations(t) @@ -326,7 +329,7 @@ func Test_importCharts(t *testing.T) { args := m.Calls[0].Arguments.Get(0).([]interface{}) assert.Equal(t, len(args), len(charts)*2, "number of selector, chart pairs to upsert") for i := 0; i < len(args); i += 2 { - c := args[i+1].(chart) + c := args[i+1].(common.Chart) assert.Equal(t, args[i], bson.M{"_id": "test/" + c.Name}, "selector") } } @@ -352,12 +355,12 @@ func Test_fetchAndImportIcon(t *testing.T) { t.Run("no icon", func(t *testing.T) { m := mock.Mock{} dbSession := mockstore.NewMockSession(&m) - c := chart{ID: "test/acs-engine-autoscaler"} + c := common.Chart{ID: "test/acs-engine-autoscaler"} assert.NoErr(t, fetchAndImportIcon(dbSession, c)) }) - index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) - charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com"}) + index, _ := common.ParseRepoIndex([]byte(validRepoIndexYAML)) + charts := common.ChartsFromIndex(index, common.Repo{Name: "test", URL: "http://testrepo.com"}) t.Run("failed download", func(t *testing.T) { netClient = &badHTTPClient{} @@ -387,10 +390,10 @@ func Test_fetchAndImportIcon(t *testing.T) { t.Run("valid SVG icon", func(t *testing.T) { netClient = &svgIconClient{} - c := chart{ + c := common.Chart{ ID: "foo", Icon: "https://foo/bar/logo.svg", - Repo: repo{}, + Repo: common.Repo{}, } m := mock.Mock{} dbSession := mockstore.NewMockSession(&m) @@ -401,8 +404,8 @@ func Test_fetchAndImportIcon(t *testing.T) { } func Test_fetchAndImportFiles(t *testing.T) { - index, _ := parseRepoIndex([]byte(validRepoIndexYAML)) - charts := chartsFromIndex(index, repo{Name: "test", URL: "http://testrepo.com", AuthorizationHeader: "Bearer ThisSecretAccessTokenAuthenticatesTheClient1s"}) + index, _ := common.ParseRepoIndex([]byte(validRepoIndexYAML)) + charts := common.ChartsFromIndex(index, common.Repo{Name: "test", URL: "http://testrepo.com", AuthorizationHeader: "Bearer ThisSecretAccessTokenAuthenticatesTheClient1s"}) cv := charts[0].ChartVersions[0] t.Run("http error", func(t *testing.T) { @@ -418,7 +421,7 @@ func Test_fetchAndImportFiles(t *testing.T) { m := mock.Mock{} m.On("One", mock.Anything).Return(errors.New("return an error when checking if files already exists to force fetching")) chartFilesID := fmt.Sprintf("%s/%s-%s", charts[0].Repo.Name, charts[0].Name, cv.Version) - m.On("UpsertId", chartFilesID, chartFiles{chartFilesID, "", "", "", charts[0].Repo, cv.Digest}) + m.On("UpsertId", chartFilesID, common.ChartFiles{ID: chartFilesID, Readme: "", Values: "", Schema: "", Repo: charts[0].Repo, Digest: cv.Digest}) dbSession := mockstore.NewMockSession(&m) err := fetchAndImportFiles(dbSession, charts[0].Name, charts[0].Repo, cv) assert.NoErr(t, err) @@ -430,7 +433,7 @@ func Test_fetchAndImportFiles(t *testing.T) { m := mock.Mock{} m.On("One", mock.Anything).Return(errors.New("return an error when checking if files already exists to force fetching")) chartFilesID := fmt.Sprintf("%s/%s-%s", charts[0].Repo.Name, charts[0].Name, cv.Version) - m.On("UpsertId", chartFilesID, chartFiles{chartFilesID, testChartReadme, testChartValues, testChartSchema, charts[0].Repo, cv.Digest}) + m.On("UpsertId", chartFilesID, common.ChartFiles{ID: chartFilesID, Readme: testChartReadme, Values: testChartValues, Schema: testChartSchema, Repo: charts[0].Repo, Digest: cv.Digest}) dbSession := mockstore.NewMockSession(&m) err := fetchAndImportFiles(dbSession, charts[0].Name, charts[0].Repo, cv) assert.NoErr(t, err) @@ -442,7 +445,7 @@ func Test_fetchAndImportFiles(t *testing.T) { m := mock.Mock{} m.On("One", mock.Anything).Return(errors.New("return an error when checking if files already exists to force fetching")) chartFilesID := fmt.Sprintf("%s/%s-%s", charts[0].Repo.Name, charts[0].Name, cv.Version) - m.On("UpsertId", chartFilesID, chartFiles{chartFilesID, testChartReadme, testChartValues, testChartSchema, charts[0].Repo, cv.Digest}) + m.On("UpsertId", chartFilesID, common.ChartFiles{ID: chartFilesID, Readme: testChartReadme, Values: testChartValues, Schema: testChartSchema, Repo: charts[0].Repo, Digest: cv.Digest}) dbSession := mockstore.NewMockSession(&m) err := fetchAndImportFiles(dbSession, charts[0].Name, charts[0].Repo, cv) assert.NoErr(t, err) @@ -461,19 +464,19 @@ func Test_fetchAndImportFiles(t *testing.T) { } func Test_chartTarballURL(t *testing.T) { - r := repo{Name: "test", URL: "http://testrepo.com"} + r := common.Repo{Name: "test", URL: "http://testrepo.com"} tests := []struct { name string - cv chartVersion + cv common.ChartVersion wanted string }{ - {"absolute url", chartVersion{URLs: []string{"http://testrepo.com/wordpress-0.1.0.tgz"}}, "http://testrepo.com/wordpress-0.1.0.tgz"}, - {"relative url", chartVersion{URLs: []string{"wordpress-0.1.0.tgz"}}, "http://testrepo.com/wordpress-0.1.0.tgz"}, + {"absolute url", common.ChartVersion{URLs: []string{"http://testrepo.com/wordpress-0.1.0.tgz"}}, "http://testrepo.com/wordpress-0.1.0.tgz"}, + {"relative url", common.ChartVersion{URLs: []string{"wordpress-0.1.0.tgz"}}, "http://testrepo.com/wordpress-0.1.0.tgz"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, chartTarballURL(r, tt.cv), tt.wanted, "url") + assert.Equal(t, common.ChartTarballURL(r, tt.cv), tt.wanted, "url") }) } } @@ -497,7 +500,7 @@ func Test_extractFilesFromTarball(t *testing.T) { createTestTarball(&b, tt.files) r := bytes.NewReader(b.Bytes()) tarf := tar.NewReader(r) - files, err := extractFilesFromTarball([]string{tt.filename}, tarf) + files, err := common.ExtractFilesFromTarball([]string{tt.filename}, tarf) assert.NoErr(t, err) assert.Equal(t, files[tt.filename], tt.want, "file body") }) @@ -509,7 +512,7 @@ func Test_extractFilesFromTarball(t *testing.T) { createTestTarball(&b, tFiles) r := bytes.NewReader(b.Bytes()) tarf := tar.NewReader(r) - files, err := extractFilesFromTarball([]string{tFiles[0].Name, tFiles[1].Name}, tarf) + files, err := common.ExtractFilesFromTarball([]string{tFiles[0].Name, tFiles[1].Name}, tarf) assert.NoErr(t, err) assert.Equal(t, len(files), 2, "matches") for _, f := range tFiles { @@ -523,7 +526,7 @@ func Test_extractFilesFromTarball(t *testing.T) { r := bytes.NewReader(b.Bytes()) tarf := tar.NewReader(r) name := "file2.txt" - files, err := extractFilesFromTarball([]string{name}, tarf) + files, err := common.ExtractFilesFromTarball([]string{name}, tarf) assert.NoErr(t, err) assert.Equal(t, files[name], "", "file body") }) @@ -533,7 +536,7 @@ func Test_extractFilesFromTarball(t *testing.T) { rand.Read(b) r := bytes.NewReader(b) tarf := tar.NewReader(r) - files, err := extractFilesFromTarball([]string{"file2.txt"}, tarf) + files, err := common.ExtractFilesFromTarball([]string{"file2.txt"}, tarf) assert.Err(t, io.ErrUnexpectedEOF, err) assert.Equal(t, len(files), 0, "file body") }) @@ -600,7 +603,7 @@ h251U/Daz6NiQBM9AxyAw6EHm8XAZBvCuebfzyrT t.Error(err) } - _, err = initNetClient(otherCA) + _, err = common.InitNetClient(otherCA, defaultTimeoutSeconds) if err != nil { t.Error(err) } @@ -620,14 +623,14 @@ func (h *emptyChartRepoHTTPClient) Do(req *http.Request) (*http.Response, error) func Test_emptyChartRepo(t *testing.T) { netClient = &emptyChartRepoHTTPClient{} m := mock.Mock{} - m.On("One", &repoCheck{}).Return(nil) + m.On("One", &common.RepoCheck{}).Return(nil) dbSession := mockstore.NewMockSession(&m) err := syncRepo(dbSession, "testRepo", "https://my.examplerepo.com", "") assert.ExistsErr(t, err, "Failed Request") } func Test_getSha256(t *testing.T) { - sha, err := getSha256([]byte("this is a test")) + sha, err := common.GetSha256([]byte("this is a test")) assert.Equal(t, err, nil, "Unable to get sha") assert.Equal(t, sha, "2e99758548972a8e8822ad47fa1017ff72f06f3ff6a016851f45c398732bc50c", "Unable to get sha") } @@ -636,19 +639,19 @@ func Test_repoAlreadyProcessed(t *testing.T) { tests := []struct { name string checksum string - mockedLastCheck repoCheck + mockedLastCheck common.RepoCheck processed bool }{ - {"not processed yet", "bar", repoCheck{}, false}, - {"already processed", "bar", repoCheck{Checksum: "bar"}, true}, + {"not processed yet", "bar", common.RepoCheck{}, false}, + {"already processed", "bar", common.RepoCheck{Checksum: "bar"}, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { m := mock.Mock{} - repo := &repoCheck{} + repo := &common.RepoCheck{} m.On("One", repo).Run(func(args mock.Arguments) { - *args.Get(0).(*repoCheck) = tt.mockedLastCheck + *args.Get(0).(*common.RepoCheck) = tt.mockedLastCheck }).Return(nil) dbSession := mockstore.NewMockSession(&m) res := repoAlreadyProcessed(dbSession, "", tt.checksum) diff --git a/cmd/chart-repo/sync.go b/cmd/chart-repo/sync.go index 8527876db..494d1cba5 100644 --- a/cmd/chart-repo/sync.go +++ b/cmd/chart-repo/sync.go @@ -17,54 +17,37 @@ limitations under the License. package main import ( - "os" + "github.com/helm/monocular/cmd/chart-repo/foundationdb" + "github.com/helm/monocular/cmd/chart-repo/mongodb" - "github.com/kubeapps/common/datastore" - "github.com/sirupsen/logrus" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) -var syncCmd = &cobra.Command{ +//SyncCmd Sync a chart repository with Monocular +var SyncCmd = &cobra.Command{ Use: "sync [REPO NAME] [REPO URL]", Short: "add a new chart repository, and resync its charts periodically", Run: func(cmd *cobra.Command, args []string) { + if len(args) != 2 { - logrus.Info("Need exactly two arguments: [REPO NAME] [REPO URL]") + log.Info("Need exactly two arguments: [REPO NAME] [REPO URL]") cmd.Help() return } - mongoURL, err := cmd.Flags().GetString("mongo-url") - if err != nil { - logrus.Fatal(err) - } - mongoDB, err := cmd.Flags().GetString("mongo-database") + dbType, err := cmd.Flags().GetString("db-type") if err != nil { - logrus.Fatal(err) - } - mongoUser, err := cmd.Flags().GetString("mongo-user") - if err != nil { - logrus.Fatal(err) - } - mongoPW := os.Getenv("MONGO_PASSWORD") - debug, err := cmd.Flags().GetBool("debug") - if err != nil { - logrus.Fatal(err) - } - if debug { - logrus.SetLevel(logrus.DebugLevel) - } - mongoConfig := datastore.Config{URL: mongoURL, Database: mongoDB, Username: mongoUser, Password: mongoPW} - dbSession, err := datastore.NewSession(mongoConfig) - if err != nil { - logrus.Fatalf("Can't connect to mongoDB: %v", err) + mongodb.Sync(cmd, args) } - authorizationHeader := os.Getenv("AUTHORIZATION_HEADER") - if err = syncRepo(dbSession, args[0], args[1], authorizationHeader); err != nil { - logrus.Fatalf("Can't add chart repository to database: %v", err) + switch dbType { + case "mongodb": + mongodb.Sync(cmd, args) + case "fdb": + foundationdb.Sync(cmd, args) + default: + log.Fatalf("Unknown database type: %v. db-type, if set, must be either 'mongodb' or 'fdb'.", dbType) } - - logrus.Infof("Successfully added the chart repository %s to database", args[0]) }, } diff --git a/cmd/chart-repo/version.go b/cmd/chart-repo/utils/version.go similarity index 70% rename from cmd/chart-repo/version.go rename to cmd/chart-repo/utils/version.go index 52c38d76d..e30f658e8 100644 --- a/cmd/chart-repo/version.go +++ b/cmd/chart-repo/utils/version.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package utils import ( "fmt" @@ -23,28 +23,29 @@ import ( ) var ( - version = "devel" - userAgentComment string + Version = "devel" + UserAgentComment string ) -// Returns the user agent to be used during calls to the chart repositories +// UserAgent returns the user agent to be used during calls to the chart repositories // Examples: // chart-repo/devel // chart-repo/1.0 // chart-repo/1.0 (monocular v1.0-beta4) // More info here https://github.com/kubeapps/kubeapps/issues/767#issuecomment-436835938 -func userAgent() string { - ua := "chart-repo/" + version - if userAgentComment != "" { - ua = fmt.Sprintf("%s (%s)", ua, userAgentComment) +func UserAgent() string { + ua := "chart-repo/" + Version + if UserAgentComment != "" { + ua = fmt.Sprintf("%s (%s)", ua, UserAgentComment) } return ua } -var versionCmd = &cobra.Command{ +//VersionCmd returns Monocular version information +var VersionCmd = &cobra.Command{ Use: "version", Short: "returns version information", Run: func(cmd *cobra.Command, args []string) { - fmt.Println(version) + fmt.Println(Version) }, } diff --git a/cmd/chartsvc/Dockerfile b/cmd/chartsvc/Dockerfile index 960f9e281..42d494ab8 100644 --- a/cmd/chartsvc/Dockerfile +++ b/cmd/chartsvc/Dockerfile @@ -2,7 +2,6 @@ FROM golang:1.12 as builder COPY . /go/src/github.com/helm/monocular WORKDIR /go/src/github.com/helm/monocular RUN GO111MODULE=on GOPROXY=https://gocenter.io CGO_ENABLED=0 go build -a -installsuffix cgo ./cmd/chartsvc - FROM scratch COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /go/src/github.com/helm/monocular/chartsvc /chartsvc diff --git a/cmd/chartsvc/Makefile b/cmd/chartsvc/Makefile index 291d260aa..678bab62e 100644 --- a/cmd/chartsvc/Makefile +++ b/cmd/chartsvc/Makefile @@ -1,4 +1,4 @@ -IMAGE_REPO ?= quay.io/helmpack/chartsvc +IMAGE_REPO ?= docker.io/kreinecke/chartsvc IMAGE_TAG ?= latest docker-build: diff --git a/cmd/chartsvc/foundationdb/datastore/datastore.go b/cmd/chartsvc/foundationdb/datastore/datastore.go new file mode 100644 index 000000000..2580d1f93 --- /dev/null +++ b/cmd/chartsvc/foundationdb/datastore/datastore.go @@ -0,0 +1,145 @@ +/* +Copyright (c) 2019 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package datastore implements an interface on top of the mgo mongo client +package datastore + +import ( + "context" + "fmt" + "reflect" + "time" + + log "github.com/sirupsen/logrus" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const defaultTimeout = 30 * time.Second + +// Config configures the database connection +type Config struct { + URL string + Database string + Timeout time.Duration +} + +// Client is an interface for a MongoDB client +type Client interface { + Database(name string) (Database, func()) +} + +// Database is an interface for accessing a MongoDB database +type Database interface { + Collection(name string) Collection +} + +// Collection is an interface for accessing a MongoDB collection +type Collection interface { + BulkWrite(ctxt context.Context, operations []mongo.WriteModel, options *options.BulkWriteOptions) (*mongo.BulkWriteResult, error) + DeleteMany(ctxt context.Context, filter interface{}, options *options.DeleteOptions) (*mongo.DeleteResult, error) + FindOne(ctxt context.Context, filter interface{}, result interface{}, options *options.FindOneOptions) error + InsertOne(ctxt context.Context, document interface{}, options *options.InsertOneOptions) (*mongo.InsertOneResult, error) + UpdateOne(ctxt context.Context, filter interface{}, update interface{}, options *options.UpdateOptions) (*mongo.UpdateResult, error) + Find(ctxt context.Context, filter interface{}, result interface{}, options *options.FindOptions) error +} + +// mongoDatabase wraps a mongo.Database and implements Collection +type mongoDatabase struct { + Database *mongo.Database +} + +// mongoClient wraps a mongo.Client and implements Database +type mongoClient struct { + Client *mongo.Client +} + +// NewDocLayerClient creates a new MongoDB client for the FoundationDB document layer. +func NewDocLayerClient(ctx context.Context, options *options.ClientOptions) (Client, error) { + client, err := mongo.Connect(ctx, options) + return &mongoClient{client}, err +} + +func (c *mongoClient) Database(dbName string) (Database, func()) { + + db := &mongoDatabase{c.Client.Database(dbName)} + + return db, func() { + err := c.Client.Disconnect(context.Background()) + + if err != nil { + log.Fatal(err) + } + fmt.Println("Connection to MongoDB closed.") + } +} + +func (d *mongoDatabase) Collection(name string) Collection { + return &mongoCollection{d.Database.Collection(name)} +} + +// mgoCollection wraps an mgo.Collection and implements Collection +type mongoCollection struct { + Collection *mongo.Collection +} + +func (c *mongoCollection) BulkWrite(ctxt context.Context, operations []mongo.WriteModel, options *options.BulkWriteOptions) (*mongo.BulkWriteResult, error) { + res, err := c.Collection.BulkWrite(ctxt, operations, options) + return res, err +} + +func (c *mongoCollection) DeleteMany(ctxt context.Context, filter interface{}, options *options.DeleteOptions) (*mongo.DeleteResult, error) { + res, err := c.Collection.DeleteMany(ctxt, filter, options) + return res, err +} + +func (c *mongoCollection) FindOne(ctxt context.Context, filter interface{}, result interface{}, options *options.FindOneOptions) error { + res := c.Collection.FindOne(ctxt, filter, options) + return res.Decode(result) +} + +func (c *mongoCollection) InsertOne(ctxt context.Context, document interface{}, options *options.InsertOneOptions) (*mongo.InsertOneResult, error) { + res, err := c.Collection.InsertOne(ctxt, document, options) + return res, err +} + +func (c *mongoCollection) UpdateOne(ctxt context.Context, filter interface{}, document interface{}, options *options.UpdateOptions) (*mongo.UpdateResult, error) { + res, err := c.Collection.UpdateOne(ctxt, filter, document, options) + return res, err +} + +func (c *mongoCollection) Find(ctxt context.Context, filter interface{}, result interface{}, options *options.FindOptions) error { + resCursor, err := c.Collection.Find(ctxt, filter, options) + if err != nil { + log.WithError(err).Errorf( + "Error fetching query result.") + } else { + resultType := reflect.ValueOf(result) + if resultType.Kind() != reflect.Ptr { + return fmt.Errorf("Result arg must be a pointer type. Got: %v", resultType.Kind()) + } + resultSliceType := resultType.Elem() + if resultSliceType.Kind() != reflect.Slice { + return fmt.Errorf("Result arg must be a slice value. Got: %v", resultSliceType.Kind()) + } + err = resCursor.All(context.Background(), result) + if err != nil { + log.WithError(err).Errorf( + "Error decoding query result.") + } + } + return err +} diff --git a/cmd/chartsvc/foundationdb/datastore/mockstore.go b/cmd/chartsvc/foundationdb/datastore/mockstore.go new file mode 100644 index 000000000..053ffea38 --- /dev/null +++ b/cmd/chartsvc/foundationdb/datastore/mockstore.go @@ -0,0 +1,87 @@ +/* +Copyright (c) 2019 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package datastore + +import ( + "context" + + "github.com/stretchr/testify/mock" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// mockDatabase acts as a mock datastore.Database +type mockDatabase struct { + *mock.Mock +} + +type mockClient struct { + *mock.Mock +} + +//NewMockClient returns a mocked FDB Document-Layer client +func NewMockClient(m *mock.Mock) Client { + return mockClient{m} +} + +// DB returns a mocked datastore.Database and empty closer function +func (c mockClient) Database(dbName string) (Database, func()) { + + db := mockDatabase{c.Mock} + + return db, func() { + } +} + +func (d mockDatabase) Collection(name string) Collection { + return mockCollection{d.Mock} +} + +// mockCollection acts as a mock datastore.Collection +type mockCollection struct { + *mock.Mock +} + +func (c mockCollection) BulkWrite(ctxt context.Context, operations []mongo.WriteModel, options *options.BulkWriteOptions) (*mongo.BulkWriteResult, error) { + args := c.Called(ctxt, operations, options) + return args.Get(0).(*mongo.BulkWriteResult), args.Error(1) +} + +func (c mockCollection) DeleteMany(ctxt context.Context, filter interface{}, options *options.DeleteOptions) (*mongo.DeleteResult, error) { + args := c.Called(ctxt, filter, options) + return args.Get(0).(*mongo.DeleteResult), args.Error(1) +} + +func (c mockCollection) FindOne(ctxt context.Context, filter interface{}, result interface{}, options *options.FindOneOptions) error { + args := c.Called(ctxt, filter, result, options) + return args.Error(0) +} + +func (c mockCollection) InsertOne(ctxt context.Context, document interface{}, options *options.InsertOneOptions) (*mongo.InsertOneResult, error) { + args := c.Called(ctxt, document, options) + return args.Get(0).(*mongo.InsertOneResult), args.Error(1) +} + +func (c mockCollection) UpdateOne(ctxt context.Context, filter interface{}, document interface{}, options *options.UpdateOptions) (*mongo.UpdateResult, error) { + args := c.Called(ctxt, filter, document, options) + return args.Get(0).(*mongo.UpdateResult), args.Error(1) +} + +func (c mockCollection) Find(ctxt context.Context, filter interface{}, result interface{}, options *options.FindOptions) error { + args := c.Called(ctxt, filter, result, options) + return args.Error(0) +} diff --git a/cmd/chartsvc/foundationdb/handler.go b/cmd/chartsvc/foundationdb/handler.go new file mode 100644 index 000000000..a28d778b9 --- /dev/null +++ b/cmd/chartsvc/foundationdb/handler.go @@ -0,0 +1,506 @@ +/* +Copyright (c) 2019 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package foundationdb + +import ( + "context" + "fmt" + "math" + "net/http" + "sort" + "strconv" + + "github.com/helm/monocular/cmd/chartsvc/foundationdb/datastore" + "github.com/helm/monocular/cmd/chartsvc/models" + "github.com/helm/monocular/cmd/chartsvc/utils" + + "github.com/gorilla/mux" + "github.com/kubeapps/common/response" + log "github.com/sirupsen/logrus" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// Params a key-value map of path params +type Params map[string]string + +// WithParams can be used to wrap handlers to take an extra arg for path params +type WithParams func(http.ResponseWriter, *http.Request, Params) + +func (h WithParams) ServeHTTP(w http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + h(w, req, vars) +} + +const chartCollection = "charts" +const filesCollection = "files" + +// count is used to parse the result of a $count operation in the database +type count struct { + Count int +} + +var dbClient datastore.Client +var db datastore.Database +var dbCloser func() + +//var db mongo.Database +var dbName string +var pathPrefix string + +//SetPathPrefix sets the URL prefix for the ChartSVC API endpoint +func SetPathPrefix(prefix string) { + pathPrefix = prefix +} + +//InitDBConfig sets FDB Document-Layer client and DB config for the ChartSVC API handler +func InitDBConfig(client datastore.Client, name string) { + dbClient = client + db, dbCloser = dbClient.Database(name) + dbName = name +} + +// getPageNumberAndSize extracts the page number and size of a request. Default (1, 0) if not set +func getPageNumberAndSize(req *http.Request) (int, int) { + page := req.FormValue("page") + size := req.FormValue("size") + pageInt, err := strconv.ParseUint(page, 10, 64) + if err != nil { + pageInt = 1 + } + // ParseUint will return 0 if size is a not positive integer + sizeInt, _ := strconv.ParseUint(size, 10, 64) + return int(pageInt), int(sizeInt) +} + +// showDuplicates returns if a request wants to retrieve charts. Default false +func showDuplicates(req *http.Request) bool { + return len(req.FormValue("showDuplicates")) > 0 +} + +// min returns the minimum of two integers. +// We are not using math.Min since that compares float64 +// and it's unnecessarily complex. +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func uniqChartList(charts []*models.Chart) []*models.Chart { + // We will keep track of unique digest:chart to avoid duplicates + chartDigests := map[string]bool{} + res := []*models.Chart{} + for _, c := range charts { + digest := c.ChartVersions[0].Digest + // Filter out the chart if we've seen the same digest before + if _, ok := chartDigests[digest]; !ok { + chartDigests[digest] = true + res = append(res, c) + } + } + return res +} + +func getPaginatedChartList(repo string, pageNumber, pageSize int, showDuplicates bool) (utils.ApiListResponse, interface{}, error) { + log.Debugf("Request for paginated chart list..") + + //Find all charts for repo name and sort by chart name + collection := db.Collection(chartCollection) + filter := bson.M{} + if repo != "" { + filter = bson.M{"repo.name": repo} + } + var charts []*models.Chart + err := collection.Find(context.Background(), filter, &charts, options.Find()) + if err != nil { + log.WithError(err).Errorf( + "Error fetching charts from DB for pagination %s", + repo, + ) + return newChartListResponse([]*models.Chart{}), utils.Meta{TotalPages: 0}, err + } + var tempChartMap map[string]*models.Chart = make(map[string]*models.Chart) + + chartsToSort := make([]*models.Chart, 0, len(tempChartMap)) + + if !showDuplicates { + for _, chart := range charts { + log.Debugf("Chart digest: %v.", chart.ChartVersions[0].Digest) + tempChartMap[chart.ChartVersions[0].Digest] = chart + } + log.Debugf("Charts in map: %v", len(tempChartMap)) + //Now just get all the values from our map + for _, v := range tempChartMap { + log.Debugf("Adding chart: %v to unique chart list.", *v) + chartsToSort = append(chartsToSort, v) + } + } else { + chartsToSort = charts + } + + //Sort the list of paginated charts by name + sort.Slice(chartsToSort, func(i, j int) bool { + return chartsToSort[i].Name < chartsToSort[j].Name + }) + + sortedCharts := chartsToSort + log.Debugf("Charts in sorted list: %v", len(sortedCharts)) + log.Debugf("Page size requested: %v", pageSize) + var paginatedCharts = sortedCharts + totalPages := 1 + if pageSize != 0 { + // If a pageSize is given, returns only the the specified number of charts and + // the number of pages + cc := count{} + cc.Count = len(sortedCharts) + totalPages = int(math.Ceil(float64(cc.Count) / float64(pageSize))) + + // If the page number is out of range, return the last one + if pageNumber > totalPages { + pageNumber = totalPages + } + paginatedCharts = sortedCharts[(pageNumber-1)*pageSize : pageNumber*pageSize] + } + + log.Debugf("Returning %v charts, Done.", len(paginatedCharts)) + return newChartListResponse(paginatedCharts), utils.Meta{TotalPages: totalPages}, nil +} + +// ListCharts returns a list of charts +func ListCharts(w http.ResponseWriter, req *http.Request) { + log.Debug("Request for charts..") + pageNumber, pageSize := getPageNumberAndSize(req) + cl, meta, err := getPaginatedChartList("", pageNumber, pageSize, showDuplicates(req)) + if err != nil { + log.WithError(err).Error("could not fetch charts") + response.NewErrorResponse(http.StatusInternalServerError, "could not fetch all charts").Write(w) + return + } + response.NewDataResponseWithMeta(cl, meta).Write(w) + log.Debug("Done.") +} + +// ListRepoCharts returns a list of charts in the given repo +func ListRepoCharts(w http.ResponseWriter, req *http.Request, params Params) { + log.Debug("Request for charts..") + pageNumber, pageSize := getPageNumberAndSize(req) + cl, meta, err := getPaginatedChartList(params["repo"], pageNumber, pageSize, showDuplicates(req)) + if err != nil { + log.WithError(err).Error("could not fetch charts") + response.NewErrorResponse(http.StatusInternalServerError, "could not fetch all charts").Write(w) + return + } + response.NewDataResponseWithMeta(cl, meta).Write(w) + log.Debug("Done.") +} + +// GetChart returns the chart from the given repo +func GetChart(w http.ResponseWriter, req *http.Request, params Params) { + var chart models.Chart + chartID := fmt.Sprintf("%s/%s", params["repo"], params["chartName"]) + + chartCollection := db.Collection(chartCollection) + filter := bson.M{"_id": chartID} + findResult := chartCollection.FindOne(context.Background(), filter, &chart, options.FindOne()) + if findResult == mongo.ErrNoDocuments { + log.WithError(findResult).Errorf("could not find chart with id %s", chartID) + response.NewErrorResponse(http.StatusNotFound, "could not find chart").Write(w) + return + } + + cr := newChartResponse(&chart) + response.NewDataResponse(cr).Write(w) +} + +// ListChartVersions returns a list of chart versions for the given chart +func ListChartVersions(w http.ResponseWriter, req *http.Request, params Params) { + var chart models.Chart + chartID := fmt.Sprintf("%s/%s", params["repo"], params["chartName"]) + + chartCollection := db.Collection(chartCollection) + filter := bson.M{"_id": chartID} + findResult := chartCollection.FindOne(context.Background(), filter, &chart, options.FindOne()) + if findResult == mongo.ErrNoDocuments { + log.WithError(findResult).Errorf("could not find chart with id %s", chartID) + response.NewErrorResponse(http.StatusNotFound, "could not find chart").Write(w) + return + } + + cvl := newChartVersionListResponse(&chart) + response.NewDataResponse(cvl).Write(w) +} + +// GetChartVersion returns the given chart version +func GetChartVersion(w http.ResponseWriter, req *http.Request, params Params) { + var chart models.Chart + chartID := fmt.Sprintf("%s/%s", params["repo"], params["chartName"]) + + chartCollection := db.Collection(chartCollection) + filter := bson.M{ + "_id": chartID, + "chartversions": bson.M{"$elemMatch": bson.M{"version": params["version"]}}, + } + projection := bson.M{ + "name": 1, "repo": 1, "description": 1, "home": 1, "keywords": 1, "maintainers": 1, "sources": 1, + "chartversions": 1, + } + findResult := chartCollection.FindOne(context.Background(), filter, &chart, options.FindOne().SetProjection(projection)) + if findResult == mongo.ErrNoDocuments { + log.WithError(findResult).Errorf("could not find chart with id %s", chartID) + response.NewErrorResponse(http.StatusNotFound, "could not find chart").Write(w) + return + } + + for i := range chart.ChartVersions { + if chart.ChartVersions[i].Version == params["version"] { + chart.ChartVersions = chart.ChartVersions[i : i+1] + break + } + } + // Cut the versions slice down to just one element + cvr := newChartVersionResponse(&chart, chart.ChartVersions[0]) + response.NewDataResponse(cvr).Write(w) +} + +// GetChartIcon returns the icon for a given chart +func GetChartIcon(w http.ResponseWriter, req *http.Request, params Params) { + var chart models.Chart + chartID := fmt.Sprintf("%s/%s", params["repo"], params["chartName"]) + + chartCollection := db.Collection(chartCollection) + filter := bson.M{"_id": chartID} + findResult := chartCollection.FindOne(context.Background(), filter, &chart, options.FindOne()) + if findResult == mongo.ErrNoDocuments { + log.WithError(findResult).Errorf("could not find chart with id %s", chartID) + http.NotFound(w, req) + return + } + if chart.RawIcon == nil { + http.NotFound(w, req) + return + } + + if chart.IconContentType != "" { + // Force the Content-Type header because the autogenerated type does not work for + // image/svg+xml. It is detected as plain text + w.Header().Set("Content-Type", chart.IconContentType) + } + + w.Write(chart.RawIcon) +} + +// GetChartVersionReadme returns the README for a given chart +func GetChartVersionReadme(w http.ResponseWriter, req *http.Request, params Params) { + + var files models.ChartFiles + fileID := fmt.Sprintf("%s/%s-%s", params["repo"], params["chartName"], params["version"]) + + filesCollection := db.Collection(filesCollection) + filter := bson.M{"_id": fileID} + findResult := filesCollection.FindOne(context.Background(), filter, &files, options.FindOne()) + if findResult == mongo.ErrNoDocuments { + log.WithError(findResult).Errorf("could not find files with id %s", fileID) + http.NotFound(w, req) + return + } + readme := []byte(files.Readme) + if len(readme) == 0 { + log.Errorf("could not find a README for id %s", fileID) + http.NotFound(w, req) + return + } + w.Write(readme) +} + +// GetChartVersionValues returns the values.yaml for a given chart +func GetChartVersionValues(w http.ResponseWriter, req *http.Request, params Params) { + var files models.ChartFiles + + fileID := fmt.Sprintf("%s/%s-%s", params["repo"], params["chartName"], params["version"]) + filesCollection := db.Collection(filesCollection) + filter := bson.M{"_id": fileID} + findResult := filesCollection.FindOne(context.Background(), filter, &files, options.FindOne()) + if findResult == mongo.ErrNoDocuments { + log.WithError(findResult).Errorf("could not find values.yaml with id %s", fileID) + http.NotFound(w, req) + return + } + + w.Write([]byte(files.Values)) +} + +// GetChartVersionSchema returns the values.schema.json for a given chart +func GetChartVersionSchema(w http.ResponseWriter, req *http.Request, params Params) { + + var files models.ChartFiles + + fileID := fmt.Sprintf("%s/%s-%s", params["repo"], params["chartName"], params["version"]) + filter := bson.M{"_id": fileID} + filesCollection := db.Collection(filesCollection) + findResult := filesCollection.FindOne(context.Background(), filter, &files, options.FindOne()) + if findResult == mongo.ErrNoDocuments { + log.WithError(findResult).Errorf("could not find values.schema.json with id %s", fileID) + http.NotFound(w, req) + return + } + + w.Write([]byte(files.Schema)) +} + +// ListChartsWithFilters returns the list of repos that contains the given chart and the latest version found +func ListChartsWithFilters(w http.ResponseWriter, req *http.Request, params Params) { + + var charts []*models.Chart + + chartCollection := db.Collection(chartCollection) + filter := bson.M{ + "name": params["chartName"], + "chartversions": bson.M{ + "$elemMatch": bson.M{"version": req.FormValue("version"), "appversion": req.FormValue("appversion")}, + }} + projection := bson.M{ + "name": 1, "repo": 1, + "chartversions": bson.M{"$slice": 1}, + } + err := chartCollection.Find(context.Background(), filter, &charts, options.Find().SetProjection(projection)) + if err != nil { + log.WithError(err).Errorf( + "Error finding charts with the given name %s, version %s and appversion %s", + params["chartName"], req.FormValue("version"), req.FormValue("appversion"), + ) + // continue to return empty list + } + + chartResponse := charts + if !showDuplicates(req) { + chartResponse = uniqChartList(charts) + } + + cl := newChartListResponse(chartResponse) + response.NewDataResponse(cl).Write(w) +} + +// SearchCharts returns the list of charts that matches the query param in any of these fields: +// - name +// - description +// - repository name +// - any keyword +// - any source +// - any maintainer name +func SearchCharts(w http.ResponseWriter, req *http.Request, params Params) { + + query := req.FormValue("q") + var charts []*models.Chart + + chartCollection := db.Collection(chartCollection) + filter := bson.M{ + "$or": []bson.M{ + {"name": bson.M{"$regex": query}}, + {"description": bson.M{"$regex": query}}, + {"repo.name": bson.M{"$regex": query}}, + {"keywords": bson.M{"$elemMatch": bson.M{"$regex": query}}}, + {"sources": bson.M{"$elemMatch": bson.M{"$regex": query}}}, + {"maintainers": bson.M{"$elemMatch": bson.M{"name": bson.M{"$regex": query}}}}, + }, + } + if params["repo"] != "" { + filter["repo.name"] = params["repo"] + } + err := chartCollection.Find(context.Background(), filter, &charts, options.Find()) + if err != nil { + log.WithError(err).Errorf( + "Error finding charts with the given name %s, version %s and appversion %s", + params["chartName"], req.FormValue("version"), req.FormValue("appversion"), + ) + // continue to return empty list + } + + chartResponse := charts + if !showDuplicates(req) { + chartResponse = uniqChartList(charts) + } + + cl := newChartListResponse(uniqChartList(chartResponse)) + response.NewDataResponse(cl).Write(w) +} + +func newChartResponse(c *models.Chart) *utils.ApiResponse { + latestCV := c.ChartVersions[0] + return &utils.ApiResponse{ + Type: "chart", + ID: c.ID, + Attributes: chartAttributes(*c), + Links: utils.SelfLink{Self: pathPrefix + "/charts/" + c.ID}, + Relationships: utils.RelMap{ + "latestChartVersion": utils.Rel{ + Data: chartVersionAttributes(c.ID, latestCV), + Links: utils.SelfLink{Self: pathPrefix + "/charts/" + c.ID + "/versions/" + latestCV.Version}, + }, + }, + } +} + +func newChartListResponse(charts []*models.Chart) utils.ApiListResponse { + cl := utils.ApiListResponse{} + for _, c := range charts { + cl = append(cl, newChartResponse(c)) + } + return cl +} + +func chartVersionAttributes(cid string, cv models.ChartVersion) models.ChartVersion { + cv.Readme = pathPrefix + "/assets/" + cid + "/versions/" + cv.Version + "/README.md" + cv.Values = pathPrefix + "/assets/" + cid + "/versions/" + cv.Version + "/values.yaml" + return cv +} + +func chartAttributes(c models.Chart) models.Chart { + if c.RawIcon != nil { + c.Icon = pathPrefix + "/assets/" + c.ID + "/logo" + } else { + // If the icon wasn't processed, it is either not set or invalid + c.Icon = "" + } + return c +} + +func newChartVersionResponse(c *models.Chart, cv models.ChartVersion) *utils.ApiResponse { + return &utils.ApiResponse{ + Type: "chartVersion", + ID: fmt.Sprintf("%s-%s", c.ID, cv.Version), + Attributes: chartVersionAttributes(c.ID, cv), + Links: utils.SelfLink{Self: pathPrefix + "/charts/" + c.ID + "/versions/" + cv.Version}, + Relationships: utils.RelMap{ + "chart": utils.Rel{ + Data: chartAttributes(*c), + Links: utils.SelfLink{Self: pathPrefix + "/charts/" + c.ID}, + }, + }, + } +} + +func newChartVersionListResponse(c *models.Chart) utils.ApiListResponse { + var cvl utils.ApiListResponse + for _, cv := range c.ChartVersions { + cvl = append(cvl, newChartVersionResponse(c, cv)) + } + + return cvl +} diff --git a/cmd/chartsvc/foundationdb/handler_test.go b/cmd/chartsvc/foundationdb/handler_test.go new file mode 100644 index 000000000..2c8eaeeda --- /dev/null +++ b/cmd/chartsvc/foundationdb/handler_test.go @@ -0,0 +1,867 @@ +/* +Copyright (c) 2019 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package foundationdb + +import ( + "bytes" + "encoding/json" + "image/color" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/helm/monocular/cmd/chartsvc/foundationdb/datastore" + "github.com/helm/monocular/cmd/chartsvc/models" + "github.com/helm/monocular/cmd/chartsvc/utils" + + "github.com/disintegration/imaging" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +var cc count + +const testChartReadme = "# Quickstart\n\n```bash\nhelm install my-repo/my-chart\n```" +const testChartValues = "image:\n registry: docker.io\n repository: my-repo/my-chart\n tag: 0.1.0" + +const fdbURL = "mongodb://fdb-service/27016" +const fDB = "charts" + +func iconBytes() []byte { + var b bytes.Buffer + img := imaging.New(1, 1, color.White) + imaging.Encode(&b, img, imaging.PNG) + return b.Bytes() +} + +func Test_chartAttributes(t *testing.T) { + tests := []struct { + name string + chart models.Chart + }{ + {"chart has no icon", models.Chart{ + ID: "stable/wordpress", + }}, + {"chart has a icon", models.Chart{ + ID: "repo/mychart", RawIcon: iconBytes(), IconContentType: "image/svg", + }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := chartAttributes(tt.chart) + assert.Equal(t, tt.chart.ID, c.ID) + assert.Equal(t, tt.chart.RawIcon, c.RawIcon) + if len(tt.chart.RawIcon) == 0 { + assert.Equal(t, len(c.Icon), 0, "icon url should be undefined") + } else { + assert.Equal(t, c.Icon, pathPrefix+"/assets/"+tt.chart.ID+"/logo", "the icon url should be the same") + assert.Equal(t, c.IconContentType, tt.chart.IconContentType, "the icon content type should be the same") + } + }) + } +} + +func Test_chartVersionAttributes(t *testing.T) { + tests := []struct { + name string + chart models.Chart + }{ + {"my-chart", models.Chart{ + ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.1.0"}}, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cv := chartVersionAttributes(tt.chart.ID, tt.chart.ChartVersions[0]) + assert.Equal(t, cv.Version, tt.chart.ChartVersions[0].Version, "version string should be the same") + assert.Equal(t, cv.Readme, pathPrefix+"/assets/"+tt.chart.ID+"/versions/"+tt.chart.ChartVersions[0].Version+"/README.md", "README.md resource path should be the same") + assert.Equal(t, cv.Values, pathPrefix+"/assets/"+tt.chart.ID+"/versions/"+tt.chart.ChartVersions[0].Version+"/values.yaml", "values.yaml resource path should be the same") + }) + } +} + +func Test_newChartResponse(t *testing.T) { + tests := []struct { + name string + chart models.Chart + }{ + {"chart has only one version", models.Chart{ + ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "1.2.3"}}}, + }, + {"chart has many versions", models.Chart{ + ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.1.2"}, {Version: "0.1.0"}}, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cResponse := newChartResponse(&tt.chart) + assert.Equal(t, cResponse.Type, "chart", "response type is chart") + assert.Equal(t, cResponse.ID, tt.chart.ID, "chart ID should be the same") + assert.Equal(t, cResponse.Relationships["latestChartVersion"].Data.(models.ChartVersion).Version, tt.chart.ChartVersions[0].Version, "latestChartVersion should match version at index 0") + assert.Equal(t, cResponse.Links.(utils.SelfLink).Self, pathPrefix+"/charts/"+tt.chart.ID, "self link should be the same") + assert.Equal(t, len(cResponse.Attributes.(models.Chart).ChartVersions), len(tt.chart.ChartVersions), "number of chart versions in the response should be the same") + }) + } +} + +func Test_newChartListResponse(t *testing.T) { + tests := []struct { + name string + input []*models.Chart + result []*models.Chart + }{ + {"no charts", []*models.Chart{}, []*models.Chart{}}, + {"has one chart", []*models.Chart{ + {ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + }, []*models.Chart{ + {ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + }}, + {"has two charts", []*models.Chart{ + {ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + {ID: "stable/wordpress", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "1234"}, {Version: "1.2.2", Digest: "12345"}}}, + }, []*models.Chart{ + {ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + {ID: "stable/wordpress", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "1234"}, {Version: "1.2.2", Digest: "12345"}}}, + }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clResponse := newChartListResponse(tt.input) + assert.Equal(t, len(clResponse), len(tt.result), "number of charts in response should be the same") + for i := range tt.result { + assert.Equal(t, clResponse[i].Type, "chart", "response type is chart") + assert.Equal(t, clResponse[i].ID, tt.result[i].ID, "chart ID should be the same") + assert.Equal(t, clResponse[i].Relationships["latestChartVersion"].Data.(models.ChartVersion).Version, tt.result[i].ChartVersions[0].Version, "latestChartVersion should match version at index 0") + assert.Equal(t, clResponse[i].Links.(utils.SelfLink).Self, pathPrefix+"/charts/"+tt.result[i].ID, "self link should be the same") + assert.Equal(t, len(clResponse[i].Attributes.(models.Chart).ChartVersions), len(tt.result[i].ChartVersions), "number of chart versions in the response should be the same") + } + }) + } +} + +func Test_newChartVersionResponse(t *testing.T) { + tests := []struct { + name string + chart models.Chart + }{ + {"my-chart", models.Chart{ + ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.1.0"}, {Version: "0.2.3"}}, + }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for i := range tt.chart.ChartVersions { + cvResponse := newChartVersionResponse(&tt.chart, tt.chart.ChartVersions[i]) + assert.Equal(t, cvResponse.Type, "chartVersion", "response type is chartVersion") + assert.Equal(t, cvResponse.ID, tt.chart.ID+"-"+tt.chart.ChartVersions[i].Version, "reponse id should have chart version suffix") + assert.Equal(t, cvResponse.Links.(interface{}).(utils.SelfLink).Self, pathPrefix+"/charts/"+tt.chart.ID+"/versions/"+tt.chart.ChartVersions[i].Version, "self link should be the same") + assert.Equal(t, cvResponse.Attributes.(models.ChartVersion).Version, tt.chart.ChartVersions[i].Version, "chart version in the response should be the same") + assert.Equal(t, cvResponse.Relationships["chart"].Data.(interface{}).(models.Chart), tt.chart, "chart in relatioship matches") + } + }) + } +} + +func Test_newChartVersionListResponse(t *testing.T) { + tests := []struct { + name string + chart models.Chart + }{ + {"chart has no versions", models.Chart{ + ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{}, + }}, + {"chart has one version", models.Chart{ + ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1"}}, + }}, + {"chart has many versions", models.Chart{ + ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1"}, {Version: "0.0.2"}}, + }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cvListResponse := newChartVersionListResponse(&tt.chart) + assert.Equal(t, len(cvListResponse), len(tt.chart.ChartVersions), "number of chart versions in response should be the same") + for i := range tt.chart.ChartVersions { + assert.Equal(t, cvListResponse[i].Type, "chartVersion", "response type is chartVersion") + assert.Equal(t, cvListResponse[i].ID, tt.chart.ID+"-"+tt.chart.ChartVersions[i].Version, "reponse id should have chart version suffix") + assert.Equal(t, cvListResponse[i].Links.(interface{}).(utils.SelfLink).Self, pathPrefix+"/charts/"+tt.chart.ID+"/versions/"+tt.chart.ChartVersions[i].Version, "self link should be the same") + assert.Equal(t, cvListResponse[i].Attributes.(models.ChartVersion).Version, tt.chart.ChartVersions[i].Version, "chart version in the response should be the same") + } + }) + } +} + +func Test_listCharts(t *testing.T) { + pageSize := 2 + tests := []struct { + name string + query string + dbQueryResult []*models.Chart + chartListResult []*models.Chart + meta utils.Meta + }{ + {"no charts", "", []*models.Chart{}, []*models.Chart{}, utils.Meta{TotalPages: 1}}, + {"one chart", "", []*models.Chart{ + {ID: "my-repo/my-chart", Name: "my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + }, []*models.Chart{ + {ID: "my-repo/my-chart", Name: "my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + }, utils.Meta{TotalPages: 1}}, + {"two charts", "", []*models.Chart{ + {ID: "my-repo/my-chart", Name: "my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + {ID: "stable/dokuwiki", Name: "dokuwiki", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "1234"}, {Version: "1.2.2", Digest: "12345"}}}, + }, []*models.Chart{ + {ID: "stable/dokuwiki", Name: "dokuwiki", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "1234"}, {Version: "1.2.2", Digest: "12345"}}}, + {ID: "my-repo/my-chart", Name: "my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + }, utils.Meta{TotalPages: 1}}, + // Pagination tests + {"four charts with pagination", "?size=" + strconv.Itoa(pageSize), []*models.Chart{ + {ID: "my-repo/my-chart", Name: "my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + {ID: "stable/dokuwiki", Name: "dokuwiki", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "1234"}}}, + {ID: "stable/drupal", Name: "drupal", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "12345"}}}, + {ID: "stable/wordpress", Name: "wordpress", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "123456"}}}, + }, []*models.Chart{ + {ID: "stable/dokuwiki", Name: "dokuwiki", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "1234"}}}, + {ID: "stable/drupal", Name: "drupal", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "12345"}}}, + }, utils.Meta{TotalPages: 2}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var m mock.Mock + dbClient = datastore.NewMockClient(&m) + db, _ = dbClient.Database("test") + var chartsList []*models.Chart + + m.On("Find", mock.Anything, mock.Anything, &chartsList, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(2).(*[]*models.Chart) = tt.dbQueryResult + }) + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/charts"+tt.query, nil) + ListCharts(w, req) + + m.AssertExpectations(t) + assert.Equal(t, http.StatusOK, w.Code) + + var b utils.BodyAPIListResponse + json.NewDecoder(w.Body).Decode(&b) + if b.Data == nil { + t.Fatal("chart list shouldn't be null") + } + data := *b.Data + + assert.Len(t, data, len(tt.chartListResult)) + + for i, resp := range data { + assert.Equal(t, resp.ID, tt.chartListResult[i].ID, "chart id in the response should be the same") + assert.Equal(t, resp.Type, "chart", "response type is chart") + assert.Equal(t, resp.Links.(map[string]interface{})["self"], pathPrefix+"/charts/"+tt.chartListResult[i].ID, "self link should be the same") + assert.Equal(t, resp.Relationships["latestChartVersion"].Data.(map[string]interface{})["version"], tt.chartListResult[i].ChartVersions[0].Version, "latestChartVersion should match version at index 0") + } + assert.Equal(t, b.Meta, tt.meta, "response meta should be the same") + }) + } +} + +func Test_listRepoCharts(t *testing.T) { + pageSize := 2 + tests := []struct { + name string + repo string + query string + dbQueryResult []*models.Chart + chartListResult []*models.Chart + meta utils.Meta + }{ + {"repo has no charts", "my-repo", "", []*models.Chart{}, []*models.Chart{}, utils.Meta{TotalPages: 1}}, + {"repo has one chart", "my-repo", "", []*models.Chart{ + {ID: "my-repo/my-chart", Name: "my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + }, []*models.Chart{ + {ID: "my-repo/my-chart", Name: "my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + }, utils.Meta{TotalPages: 1}}, + {"repo has many charts", "my-repo", "", []*models.Chart{ + {ID: "my-repo/my-chart", Name: "my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + {ID: "my-repo/dokuwiki", Name: "dokuwiki", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "1234"}, {Version: "1.2.2", Digest: "12345"}}}, + }, []*models.Chart{ + {ID: "my-repo/dokuwiki", Name: "dokuwiki", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "1234"}, {Version: "1.2.2", Digest: "12345"}}}, + {ID: "my-repo/my-chart", Name: "my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + }, utils.Meta{TotalPages: 1}}, + {"repo has many charts with pagination", "my-repo", "?size=" + strconv.Itoa(pageSize), []*models.Chart{ + {ID: "my-repo/my-chart3", Name: "my-chart3", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, + {ID: "my-repo/my-chart1", Name: "my-chart1", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "1234"}}}, + {ID: "my-repo/my-chart2", Name: "my-chart2", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "12345"}}}, + }, []*models.Chart{ + {ID: "my-repo/my-chart1", Name: "my-chart1", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "1234"}}}, + {ID: "my-repo/my-chart2", Name: "my-chart2", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "12345"}}}, + }, utils.Meta{TotalPages: 2}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var m mock.Mock + dbClient = datastore.NewMockClient(&m) + db, _ = dbClient.Database("test") + var chartsList []*models.Chart + m.On("Find", mock.Anything, bson.M{"repo.name": "my-repo"}, &chartsList, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(2).(*[]*models.Chart) = tt.dbQueryResult + }) + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/charts/"+tt.repo+tt.query, nil) + params := Params{ + "repo": "my-repo", + } + + ListRepoCharts(w, req, params) + + m.AssertExpectations(t) + assert.Equal(t, http.StatusOK, w.Code) + + var b utils.BodyAPIListResponse + json.NewDecoder(w.Body).Decode(&b) + data := *b.Data + assert.Len(t, data, len(tt.chartListResult)) + for i, resp := range data { + assert.Equal(t, resp.ID, tt.chartListResult[i].ID, "chart id in the response should be the same") + assert.Equal(t, resp.Type, "chart", "response type is chart") + assert.Equal(t, resp.Relationships["latestChartVersion"].Data.(map[string]interface{})["version"], tt.chartListResult[i].ChartVersions[0].Version, "latestChartVersion should match version at index 0") + } + assert.Equal(t, b.Meta, tt.meta, "response meta should be the same") + }) + } +} + +func Test_getChart(t *testing.T) { + tests := []struct { + name string + err error + chart models.Chart + wantCode int + }{ + { + "chart does not exist", + mongo.ErrNoDocuments, + models.Chart{ID: "my-repo/my-chart"}, + http.StatusNotFound, + }, + { + "chart exists", + nil, + models.Chart{ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.1.0"}}}, + http.StatusOK, + }, + { + "chart has multiple versions", + nil, + models.Chart{ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.1.0"}, {Version: "0.0.1"}}}, + http.StatusOK, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var m mock.Mock + dbClient = datastore.NewMockClient(&m) + db, _ = dbClient.Database("test") + + if tt.err != nil { + m.On("FindOne", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tt.err) + } else { + m.On("FindOne", mock.Anything, mock.Anything, &models.Chart{}, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(2).(*models.Chart) = tt.chart + }) + } + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/charts/"+tt.chart.ID, nil) + parts := strings.Split(tt.chart.ID, "/") + params := Params{ + "repo": parts[0], + "chartName": parts[1], + } + + GetChart(w, req, params) + + m.AssertExpectations(t) + assert.Equal(t, tt.wantCode, w.Code) + if tt.wantCode == http.StatusOK { + var b utils.BodyAPIResponse + json.NewDecoder(w.Body).Decode(&b) + assert.Equal(t, b.Data.ID, tt.chart.ID, "chart id in the response should be the same") + assert.Equal(t, b.Data.Type, "chart", "response type is chart") + assert.Equal(t, b.Data.Links.(map[string]interface{})["self"], pathPrefix+"/charts/"+tt.chart.ID, "self link should be the same") + assert.Equal(t, b.Data.Relationships["latestChartVersion"].Data.(map[string]interface{})["version"], tt.chart.ChartVersions[0].Version, "latestChartVersion should match version at index 0") + } + }) + } +} + +func Test_listChartVersions(t *testing.T) { + tests := []struct { + name string + err error + chart models.Chart + wantCode int + }{ + { + "chart does not exist", + mongo.ErrNoDocuments, + models.Chart{ID: "my-repo/my-chart"}, + http.StatusNotFound, + }, + { + "chart exists", + nil, + models.Chart{ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.1.0"}}}, + http.StatusOK, + }, + { + "chart has multiple versions", + nil, + models.Chart{ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.1.0"}, {Version: "0.0.1"}}}, + http.StatusOK, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var m mock.Mock + dbClient = datastore.NewMockClient(&m) + db, _ = dbClient.Database("test") + + if tt.err != nil { + m.On("FindOne", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tt.err) + } else { + m.On("FindOne", mock.Anything, mock.Anything, &models.Chart{}, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(2).(*models.Chart) = tt.chart + }) + } + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/charts/"+tt.chart.ID+"/versions", nil) + parts := strings.Split(tt.chart.ID, "/") + params := Params{ + "repo": parts[0], + "chartName": parts[1], + } + + ListChartVersions(w, req, params) + + m.AssertExpectations(t) + assert.Equal(t, tt.wantCode, w.Code) + if tt.wantCode == http.StatusOK { + var b utils.BodyAPIListResponse + json.NewDecoder(w.Body).Decode(&b) + data := *b.Data + for i, resp := range data { + assert.Equal(t, resp.ID, tt.chart.ID+"-"+tt.chart.ChartVersions[i].Version, "chart id in the response should be the same") + assert.Equal(t, resp.Type, "chartVersion", "response type is chartVersion") + assert.Equal(t, resp.Attributes.(map[string]interface{})["version"], tt.chart.ChartVersions[i].Version, "chart version should match") + } + } + }) + } +} + +func Test_getChartVersion(t *testing.T) { + tests := []struct { + name string + err error + chart models.Chart + wantCode int + }{ + { + "chart does not exist", + mongo.ErrNoDocuments, + models.Chart{ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.1.0"}}}, + http.StatusNotFound, + }, + { + "chart exists", + nil, + models.Chart{ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.1.0"}}}, + http.StatusOK, + }, + { + "chart has multiple versions", + nil, + models.Chart{ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.1.0"}, {Version: "0.0.1"}}}, + http.StatusOK, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var m mock.Mock + dbClient = datastore.NewMockClient(&m) + db, _ = dbClient.Database("test") + + if tt.err != nil { + m.On("FindOne", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tt.err) + } else { + m.On("FindOne", mock.Anything, mock.Anything, &models.Chart{}, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(2).(*models.Chart) = tt.chart + }) + } + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/charts/"+tt.chart.ID+"/versions/"+tt.chart.ChartVersions[0].Version, nil) + parts := strings.Split(tt.chart.ID, "/") + params := Params{ + "repo": parts[0], + "chartName": parts[1], + "version": tt.chart.ChartVersions[0].Version, + } + + GetChartVersion(w, req, params) + + m.AssertExpectations(t) + assert.Equal(t, tt.wantCode, w.Code) + if tt.wantCode == http.StatusOK { + var b utils.BodyAPIResponse + json.NewDecoder(w.Body).Decode(&b) + assert.Equal(t, b.Data.ID, tt.chart.ID+"-"+tt.chart.ChartVersions[0].Version, "chart id in the response should be the same") + assert.Equal(t, b.Data.Type, "chartVersion", "response type is chartVersion") + assert.Equal(t, b.Data.Attributes.(map[string]interface{})["version"], tt.chart.ChartVersions[0].Version, "chart version should match") + } + }) + } +} + +func Test_getChartIcon(t *testing.T) { + tests := []struct { + name string + err error + chart models.Chart + wantCode int + }{ + { + "chart does not exist", + mongo.ErrNoDocuments, + models.Chart{ID: "my-repo/my-chart"}, + http.StatusNotFound, + }, + { + "chart has icon", + nil, + models.Chart{ID: "my-repo/my-chart", RawIcon: iconBytes(), IconContentType: "image/png"}, + http.StatusOK, + }, + { + "chart does not have a icon", + nil, + models.Chart{ID: "my-repo/my-chart"}, + http.StatusNotFound, + }, + { + "chart has icon with custom type", + nil, + models.Chart{ID: "my-repo/my-chart", RawIcon: iconBytes(), IconContentType: "image/svg"}, + http.StatusOK, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var m mock.Mock + dbClient = datastore.NewMockClient(&m) + db, _ = dbClient.Database("test") + + if tt.err != nil { + m.On("FindOne", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tt.err) + } else { + m.On("FindOne", mock.Anything, mock.Anything, &models.Chart{}, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(2).(*models.Chart) = tt.chart + }) + } + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/assets/"+tt.chart.ID+"/logo", nil) + parts := strings.Split(tt.chart.ID, "/") + params := Params{ + "repo": parts[0], + "chartName": parts[1], + } + + GetChartIcon(w, req, params) + + m.AssertExpectations(t) + assert.Equal(t, tt.wantCode, w.Code, "http status code should match") + if tt.wantCode == http.StatusOK { + assert.Equal(t, w.Body.Bytes(), tt.chart.RawIcon, "raw icon data should match") + assert.Equal(t, w.Header().Get("Content-Type"), tt.chart.IconContentType, "icon content type should match") + } + }) + } +} + +func Test_getChartVersionReadme(t *testing.T) { + tests := []struct { + name string + version string + err error + files models.ChartFiles + wantCode int + }{ + { + "chart does not exist", + "0.1.0", + mongo.ErrNoDocuments, + models.ChartFiles{ID: "my-repo/my-chart"}, + http.StatusNotFound, + }, + { + "chart exists", + "1.2.3", + nil, + models.ChartFiles{ID: "my-repo/my-chart", Readme: testChartReadme}, + http.StatusOK, + }, + { + "chart does not have a readme", + "1.1.1", + nil, + models.ChartFiles{ID: "my-repo/my-chart"}, + http.StatusNotFound, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var m mock.Mock + dbClient = datastore.NewMockClient(&m) + db, _ = dbClient.Database("test") + + if tt.err != nil { + m.On("FindOne", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tt.err) + } else { + m.On("FindOne", mock.Anything, mock.Anything, &models.ChartFiles{}, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(2).(*models.ChartFiles) = tt.files + }) + } + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/assets/"+tt.files.ID+"/versions/"+tt.version+"/README.md", nil) + parts := strings.Split(tt.files.ID, "/") + params := Params{ + "repo": parts[0], + "chartName": parts[1], + "version": "0.1.0", + } + + GetChartVersionReadme(w, req, params) + + m.AssertExpectations(t) + assert.Equal(t, tt.wantCode, w.Code, "http status code should match") + if tt.wantCode == http.StatusOK { + assert.Equal(t, string(w.Body.Bytes()), tt.files.Readme, "content of the readme should match") + } + }) + } +} + +func Test_getChartVersionValues(t *testing.T) { + tests := []struct { + name string + version string + err error + files models.ChartFiles + wantCode int + }{ + { + "chart does not exist", + "0.1.0", + mongo.ErrNoDocuments, + models.ChartFiles{ID: "my-repo/my-chart"}, + http.StatusNotFound, + }, + { + "chart exists", + "3.2.1", + nil, + models.ChartFiles{ID: "my-repo/my-chart", Values: testChartValues}, + http.StatusOK, + }, + { + "chart does not have values.yaml", + "2.2.2", + nil, + models.ChartFiles{ID: "my-repo/my-chart"}, + http.StatusOK, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var m mock.Mock + dbClient = datastore.NewMockClient(&m) + db, _ = dbClient.Database("test") + + if tt.err != nil { + m.On("FindOne", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(tt.err) + } else { + m.On("FindOne", mock.Anything, mock.Anything, &models.ChartFiles{}, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(2).(*models.ChartFiles) = tt.files + }) + } + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/assets/"+tt.files.ID+"/versions/"+tt.version+"/values.yaml", nil) + parts := strings.Split(tt.files.ID, "/") + params := Params{ + "repo": parts[0], + "chartName": parts[1], + "version": "0.1.0", + } + + GetChartVersionValues(w, req, params) + + m.AssertExpectations(t) + assert.Equal(t, tt.wantCode, w.Code, "http status code should match") + if tt.wantCode == http.StatusOK { + assert.Equal(t, string(w.Body.Bytes()), tt.files.Values, "content of values.yaml should match") + } + }) + } +} + +func Test_findLatestChart(t *testing.T) { + t.Run("returns mocked chart", func(t *testing.T) { + chart := &models.Chart{ + Name: "foo", + ID: "foo", + Repo: models.Repo{Name: "bar"}, + ChartVersions: []models.ChartVersion{ + models.ChartVersion{Version: "1.0.0", AppVersion: "0.1.0"}, + models.ChartVersion{Version: "0.0.1", AppVersion: "0.1.0"}, + }, + } + charts := []*models.Chart{chart} + reqVersion := "1.0.0" + reqAppVersion := "0.1.0" + + var m mock.Mock + dbClient = datastore.NewMockClient(&m) + db, _ = dbClient.Database("test") + var chartsList []*models.Chart + m.On("Find", mock.Anything, mock.Anything, &chartsList, mock.Anything).Run(func(args mock.Arguments) { + *args.Get(2).(*[]*models.Chart) = charts + }).Return(nil) + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/charts?name="+chart.Name+"&version="+reqVersion+"&appversion="+reqAppVersion, nil) + params := Params{ + "name": chart.Name, + "version": reqVersion, + "appversion": reqAppVersion, + } + + ListChartsWithFilters(w, req, params) + + var b utils.BodyAPIListResponse + json.NewDecoder(w.Body).Decode(&b) + if b.Data == nil { + t.Fatal("chart list shouldn't be null") + } + data := *b.Data + + if data[0].ID != chart.ID { + t.Errorf("Expecting %v, received %v", chart, data[0].ID) + } + }) + t.Run("ignores duplicated chart", func(t *testing.T) { + charts := []*models.Chart{ + {Name: "foo", ID: "stable/foo", Repo: models.Repo{Name: "bar"}, ChartVersions: []models.ChartVersion{models.ChartVersion{Version: "1.0.0", AppVersion: "0.1.0", Digest: "123"}}}, + {Name: "foo", ID: "bitnami/foo", Repo: models.Repo{Name: "bar"}, ChartVersions: []models.ChartVersion{models.ChartVersion{Version: "1.0.0", AppVersion: "0.1.0", Digest: "123"}}}, + } + reqVersion := "1.0.0" + reqAppVersion := "0.1.0" + + var m mock.Mock + dbClient = datastore.NewMockClient(&m) + db, _ = dbClient.Database("test") + var chartsList []*models.Chart + m.On("Find", mock.Anything, mock.Anything, &chartsList, mock.Anything).Run(func(args mock.Arguments) { + *args.Get(2).(*[]*models.Chart) = charts + }).Return(nil) + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/charts?name="+charts[0].Name+"&version="+reqVersion+"&appversion="+reqAppVersion, nil) + params := Params{ + "name": charts[0].Name, + "version": reqVersion, + "appversion": reqAppVersion, + } + + ListChartsWithFilters(w, req, params) + + var b utils.BodyAPIListResponse + json.NewDecoder(w.Body).Decode(&b) + if b.Data == nil { + t.Fatal("chart list shouldn't be null") + } + data := *b.Data + + assert.Equal(t, len(data), 1, "it should return a single chart") + if data[0].ID != charts[0].ID { + t.Errorf("Expecting %v, received %v", charts[0], data[0].ID) + } + }) + t.Run("includes duplicated charts when showDuplicates param set", func(t *testing.T) { + charts := []*models.Chart{ + {Name: "foo", ID: "stable/foo", Repo: models.Repo{Name: "bar"}, ChartVersions: []models.ChartVersion{models.ChartVersion{Version: "1.0.0", AppVersion: "0.1.0", Digest: "123"}}}, + {Name: "foo", ID: "bitnami/foo", Repo: models.Repo{Name: "bar"}, ChartVersions: []models.ChartVersion{models.ChartVersion{Version: "1.0.0", AppVersion: "0.1.0", Digest: "123"}}}, + } + reqVersion := "1.0.0" + reqAppVersion := "0.1.0" + + var m mock.Mock + dbClient = datastore.NewMockClient(&m) + db, _ = dbClient.Database("test") + var chartsList []*models.Chart + m.On("Find", mock.Anything, mock.Anything, &chartsList, mock.Anything).Run(func(args mock.Arguments) { + *args.Get(2).(*[]*models.Chart) = charts + }).Return(nil) + + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/charts?showDuplicates=true&name="+charts[0].Name+"&version="+reqVersion+"&appversion="+reqAppVersion, nil) + params := Params{ + "name": charts[0].Name, + "version": reqVersion, + "appversion": reqAppVersion, + } + + ListChartsWithFilters(w, req, params) + + var b utils.BodyAPIListResponse + json.NewDecoder(w.Body).Decode(&b) + if b.Data == nil { + t.Fatal("chart list shouldn't be null") + } + data := *b.Data + + assert.Equal(t, 2, len(data), "it should return both charts") + }) +} diff --git a/cmd/chartsvc/main.go b/cmd/chartsvc/main.go index 13669c4f3..ae8376fec 100644 --- a/cmd/chartsvc/main.go +++ b/cmd/chartsvc/main.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "context" "flag" "net/http" "os" @@ -24,15 +25,23 @@ import ( "github.com/gorilla/mux" "github.com/heptiolabs/healthcheck" "github.com/kubeapps/common/datastore" + mongoDatastore "github.com/kubeapps/common/datastore" log "github.com/sirupsen/logrus" "github.com/urfave/negroni" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + fdb "github.com/helm/monocular/cmd/chartsvc/foundationdb" + fdbDatastore "github.com/helm/monocular/cmd/chartsvc/foundationdb/datastore" + "github.com/helm/monocular/cmd/chartsvc/mongodb" ) const pathPrefix = "/v1" -var dbSession datastore.Session +var client *mongo.Client +var dbSession mongoDatastore.Session -func setupRoutes() http.Handler { +func setupRoutes(dbType *string) http.Handler { r := mux.NewRouter() // Healthcheck @@ -40,26 +49,52 @@ func setupRoutes() http.Handler { r.Handle("/live", health) r.Handle("/ready", health) - // Routes - apiv1 := r.PathPrefix(pathPrefix).Subrouter() - apiv1.Methods("GET").Path("/charts").Queries("name", "{chartName}", "version", "{version}", "appversion", "{appversion}").Handler(WithParams(listChartsWithFilters)) - apiv1.Methods("GET").Path("/charts").Queries("name", "{chartName}", "version", "{version}", "appversion", "{appversion}", "showDuplicates", "{showDuplicates}").Handler(WithParams(listChartsWithFilters)) - apiv1.Methods("GET").Path("/charts").HandlerFunc(listCharts) - apiv1.Methods("GET").Path("/charts").Queries("showDuplicates", "{showDuplicates}").HandlerFunc(listCharts) - apiv1.Methods("GET").Path("/charts/search").Queries("q", "{query}").Handler(WithParams(searchCharts)) - apiv1.Methods("GET").Path("/charts/search").Queries("q", "{query}", "showDuplicates", "{showDuplicates}").Handler(WithParams(searchCharts)) - apiv1.Methods("GET").Path("/charts/{repo}").Handler(WithParams(listRepoCharts)) - apiv1.Methods("GET").Path("/charts/{repo}/search").Queries("q", "{query}").Handler(WithParams(searchCharts)) - apiv1.Methods("GET").Path("/charts/{repo}/search").Queries("q", "{query}", "showDuplicates", "{showDuplicates}").Handler(WithParams(searchCharts)) - apiv1.Methods("GET").Path("/charts/{repo}/{chartName}").Handler(WithParams(getChart)) - apiv1.Methods("GET").Path("/charts/{repo}/{chartName}/versions").Handler(WithParams(listChartVersions)) - apiv1.Methods("GET").Path("/charts/{repo}/{chartName}/versions/{version}").Handler(WithParams(getChartVersion)) - apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/logo").Handler(WithParams(getChartIcon)) - // Maintain the logo-160x160-fit.png endpoint for backward compatibility /assets/{repo}/{chartName}/logo should be used instead - apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/logo-160x160-fit.png").Handler(WithParams(getChartIcon)) - apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/versions/{version}/README.md").Handler(WithParams(getChartVersionReadme)) - apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/versions/{version}/values.yaml").Handler(WithParams(getChartVersionValues)) - apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/versions/{version}/values.schema.json").Handler(WithParams(getChartVersionSchema)) + switch *dbType { + case "mongodb": + // Routes + apiv1 := r.PathPrefix(pathPrefix).Subrouter() + apiv1.Methods("GET").Path("/charts").Queries("name", "{chartName}", "version", "{version}", "appversion", "{appversion}").Handler(mongodb.WithParams(mongodb.ListChartsWithFilters)) + apiv1.Methods("GET").Path("/charts").Queries("name", "{chartName}", "version", "{version}", "appversion", "{appversion}", "showDuplicates", "{showDuplicates}").Handler(mongodb.WithParams(mongodb.ListChartsWithFilters)) + apiv1.Methods("GET").Path("/charts").HandlerFunc(mongodb.ListCharts) + apiv1.Methods("GET").Path("/charts").Queries("showDuplicates", "{showDuplicates}").HandlerFunc(mongodb.ListCharts) + apiv1.Methods("GET").Path("/charts/search").Queries("q", "{query}").Handler(mongodb.WithParams(mongodb.SearchCharts)) + apiv1.Methods("GET").Path("/charts/search").Queries("q", "{query}", "showDuplicates", "{showDuplicates}").Handler(mongodb.WithParams(mongodb.SearchCharts)) + apiv1.Methods("GET").Path("/charts/{repo}").Handler(mongodb.WithParams(mongodb.ListRepoCharts)) + apiv1.Methods("GET").Path("/charts/{repo}/search").Queries("q", "{query}").Handler(mongodb.WithParams(mongodb.SearchCharts)) + apiv1.Methods("GET").Path("/charts/{repo}/search").Queries("q", "{query}", "showDuplicates", "{showDuplicates}").Handler(mongodb.WithParams(mongodb.SearchCharts)) + apiv1.Methods("GET").Path("/charts/{repo}/{chartName}").Handler(mongodb.WithParams(mongodb.GetChart)) + apiv1.Methods("GET").Path("/charts/{repo}/{chartName}/versions").Handler(mongodb.WithParams(mongodb.ListChartVersions)) + apiv1.Methods("GET").Path("/charts/{repo}/{chartName}/versions/{version}").Handler(mongodb.WithParams(mongodb.GetChartVersion)) + apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/logo").Handler(mongodb.WithParams(mongodb.GetChartIcon)) + // Maintain the logo-160x160-fit.png endpoint for backward compatibility /assets/{repo}/{chartName}/logo should be used instead + apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/logo-160x160-fit.png").Handler(mongodb.WithParams(mongodb.GetChartIcon)) + apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/versions/{version}/README.md").Handler(mongodb.WithParams(mongodb.GetChartVersionReadme)) + apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/versions/{version}/values.yaml").Handler(mongodb.WithParams(mongodb.GetChartVersionValues)) + apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/versions/{version}/values.schema.json").Handler(mongodb.WithParams(mongodb.GetChartVersionSchema)) + case "fdb": + // Routes + apiv1 := r.PathPrefix(pathPrefix).Subrouter() + apiv1.Methods("GET").Path("/charts").Queries("name", "{chartName}", "version", "{version}", "appversion", "{appversion}").Handler(fdb.WithParams(fdb.ListChartsWithFilters)) + apiv1.Methods("GET").Path("/charts").Queries("name", "{chartName}", "version", "{version}", "appversion", "{appversion}", "showDuplicates", "{showDuplicates}").Handler(fdb.WithParams(fdb.ListChartsWithFilters)) + apiv1.Methods("GET").Path("/charts").HandlerFunc(fdb.ListCharts) + apiv1.Methods("GET").Path("/charts").Queries("showDuplicates", "{showDuplicates}").HandlerFunc(fdb.ListCharts) + apiv1.Methods("GET").Path("/charts/search").Queries("q", "{query}").Handler(fdb.WithParams(fdb.SearchCharts)) + apiv1.Methods("GET").Path("/charts/search").Queries("q", "{query}", "showDuplicates", "{showDuplicates}").Handler(fdb.WithParams(fdb.SearchCharts)) + apiv1.Methods("GET").Path("/charts/{repo}").Handler(fdb.WithParams(fdb.ListRepoCharts)) + apiv1.Methods("GET").Path("/charts/{repo}/search").Queries("q", "{query}").Handler(fdb.WithParams(fdb.SearchCharts)) + apiv1.Methods("GET").Path("/charts/{repo}/search").Queries("q", "{query}", "showDuplicates", "{showDuplicates}").Handler(fdb.WithParams(fdb.SearchCharts)) + apiv1.Methods("GET").Path("/charts/{repo}/{chartName}").Handler(fdb.WithParams(fdb.GetChart)) + apiv1.Methods("GET").Path("/charts/{repo}/{chartName}/versions").Handler(fdb.WithParams(fdb.ListChartVersions)) + apiv1.Methods("GET").Path("/charts/{repo}/{chartName}/versions/{version}").Handler(fdb.WithParams(fdb.GetChartVersion)) + apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/logo").Handler(fdb.WithParams(fdb.GetChartIcon)) + // Maintain the logo-160x160-fit.png endpoint for backward compatibility /assets/{repo}/{chartName}/logo should be used instead + apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/logo-160x160-fit.png").Handler(fdb.WithParams(fdb.GetChartIcon)) + apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/versions/{version}/README.md").Handler(fdb.WithParams(fdb.GetChartVersionReadme)) + apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/versions/{version}/values.yaml").Handler(fdb.WithParams(fdb.GetChartVersionValues)) + apiv1.Methods("GET").Path("/assets/{repo}/{chartName}/versions/{version}/values.schema.json").Handler(fdb.WithParams(fdb.GetChartVersionSchema)) + default: + log.Fatalf("Unknown database type: %v. db-type, if set, must be either 'mongodb' or 'fdb'.", dbType) + } n := negroni.Classic() n.UseHandler(r) @@ -67,20 +102,39 @@ func setupRoutes() http.Handler { } func main() { + + //Flag to configure running sync with either MongoDB or FoundationDB + dbType := flag.String("db-type", "mongodb", "Database backend. Either \"fdb\" (FoundationDB Document Layer) or \"mongodb\". Defaults to MongoDB if not specified.") + debug := flag.Bool("debug", false, "Debug Logging") + + //Flags for optional FoundationDB + Document Layer backend + fdbURL := flag.String("doclayer-url", "mongodb://fdb-service/27016", "FoundationDB Document Layer URL") + fDB := flag.String("doclayer-database", "charts", "FoundationDB Document-Layer database") + + //Flags for default mongoDB backend dbURL := flag.String("mongo-url", "localhost", "MongoDB URL (see https://godoc.org/github.com/globalsign/mgo#Dial for format)") dbName := flag.String("mongo-database", "charts", "MongoDB database") dbUsername := flag.String("mongo-user", "", "MongoDB user") dbPassword := os.Getenv("MONGO_PASSWORD") + flag.Parse() - mongoConfig := datastore.Config{URL: *dbURL, Database: *dbName, Username: *dbUsername, Password: dbPassword} - var err error - dbSession, err = datastore.NewSession(mongoConfig) - if err != nil { - log.WithFields(log.Fields{"host": *dbURL}).Fatal(err) + if *debug { + log.SetLevel(log.DebugLevel) } - n := setupRoutes() + log.Debugf("DB type: %v", *dbType) + + switch *dbType { + case "mongodb": + initMongoDBConnection(dbURL, dbName, dbUsername, dbPassword, debug) + case "fdb": + initFDBDocLayerConnection(fdbURL, fDB, debug) + default: + initMongoDBConnection(dbURL, dbName, dbUsername, dbPassword, debug) + } + + n := setupRoutes(dbType) port := os.Getenv("PORT") if port == "" { @@ -90,3 +144,32 @@ func main() { log.WithFields(log.Fields{"addr": addr}).Info("Started chartsvc") http.ListenAndServe(addr, n) } + +func initFDBDocLayerConnection(fdbURL *string, fDB *string, debug *bool) { + + log.Debugf("Attempting to connect to FDB: %v, %v, debug: %v", *fdbURL, *fDB, *debug) + + clientOptions := options.Client().ApplyURI(*fdbURL) + client, err := fdbDatastore.NewDocLayerClient(context.Background(), clientOptions) + if err != nil { + log.Fatalf("Can't create client for FoundationDB document layer: %v", err) + return + } + log.Debugf("FDB Document Layer client created.") + + fdb.InitDBConfig(client, *fDB) + fdb.SetPathPrefix(pathPrefix) +} + +func initMongoDBConnection(dbURL *string, dbName *string, dbUsername *string, dbPassword string, debug *bool) { + + mongoConfig := mongoDatastore.Config{URL: *dbURL, Database: *dbName, Username: *dbUsername, Password: dbPassword} + var err error + dbSession, err := datastore.NewSession(mongoConfig) + if err != nil { + log.WithFields(log.Fields{"host": *dbURL}).Fatal(err) + } + + mongodb.InitDBConfig(dbSession, *dbName) + mongodb.SetPathPrefix(pathPrefix) +} diff --git a/cmd/chartsvc/main_test.go b/cmd/chartsvc/main_test.go index 4763a9b67..4092e2e61 100644 --- a/cmd/chartsvc/main_test.go +++ b/cmd/chartsvc/main_test.go @@ -24,17 +24,24 @@ import ( "testing" "github.com/helm/monocular/cmd/chartsvc/models" + "github.com/helm/monocular/cmd/chartsvc/mongodb" + "github.com/helm/monocular/cmd/chartsvc/utils" + "github.com/kubeapps/common/datastore/mockstore" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) +var dbType string = "mongodb" + +const testChartSchema = `{"properties": {"type": "object"}}` + // tests the GET /live endpoint func Test_GetLive(t *testing.T) { var m mock.Mock dbSession = mockstore.NewMockSession(&m) - ts := httptest.NewServer(setupRoutes()) + ts := httptest.NewServer(setupRoutes(&dbType)) defer ts.Close() res, err := http.Get(ts.URL + "/live") @@ -48,7 +55,7 @@ func Test_GetReady(t *testing.T) { var m mock.Mock dbSession = mockstore.NewMockSession(&m) - ts := httptest.NewServer(setupRoutes()) + ts := httptest.NewServer(setupRoutes(&dbType)) defer ts.Close() res, err := http.Get(ts.URL + "/ready") @@ -59,7 +66,7 @@ func Test_GetReady(t *testing.T) { // tests the GET /{apiVersion}/charts endpoint func Test_GetCharts(t *testing.T) { - ts := httptest.NewServer(setupRoutes()) + ts := httptest.NewServer(setupRoutes(&dbType)) defer ts.Close() tests := []struct { @@ -80,7 +87,9 @@ func Test_GetCharts(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var m mock.Mock dbSession = mockstore.NewMockSession(&m) - m.On("All", &chartsList).Run(func(args mock.Arguments) { + mongodb.InitDBConfig(dbSession, "test") + + m.On("All", &utils.ChartsList).Run(func(args mock.Arguments) { *args.Get(0).(*[]*models.Chart) = tt.charts }) @@ -91,7 +100,7 @@ func Test_GetCharts(t *testing.T) { m.AssertExpectations(t) assert.Equal(t, res.StatusCode, http.StatusOK, "http status code should match") - var b bodyAPIListResponse + var b utils.BodyAPIListResponse json.NewDecoder(res.Body).Decode(&b) assert.Len(t, *b.Data, len(tt.charts)) }) @@ -100,7 +109,7 @@ func Test_GetCharts(t *testing.T) { // tests the GET /{apiVersion}/charts/{repo} endpoint func Test_GetChartsInRepo(t *testing.T) { - ts := httptest.NewServer(setupRoutes()) + ts := httptest.NewServer(setupRoutes(&dbType)) defer ts.Close() tests := []struct { @@ -122,7 +131,8 @@ func Test_GetChartsInRepo(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var m mock.Mock dbSession = mockstore.NewMockSession(&m) - m.On("All", &chartsList).Run(func(args mock.Arguments) { + mongodb.InitDBConfig(dbSession, "test") + m.On("All", &utils.ChartsList).Run(func(args mock.Arguments) { *args.Get(0).(*[]*models.Chart) = tt.charts }) @@ -133,7 +143,7 @@ func Test_GetChartsInRepo(t *testing.T) { m.AssertExpectations(t) assert.Equal(t, res.StatusCode, http.StatusOK, "http status code should match") - var b bodyAPIListResponse + var b utils.BodyAPIListResponse json.NewDecoder(res.Body).Decode(&b) assert.Len(t, *b.Data, len(tt.charts)) }) @@ -142,7 +152,7 @@ func Test_GetChartsInRepo(t *testing.T) { // tests the GET /{apiVersion}/charts/{repo}/{chartName} endpoint func Test_GetChartInRepo(t *testing.T) { - ts := httptest.NewServer(setupRoutes()) + ts := httptest.NewServer(setupRoutes(&dbType)) defer ts.Close() tests := []struct { @@ -175,6 +185,7 @@ func Test_GetChartInRepo(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var m mock.Mock dbSession = mockstore.NewMockSession(&m) + mongodb.InitDBConfig(dbSession, "test") if tt.err != nil { m.On("One", mock.Anything).Return(tt.err) } else { @@ -195,7 +206,7 @@ func Test_GetChartInRepo(t *testing.T) { // tests the GET /{apiVersion}/charts/{repo}/{chartName}/versions endpoint func Test_ListChartVersions(t *testing.T) { - ts := httptest.NewServer(setupRoutes()) + ts := httptest.NewServer(setupRoutes(&dbType)) defer ts.Close() tests := []struct { @@ -228,6 +239,7 @@ func Test_ListChartVersions(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var m mock.Mock dbSession = mockstore.NewMockSession(&m) + mongodb.InitDBConfig(dbSession, "test") if tt.err != nil { m.On("One", mock.Anything).Return(tt.err) } else { @@ -248,7 +260,7 @@ func Test_ListChartVersions(t *testing.T) { // tests the GET /{apiVersion}/charts/{repo}/{chartName}/versions/{:version} endpoint func Test_GetChartVersion(t *testing.T) { - ts := httptest.NewServer(setupRoutes()) + ts := httptest.NewServer(setupRoutes(&dbType)) defer ts.Close() tests := []struct { @@ -281,6 +293,7 @@ func Test_GetChartVersion(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var m mock.Mock dbSession = mockstore.NewMockSession(&m) + mongodb.InitDBConfig(dbSession, "test") if tt.err != nil { m.On("One", mock.Anything).Return(tt.err) } else { @@ -301,7 +314,7 @@ func Test_GetChartVersion(t *testing.T) { // tests the GET /{apiVersion}/assets/{repo}/{chartName}/logo-160x160-fit.png endpoint func Test_GetChartIcon(t *testing.T) { - ts := httptest.NewServer(setupRoutes()) + ts := httptest.NewServer(setupRoutes(&dbType)) defer ts.Close() tests := []struct { @@ -319,7 +332,7 @@ func Test_GetChartIcon(t *testing.T) { { "chart has icon", nil, - models.Chart{ID: "my-repo/my-chart", RawIcon: iconBytes()}, + models.Chart{ID: "my-repo/my-chart", RawIcon: utils.IconBytes()}, http.StatusOK, }, { @@ -334,6 +347,7 @@ func Test_GetChartIcon(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var m mock.Mock dbSession = mockstore.NewMockSession(&m) + mongodb.InitDBConfig(dbSession, "test") if tt.err != nil { m.On("One", mock.Anything).Return(tt.err) } else { @@ -354,7 +368,7 @@ func Test_GetChartIcon(t *testing.T) { // tests the GET /{apiVersion}/assets/{repo}/{chartName}/versions/{version}/README.md endpoint func Test_GetChartReadme(t *testing.T) { - ts := httptest.NewServer(setupRoutes()) + ts := httptest.NewServer(setupRoutes(&dbType)) defer ts.Close() tests := []struct { @@ -375,7 +389,7 @@ func Test_GetChartReadme(t *testing.T) { "chart exists", "1.2.3", nil, - models.ChartFiles{ID: "my-repo/my-chart", Readme: testChartReadme}, + models.ChartFiles{ID: "my-repo/my-chart", Readme: utils.TestChartReadme}, http.StatusOK, }, { @@ -391,6 +405,7 @@ func Test_GetChartReadme(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var m mock.Mock dbSession = mockstore.NewMockSession(&m) + mongodb.InitDBConfig(dbSession, "test") if tt.err != nil { m.On("One", mock.Anything).Return(tt.err) } else { @@ -411,7 +426,7 @@ func Test_GetChartReadme(t *testing.T) { // tests the GET /{apiVersion}/assets/{repo}/{chartName}/versions/{version}/values.yaml endpoint func Test_GetChartValues(t *testing.T) { - ts := httptest.NewServer(setupRoutes()) + ts := httptest.NewServer(setupRoutes(&dbType)) defer ts.Close() tests := []struct { @@ -432,7 +447,7 @@ func Test_GetChartValues(t *testing.T) { "chart exists", "3.2.1", nil, - models.ChartFiles{ID: "my-repo/my-chart", Values: testChartValues}, + models.ChartFiles{ID: "my-repo/my-chart", Values: utils.TestChartValues}, http.StatusOK, }, { @@ -448,6 +463,7 @@ func Test_GetChartValues(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var m mock.Mock dbSession = mockstore.NewMockSession(&m) + mongodb.InitDBConfig(dbSession, "test") if tt.err != nil { m.On("One", mock.Anything).Return(tt.err) } else { @@ -468,7 +484,7 @@ func Test_GetChartValues(t *testing.T) { // tests the GET /{apiVersion}/assets/{repo}/{chartName}/versions/{version}/values/schema.json endpoint func Test_GetChartSchema(t *testing.T) { - ts := httptest.NewServer(setupRoutes()) + ts := httptest.NewServer(setupRoutes(&dbType)) defer ts.Close() tests := []struct { @@ -505,6 +521,7 @@ func Test_GetChartSchema(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var m mock.Mock dbSession = mockstore.NewMockSession(&m) + mongodb.InitDBConfig(dbSession, "test") if tt.err != nil { m.On("One", mock.Anything).Return(tt.err) } else { diff --git a/cmd/chartsvc/handler.go b/cmd/chartsvc/mongodb/handler.go similarity index 79% rename from cmd/chartsvc/handler.go rename to cmd/chartsvc/mongodb/handler.go index d4eeb44f9..7601b33ee 100644 --- a/cmd/chartsvc/handler.go +++ b/cmd/chartsvc/mongodb/handler.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package mongodb import ( "fmt" @@ -22,9 +22,12 @@ import ( "net/http" "strconv" + "github.com/helm/monocular/cmd/chartsvc/models" + "github.com/helm/monocular/cmd/chartsvc/utils" + "github.com/globalsign/mgo/bson" "github.com/gorilla/mux" - "github.com/helm/monocular/cmd/chartsvc/models" + "github.com/kubeapps/common/datastore" "github.com/kubeapps/common/response" log "github.com/sirupsen/logrus" ) @@ -43,33 +46,22 @@ func (h WithParams) ServeHTTP(w http.ResponseWriter, req *http.Request) { const chartCollection = "charts" const filesCollection = "files" -type apiResponse struct { - ID string `json:"id"` - Type string `json:"type"` - Attributes interface{} `json:"attributes"` - Links interface{} `json:"links"` - Relationships relMap `json:"relationships"` -} - -type apiListResponse []*apiResponse - -type selfLink struct { - Self string `json:"self"` +// count is used to parse the result of a $count operation in the database +type count struct { + Count int } -type relMap map[string]rel -type rel struct { - Data interface{} `json:"data"` - Links selfLink `json:"links"` -} +var pathPrefix string +var dbSession datastore.Session -type meta struct { - TotalPages int `json:"totalPages"` +//SetPathPrefix sets the URL prefix for the ChartSVC API endpoint +func SetPathPrefix(prefix string) { + pathPrefix = prefix } -// count is used to parse the result of a $count operation in the database -type count struct { - Count int +//InitDBConfig sets MongoDB client and DB config for the ChartSVC API handler +func InitDBConfig(session datastore.Session, name string) { + dbSession = session } // getPageNumberAndSize extracts the page number and size of a request. Default (1, 0) if not set @@ -115,7 +107,8 @@ func uniqChartList(charts []*models.Chart) []*models.Chart { return res } -func getPaginatedChartList(repo string, pageNumber, pageSize int, showDuplicates bool) (apiListResponse, interface{}, error) { +func getPaginatedChartList(repo string, pageNumber, pageSize int, showDuplicates bool) (utils.ApiListResponse, interface{}, error) { + log.Debug("Request for charts..") db, closer := dbSession.DB() defer closer() var charts []*models.Chart @@ -149,7 +142,7 @@ func getPaginatedChartList(repo string, pageNumber, pageSize int, showDuplicates cc := count{} err := c.Pipe(countPipeline).One(&cc) if err != nil { - return apiListResponse{}, 0, err + return utils.ApiListResponse{}, 0, err } totalPages = int(math.Ceil(float64(cc.Count) / float64(pageSize))) @@ -165,14 +158,15 @@ func getPaginatedChartList(repo string, pageNumber, pageSize int, showDuplicates } err := c.Pipe(pipeline).All(&charts) if err != nil { - return apiListResponse{}, 0, err + return utils.ApiListResponse{}, 0, err } - - return newChartListResponse(charts), meta{totalPages}, nil + log.Debugf("Done. Returning %v charts.", len(charts)) + return newChartListResponse(charts), utils.Meta{TotalPages: totalPages}, nil } -// listCharts returns a list of charts -func listCharts(w http.ResponseWriter, req *http.Request) { +//ListCharts returns a list of charts +func ListCharts(w http.ResponseWriter, req *http.Request) { + log.Debug("Request for charts..") pageNumber, pageSize := getPageNumberAndSize(req) cl, meta, err := getPaginatedChartList("", pageNumber, pageSize, showDuplicates(req)) if err != nil { @@ -181,10 +175,12 @@ func listCharts(w http.ResponseWriter, req *http.Request) { return } response.NewDataResponseWithMeta(cl, meta).Write(w) + log.Debug("Done.") } -// listRepoCharts returns a list of charts in the given repo -func listRepoCharts(w http.ResponseWriter, req *http.Request, params Params) { +// ListRepoCharts returns a list of charts in the given repo +func ListRepoCharts(w http.ResponseWriter, req *http.Request, params Params) { + log.Debug("Request for charts..") pageNumber, pageSize := getPageNumberAndSize(req) cl, meta, err := getPaginatedChartList(params["repo"], pageNumber, pageSize, showDuplicates(req)) if err != nil { @@ -193,10 +189,11 @@ func listRepoCharts(w http.ResponseWriter, req *http.Request, params Params) { return } response.NewDataResponseWithMeta(cl, meta).Write(w) + log.Debug("Done.") } -// getChart returns the chart from the given repo -func getChart(w http.ResponseWriter, req *http.Request, params Params) { +// GetChart returns the chart from the given repo +func GetChart(w http.ResponseWriter, req *http.Request, params Params) { db, closer := dbSession.DB() defer closer() var chart models.Chart @@ -211,8 +208,8 @@ func getChart(w http.ResponseWriter, req *http.Request, params Params) { response.NewDataResponse(cr).Write(w) } -// listChartVersions returns a list of chart versions for the given chart -func listChartVersions(w http.ResponseWriter, req *http.Request, params Params) { +// ListChartVersions returns a list of chart versions for the given chart +func ListChartVersions(w http.ResponseWriter, req *http.Request, params Params) { db, closer := dbSession.DB() defer closer() var chart models.Chart @@ -227,8 +224,8 @@ func listChartVersions(w http.ResponseWriter, req *http.Request, params Params) response.NewDataResponse(cvl).Write(w) } -// getChartVersion returns the given chart version -func getChartVersion(w http.ResponseWriter, req *http.Request, params Params) { +// GetChartVersion returns the given chart version +func GetChartVersion(w http.ResponseWriter, req *http.Request, params Params) { db, closer := dbSession.DB() defer closer() var chart models.Chart @@ -249,8 +246,8 @@ func getChartVersion(w http.ResponseWriter, req *http.Request, params Params) { response.NewDataResponse(cvr).Write(w) } -// getChartIcon returns the icon for a given chart -func getChartIcon(w http.ResponseWriter, req *http.Request, params Params) { +// GetChartIcon returns the icon for a given chart +func GetChartIcon(w http.ResponseWriter, req *http.Request, params Params) { db, closer := dbSession.DB() defer closer() var chart models.Chart @@ -275,8 +272,8 @@ func getChartIcon(w http.ResponseWriter, req *http.Request, params Params) { w.Write(chart.RawIcon) } -// getChartVersionReadme returns the README for a given chart -func getChartVersionReadme(w http.ResponseWriter, req *http.Request, params Params) { +// GetChartVersionReadme returns the README for a given chart +func GetChartVersionReadme(w http.ResponseWriter, req *http.Request, params Params) { db, closer := dbSession.DB() defer closer() var files models.ChartFiles @@ -295,8 +292,8 @@ func getChartVersionReadme(w http.ResponseWriter, req *http.Request, params Para w.Write(readme) } -// getChartVersionValues returns the values.yaml for a given chart -func getChartVersionValues(w http.ResponseWriter, req *http.Request, params Params) { +// GetChartVersionValues returns the values.yaml for a given chart +func GetChartVersionValues(w http.ResponseWriter, req *http.Request, params Params) { db, closer := dbSession.DB() defer closer() var files models.ChartFiles @@ -310,8 +307,8 @@ func getChartVersionValues(w http.ResponseWriter, req *http.Request, params Para w.Write([]byte(files.Values)) } -// getChartVersionSchema returns the values.schema.json for a given chart -func getChartVersionSchema(w http.ResponseWriter, req *http.Request, params Params) { +// GetChartVersionSchema returns the values.schema.json for a given chart +func GetChartVersionSchema(w http.ResponseWriter, req *http.Request, params Params) { db, closer := dbSession.DB() defer closer() var files models.ChartFiles @@ -325,8 +322,8 @@ func getChartVersionSchema(w http.ResponseWriter, req *http.Request, params Para w.Write([]byte(files.Schema)) } -// listChartsWithFilters returns the list of repos that contains the given chart and the latest version found -func listChartsWithFilters(w http.ResponseWriter, req *http.Request, params Params) { +// ListChartsWithFilters returns the list of repos that contains the given chart and the latest version found +func ListChartsWithFilters(w http.ResponseWriter, req *http.Request, params Params) { db, closer := dbSession.DB() defer closer() @@ -354,14 +351,14 @@ func listChartsWithFilters(w http.ResponseWriter, req *http.Request, params Para response.NewDataResponse(cl).Write(w) } -// searchCharts returns the list of charts that matches the query param in any of these fields: +// SearchCharts returns the list of charts that matches the query param in any of these fields: // - name // - description // - repository name // - any keyword // - any source // - any maintainer name -func searchCharts(w http.ResponseWriter, req *http.Request, params Params) { +func SearchCharts(w http.ResponseWriter, req *http.Request, params Params) { db, closer := dbSession.DB() defer closer() @@ -396,24 +393,24 @@ func searchCharts(w http.ResponseWriter, req *http.Request, params Params) { response.NewDataResponse(cl).Write(w) } -func newChartResponse(c *models.Chart) *apiResponse { +func newChartResponse(c *models.Chart) *utils.ApiResponse { latestCV := c.ChartVersions[0] - return &apiResponse{ + return &utils.ApiResponse{ Type: "chart", ID: c.ID, Attributes: chartAttributes(*c), - Links: selfLink{pathPrefix + "/charts/" + c.ID}, - Relationships: relMap{ - "latestChartVersion": rel{ + Links: utils.SelfLink{Self: pathPrefix + "/charts/" + c.ID}, + Relationships: utils.RelMap{ + "latestChartVersion": utils.Rel{ Data: chartVersionAttributes(c.ID, latestCV), - Links: selfLink{pathPrefix + "/charts/" + c.ID + "/versions/" + latestCV.Version}, + Links: utils.SelfLink{Self: pathPrefix + "/charts/" + c.ID + "/versions/" + latestCV.Version}, }, }, } } -func newChartListResponse(charts []*models.Chart) apiListResponse { - cl := apiListResponse{} +func newChartListResponse(charts []*models.Chart) utils.ApiListResponse { + cl := utils.ApiListResponse{} for _, c := range charts { cl = append(cl, newChartResponse(c)) } @@ -436,23 +433,23 @@ func chartAttributes(c models.Chart) models.Chart { return c } -func newChartVersionResponse(c *models.Chart, cv models.ChartVersion) *apiResponse { - return &apiResponse{ +func newChartVersionResponse(c *models.Chart, cv models.ChartVersion) *utils.ApiResponse { + return &utils.ApiResponse{ Type: "chartVersion", ID: fmt.Sprintf("%s-%s", c.ID, cv.Version), Attributes: chartVersionAttributes(c.ID, cv), - Links: selfLink{pathPrefix + "/charts/" + c.ID + "/versions/" + cv.Version}, - Relationships: relMap{ - "chart": rel{ + Links: utils.SelfLink{Self: pathPrefix + "/charts/" + c.ID + "/versions/" + cv.Version}, + Relationships: utils.RelMap{ + "chart": utils.Rel{ Data: chartAttributes(*c), - Links: selfLink{pathPrefix + "/charts/" + c.ID}, + Links: utils.SelfLink{Self: pathPrefix + "/charts/" + c.ID}, }, }, } } -func newChartVersionListResponse(c *models.Chart) apiListResponse { - var cvl apiListResponse +func newChartVersionListResponse(c *models.Chart) utils.ApiListResponse { + var cvl utils.ApiListResponse for _, cv := range c.ChartVersions { cvl = append(cvl, newChartVersionResponse(c, cv)) } diff --git a/cmd/chartsvc/handler_test.go b/cmd/chartsvc/mongodb/handler_test.go similarity index 91% rename from cmd/chartsvc/handler_test.go rename to cmd/chartsvc/mongodb/handler_test.go index ffa578774..7b5777449 100644 --- a/cmd/chartsvc/handler_test.go +++ b/cmd/chartsvc/mongodb/handler_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package mongodb import ( "bytes" @@ -26,29 +26,18 @@ import ( "strings" "testing" - "github.com/disintegration/imaging" "github.com/helm/monocular/cmd/chartsvc/models" + "github.com/helm/monocular/cmd/chartsvc/utils" + + "github.com/disintegration/imaging" "github.com/kubeapps/common/datastore/mockstore" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) -type bodyAPIListResponse struct { - Data *apiListResponse `json:"data"` - Meta meta `json:"meta,omitempty"` -} - -type bodyAPIResponse struct { - Data apiResponse `json:"data"` -} - var chartsList []*models.Chart var cc count -const testChartReadme = "# Quickstart\n\n```bash\nhelm install my-repo/my-chart\n```" -const testChartValues = "image:\n registry: docker.io\n repository: my-repo/my-chart\n tag: 0.1.0" -const testChartSchema = `{"properties": {"type": "object"}}` - func iconBytes() []byte { var b bytes.Buffer img := imaging.New(1, 1, color.White) @@ -121,7 +110,7 @@ func Test_newChartResponse(t *testing.T) { assert.Equal(t, cResponse.Type, "chart", "response type is chart") assert.Equal(t, cResponse.ID, tt.chart.ID, "chart ID should be the same") assert.Equal(t, cResponse.Relationships["latestChartVersion"].Data.(models.ChartVersion).Version, tt.chart.ChartVersions[0].Version, "latestChartVersion should match version at index 0") - assert.Equal(t, cResponse.Links.(selfLink).Self, pathPrefix+"/charts/"+tt.chart.ID, "self link should be the same") + assert.Equal(t, cResponse.Links.(utils.SelfLink).Self, pathPrefix+"/charts/"+tt.chart.ID, "self link should be the same") assert.Equal(t, len(cResponse.Attributes.(models.Chart).ChartVersions), len(tt.chart.ChartVersions), "number of chart versions in the response should be the same") }) } @@ -156,7 +145,7 @@ func Test_newChartListResponse(t *testing.T) { assert.Equal(t, clResponse[i].Type, "chart", "response type is chart") assert.Equal(t, clResponse[i].ID, tt.result[i].ID, "chart ID should be the same") assert.Equal(t, clResponse[i].Relationships["latestChartVersion"].Data.(models.ChartVersion).Version, tt.result[i].ChartVersions[0].Version, "latestChartVersion should match version at index 0") - assert.Equal(t, clResponse[i].Links.(selfLink).Self, pathPrefix+"/charts/"+tt.result[i].ID, "self link should be the same") + assert.Equal(t, clResponse[i].Links.(utils.SelfLink).Self, pathPrefix+"/charts/"+tt.result[i].ID, "self link should be the same") assert.Equal(t, len(clResponse[i].Attributes.(models.Chart).ChartVersions), len(tt.result[i].ChartVersions), "number of chart versions in the response should be the same") } }) @@ -179,7 +168,7 @@ func Test_newChartVersionResponse(t *testing.T) { cvResponse := newChartVersionResponse(&tt.chart, tt.chart.ChartVersions[i]) assert.Equal(t, cvResponse.Type, "chartVersion", "response type is chartVersion") assert.Equal(t, cvResponse.ID, tt.chart.ID+"-"+tt.chart.ChartVersions[i].Version, "reponse id should have chart version suffix") - assert.Equal(t, cvResponse.Links.(interface{}).(selfLink).Self, pathPrefix+"/charts/"+tt.chart.ID+"/versions/"+tt.chart.ChartVersions[i].Version, "self link should be the same") + assert.Equal(t, cvResponse.Links.(interface{}).(utils.SelfLink).Self, pathPrefix+"/charts/"+tt.chart.ID+"/versions/"+tt.chart.ChartVersions[i].Version, "self link should be the same") assert.Equal(t, cvResponse.Attributes.(models.ChartVersion).Version, tt.chart.ChartVersions[i].Version, "chart version in the response should be the same") assert.Equal(t, cvResponse.Relationships["chart"].Data.(interface{}).(models.Chart), tt.chart, "chart in relatioship matches") } @@ -210,7 +199,7 @@ func Test_newChartVersionListResponse(t *testing.T) { for i := range tt.chart.ChartVersions { assert.Equal(t, cvListResponse[i].Type, "chartVersion", "response type is chartVersion") assert.Equal(t, cvListResponse[i].ID, tt.chart.ID+"-"+tt.chart.ChartVersions[i].Version, "reponse id should have chart version suffix") - assert.Equal(t, cvListResponse[i].Links.(interface{}).(selfLink).Self, pathPrefix+"/charts/"+tt.chart.ID+"/versions/"+tt.chart.ChartVersions[i].Version, "self link should be the same") + assert.Equal(t, cvListResponse[i].Links.(interface{}).(utils.SelfLink).Self, pathPrefix+"/charts/"+tt.chart.ID+"/versions/"+tt.chart.ChartVersions[i].Version, "self link should be the same") assert.Equal(t, cvListResponse[i].Attributes.(models.ChartVersion).Version, tt.chart.ChartVersions[i].Version, "chart version in the response should be the same") } }) @@ -222,23 +211,23 @@ func Test_listCharts(t *testing.T) { name string query string charts []*models.Chart - meta meta + meta utils.Meta }{ - {"no charts", "", []*models.Chart{}, meta{1}}, + {"no charts", "", []*models.Chart{}, utils.Meta{TotalPages: 1}}, {"one chart", "", []*models.Chart{ {ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, - }, meta{1}}, + }, utils.Meta{TotalPages: 1}}, {"two charts", "", []*models.Chart{ {ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, {ID: "stable/dokuwiki", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "1234"}, {Version: "1.2.2", Digest: "12345"}}}, - }, meta{1}}, + }, utils.Meta{TotalPages: 1}}, // Pagination tests {"four charts with pagination", "?size=2", []*models.Chart{ {ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, {ID: "stable/dokuwiki", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "1234"}}}, {ID: "stable/drupal", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "12345"}}}, {ID: "stable/wordpress", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "123456"}}}, - }, meta{2}}, + }, utils.Meta{TotalPages: 2}}, } for _, tt := range tests { @@ -257,12 +246,12 @@ func Test_listCharts(t *testing.T) { w := httptest.NewRecorder() req := httptest.NewRequest("GET", "/charts"+tt.query, nil) - listCharts(w, req) + ListCharts(w, req) m.AssertExpectations(t) assert.Equal(t, http.StatusOK, w.Code) - var b bodyAPIListResponse + var b utils.BodyAPIListResponse json.NewDecoder(w.Body).Decode(&b) if b.Data == nil { t.Fatal("chart list shouldn't be null") @@ -286,22 +275,22 @@ func Test_listRepoCharts(t *testing.T) { repo string query string charts []*models.Chart - meta meta + meta utils.Meta }{ - {"repo has no charts", "my-repo", "", []*models.Chart{}, meta{1}}, + {"repo has no charts", "my-repo", "", []*models.Chart{}, utils.Meta{TotalPages: 1}}, {"repo has one chart", "my-repo", "", []*models.Chart{ {ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, - }, meta{1}}, + }, utils.Meta{TotalPages: 1}}, {"repo has many charts", "my-repo", "", []*models.Chart{ {ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, {ID: "my-repo/dokuwiki", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "1234"}, {Version: "1.2.2", Digest: "12345"}}}, - }, meta{1}}, + }, utils.Meta{TotalPages: 1}}, {"repo has many charts with pagination", "my-repo", "?size=2", []*models.Chart{ {ID: "my-repo/my-chart", ChartVersions: []models.ChartVersion{{Version: "0.0.1", Digest: "123"}}}, {ID: "stable/dokuwiki", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "1234"}}}, {ID: "stable/drupal", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "12345"}}}, {ID: "stable/wordpress", ChartVersions: []models.ChartVersion{{Version: "1.2.3", Digest: "123456"}}}, - }, meta{2}}, + }, utils.Meta{TotalPages: 2}}, } for _, tt := range tests { @@ -324,12 +313,12 @@ func Test_listRepoCharts(t *testing.T) { "repo": "my-repo", } - listRepoCharts(w, req, params) + ListRepoCharts(w, req, params) m.AssertExpectations(t) assert.Equal(t, http.StatusOK, w.Code) - var b bodyAPIListResponse + var b utils.BodyAPIListResponse json.NewDecoder(w.Body).Decode(&b) data := *b.Data assert.Len(t, data, len(tt.charts)) @@ -391,12 +380,12 @@ func Test_getChart(t *testing.T) { "chartName": parts[1], } - getChart(w, req, params) + GetChart(w, req, params) m.AssertExpectations(t) assert.Equal(t, tt.wantCode, w.Code) if tt.wantCode == http.StatusOK { - var b bodyAPIResponse + var b utils.BodyAPIResponse json.NewDecoder(w.Body).Decode(&b) assert.Equal(t, b.Data.ID, tt.chart.ID, "chart id in the response should be the same") assert.Equal(t, b.Data.Type, "chart", "response type is chart") @@ -455,12 +444,12 @@ func Test_listChartVersions(t *testing.T) { "chartName": parts[1], } - listChartVersions(w, req, params) + ListChartVersions(w, req, params) m.AssertExpectations(t) assert.Equal(t, tt.wantCode, w.Code) if tt.wantCode == http.StatusOK { - var b bodyAPIListResponse + var b utils.BodyAPIListResponse json.NewDecoder(w.Body).Decode(&b) data := *b.Data for i, resp := range data { @@ -522,12 +511,12 @@ func Test_getChartVersion(t *testing.T) { "version": tt.chart.ChartVersions[0].Version, } - getChartVersion(w, req, params) + GetChartVersion(w, req, params) m.AssertExpectations(t) assert.Equal(t, tt.wantCode, w.Code) if tt.wantCode == http.StatusOK { - var b bodyAPIResponse + var b utils.BodyAPIResponse json.NewDecoder(w.Body).Decode(&b) assert.Equal(t, b.Data.ID, tt.chart.ID+"-"+tt.chart.ChartVersions[0].Version, "chart id in the response should be the same") assert.Equal(t, b.Data.Type, "chartVersion", "response type is chartVersion") @@ -553,7 +542,7 @@ func Test_getChartIcon(t *testing.T) { { "chart has icon", nil, - models.Chart{ID: "my-repo/my-chart", RawIcon: iconBytes(), IconContentType: "image/png"}, + models.Chart{ID: "my-repo/my-chart", RawIcon: utils.IconBytes(), IconContentType: "image/png"}, http.StatusOK, }, { @@ -591,7 +580,7 @@ func Test_getChartIcon(t *testing.T) { "chartName": parts[1], } - getChartIcon(w, req, params) + GetChartIcon(w, req, params) m.AssertExpectations(t) assert.Equal(t, tt.wantCode, w.Code, "http status code should match") @@ -622,7 +611,7 @@ func Test_getChartVersionReadme(t *testing.T) { "chart exists", "1.2.3", nil, - models.ChartFiles{ID: "my-repo/my-chart", Readme: testChartReadme}, + models.ChartFiles{ID: "my-repo/my-chart", Readme: utils.TestChartReadme}, http.StatusOK, }, { @@ -656,7 +645,7 @@ func Test_getChartVersionReadme(t *testing.T) { "version": "0.1.0", } - getChartVersionReadme(w, req, params) + GetChartVersionReadme(w, req, params) m.AssertExpectations(t) assert.Equal(t, tt.wantCode, w.Code, "http status code should match") @@ -686,7 +675,7 @@ func Test_getChartVersionValues(t *testing.T) { "chart exists", "3.2.1", nil, - models.ChartFiles{ID: "my-repo/my-chart", Values: testChartValues}, + models.ChartFiles{ID: "my-repo/my-chart", Values: utils.TestChartValues}, http.StatusOK, }, { @@ -720,7 +709,7 @@ func Test_getChartVersionValues(t *testing.T) { "version": "0.1.0", } - getChartVersionValues(w, req, params) + GetChartVersionValues(w, req, params) m.AssertExpectations(t) assert.Equal(t, tt.wantCode, w.Code, "http status code should match") @@ -750,7 +739,7 @@ func Test_getChartVersionSchema(t *testing.T) { "chart exists", "3.2.1", nil, - models.ChartFiles{ID: "my-repo/my-chart", Schema: testChartSchema}, + models.ChartFiles{ID: "my-repo/my-chart", Schema: utils.TestChartSchema}, http.StatusOK, }, { @@ -784,7 +773,7 @@ func Test_getChartVersionSchema(t *testing.T) { "version": "0.1.0", } - getChartVersionSchema(w, req, params) + GetChartVersionSchema(w, req, params) m.AssertExpectations(t) assert.Equal(t, tt.wantCode, w.Code, "http status code should match") @@ -812,7 +801,7 @@ func Test_findLatestChart(t *testing.T) { var m mock.Mock dbSession = mockstore.NewMockSession(&m) - m.On("All", &chartsList).Run(func(args mock.Arguments) { + m.On("All", &utils.ChartsList).Run(func(args mock.Arguments) { *args.Get(0).(*[]*models.Chart) = charts }) @@ -824,9 +813,9 @@ func Test_findLatestChart(t *testing.T) { "appversion": reqAppVersion, } - listChartsWithFilters(w, req, params) + ListChartsWithFilters(w, req, params) - var b bodyAPIListResponse + var b utils.BodyAPIListResponse json.NewDecoder(w.Body).Decode(&b) if b.Data == nil { t.Fatal("chart list shouldn't be null") @@ -847,7 +836,7 @@ func Test_findLatestChart(t *testing.T) { var m mock.Mock dbSession = mockstore.NewMockSession(&m) - m.On("All", &chartsList).Run(func(args mock.Arguments) { + m.On("All", &utils.ChartsList).Run(func(args mock.Arguments) { *args.Get(0).(*[]*models.Chart) = charts }) @@ -859,9 +848,9 @@ func Test_findLatestChart(t *testing.T) { "appversion": reqAppVersion, } - listChartsWithFilters(w, req, params) + ListChartsWithFilters(w, req, params) - var b bodyAPIListResponse + var b utils.BodyAPIListResponse json.NewDecoder(w.Body).Decode(&b) if b.Data == nil { t.Fatal("chart list shouldn't be null") @@ -895,9 +884,9 @@ func Test_findLatestChart(t *testing.T) { "appversion": reqAppVersion, } - listChartsWithFilters(w, req, params) + ListChartsWithFilters(w, req, params) - var b bodyAPIListResponse + var b utils.BodyAPIListResponse json.NewDecoder(w.Body).Decode(&b) if b.Data == nil { t.Fatal("chart list shouldn't be null") diff --git a/cmd/chartsvc/utils/responses.go b/cmd/chartsvc/utils/responses.go new file mode 100644 index 000000000..00d6c105f --- /dev/null +++ b/cmd/chartsvc/utils/responses.go @@ -0,0 +1,59 @@ +/* +Copyright (c) 2019 The Helm Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +//BodyAPIListResponse is an API body response in list format including the number of results pages +type BodyAPIListResponse struct { + Data *ApiListResponse `json:"data"` + Meta Meta `json:"meta,omitempty"` +} + +//BodyAPIResponse is an API body response in non-list format +type BodyAPIResponse struct { + Data ApiResponse `json:"data"` +} + +//ApiResponse is an API response in non-list format +type ApiResponse struct { + ID string `json:"id"` + Type string `json:"type"` + Attributes interface{} `json:"attributes"` + Links interface{} `json:"links"` + Relationships RelMap `json:"relationships"` +} + +//ApiListResponse is an API response in list format +type ApiListResponse []*ApiResponse + +//SelfLink the self-referencing URL to a chart in a response +type SelfLink struct { + Self string `json:"self"` +} + +//RelMap maps elements e.g. Charts to other elements of a response e.g. Chart Versions +type RelMap map[string]Rel + +//Rel describes a relationship between element(s) in a response +type Rel struct { + Data interface{} `json:"data"` + Links SelfLink `json:"links"` +} + +//Meta the number of pages in the response +type Meta struct { + TotalPages int `json:"totalPages"` +} diff --git a/cmd/chartsvc/utils/testutils.go b/cmd/chartsvc/utils/testutils.go new file mode 100644 index 000000000..90339a370 --- /dev/null +++ b/cmd/chartsvc/utils/testutils.go @@ -0,0 +1,40 @@ +/* +Copyright (c) 2019 The Helm Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "bytes" + "image/color" + "github.com/helm/monocular/cmd/chartsvc/models" + + "github.com/disintegration/imaging" +) + +//ChartsList a list of charts used in unit tests +var ChartsList []*models.Chart + +//IconBytes the bytes of a chart icon image used in unit tests +func IconBytes() []byte { + var b bytes.Buffer + img := imaging.New(1, 1, color.White) + imaging.Encode(&b, img, imaging.PNG) + return b.Bytes() +} + +const TestChartReadme = "# Quickstart\n\n```bash\nhelm install my-repo/my-chart\n```" +const TestChartValues = "image:\n registry: docker.io\n repository: my-repo/my-chart\n tag: 0.1.0" +const TestChartSchema = `{"properties": {"type": "object"}}` diff --git a/fdb-service.yaml b/fdb-service.yaml new file mode 100644 index 000000000..ba4dd6cd9 --- /dev/null +++ b/fdb-service.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: fdb-service +spec: + type: ClusterIP + ports: + - protocol: TCP + port: 27016 + targetPort: 27016 +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: fdb-service +subsets: + - addresses: + - ip: 192.168.57.1 + ports: + - port: 27016 diff --git a/go.mod b/go.mod index 2e3afd82d..7e7ec859e 100644 --- a/go.mod +++ b/go.mod @@ -12,9 +12,11 @@ require ( github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 // indirect github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680 github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731 + github.com/go-stack/stack v1.8.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.2.1 // indirect github.com/golang/protobuf v1.2.0 // indirect + github.com/golang/snappy v0.0.1 // indirect github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/mux v1.6.2 github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 @@ -34,12 +36,18 @@ require ( github.com/stretchr/testify v1.2.2 github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d // indirect github.com/urfave/negroni v1.0.0 + github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect + github.com/xdg/stringprep v1.0.0 // indirect + go.mongodb.org/mongo-driver v1.1.3 golang.org/x/image v0.0.0-20180926015637-991ec62608f3 // indirect golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect golang.org/x/sys v0.0.0-20180928133829-e4b3c5e90611 // indirect + golang.org/x/text v0.3.2 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/yaml.v2 v2.2.1 // indirect k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d // indirect k8s.io/client-go v9.0.0+incompatible // indirect k8s.io/helm v2.13.1+incompatible ) + +replace github.com/helm/monocular => github.com/cf-stratos/monocular v1.5.1-0.20191202124725-625fe8595697 diff --git a/go.sum b/go.sum index d1ffca253..bc9410d91 100644 --- a/go.sum +++ b/go.sum @@ -18,12 +18,16 @@ github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680 h1:ZktWZesgun21uEDrwW7 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731 h1:y7wyeiA6T+TT+HGC9DYypvLkUeg99N4rqHMzn2MmjYk= github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= @@ -68,6 +72,12 @@ github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d h1:ggUgChAeyge4NZ4 github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +go.mongodb.org/mongo-driver v1.1.3 h1:++7u8r9adKhGR+I79NfEtYrk2ktjenErXM99PSufIoI= +go.mongodb.org/mongo-driver v1.1.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/image v0.0.0-20180926015637-991ec62608f3 h1:5IfA9fqItkh2alJW94tvQk+6+RF9MW2q9DzwE8DBddQ= @@ -77,7 +87,10 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180928133829-e4b3c5e90611 h1:O33LKL7WyJgjN9CvxfTIomjIClbd/Kq86/iipowHQU0= golang.org/x/sys v0.0.0-20180928133829-e4b3c5e90611/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=