diff --git a/.gitignore b/.gitignore index d4b500ee..d97a7802 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ registry/dpmd # Doc Build _build + +# Zip files +**.tgz \ No newline at end of file diff --git a/api/python/t4/data_transfer.py b/api/python/t4/data_transfer.py index ae127735..f183ce68 100644 --- a/api/python/t4/data_transfer.py +++ b/api/python/t4/data_transfer.py @@ -36,7 +36,15 @@ # Linux only allows users to modify user.* xattrs. HELIUM_XATTR = 'user.%s' % HELIUM_XATTR -s3_client = boto3.client('s3') +#s3_client = boto3.client('s3') + +s3_client = boto3.client('s3', + endpoint_url='http://localhost:9000', + aws_access_key_id='AKIAIOSFODNN7EXAMPLE', + aws_secret_access_key='wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', + config=Config(signature_version='s3v4'), + region_name='us-east-1') + try: # Ensure that user has AWS credentials that function. # quilt-example is readable by anonymous users, if the head fails @@ -47,6 +55,14 @@ # Use unsigned boto if credentials can't head the default bucket s3_client = boto3.client('s3', config=Config(signature_version=UNSIGNED)) +if True: # switch to env variable + s3_client = boto3.client('s3', + endpoint_url='http://localhost:9000', + aws_access_key_id='AKIAIOSFODNN7EXAMPLE', + aws_secret_access_key='wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', + config=Config(signature_version='s3v4'), + region_name='us-east-1') + s3_transfer_config = TransferConfig() s3_threads = 4 diff --git a/catalog/app/utils/AWS/S3.js b/catalog/app/utils/AWS/S3.js index 94282424..61e855e2 100644 --- a/catalog/app/utils/AWS/S3.js +++ b/catalog/app/utils/AWS/S3.js @@ -17,7 +17,13 @@ const Ctx = React.createContext(); export const Provider = composeComponent('AWS.S3.Provider', Config.inject(), withPropsOnChange(['awsConfig'], ({ awsConfig }) => ({ - client: new S3(awsConfig), + client: new S3({ + accessKeyId: 'AKIAIOSFODNN7EXAMPLE' , //Preferably read from an env variable + secretAccessKey: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' , //Preferably read from an env variable + endpoint: 'http://127.0.0.1:9000' , + s3ForcePathStyle: true, // needed with minio? + signatureVersion: 'v4' +}), })), provide(Ctx, 'client')); diff --git a/charts/Chart.yaml b/charts/Chart.yaml new file mode 100644 index 00000000..9c071608 --- /dev/null +++ b/charts/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +name: miniT4 +version: 0.0.1 +description: A single-sentence description of this project (optional) +keywords: + - Datascience +home: https://quiltdata.com/ +sources: + - https://github.com/quiltdata/t4/ +maintainers: # (optional) + - name: Michael Sindelar + email: sindelar@quiltdata.io diff --git a/charts/README.md b/charts/README.md new file mode 100644 index 00000000..b392c375 --- /dev/null +++ b/charts/README.md @@ -0,0 +1,14 @@ +Create a Secret in the cluster that holds your authorization token +A Kubernetes cluster uses the Secret of docker-registry type to authenticate with a container registry to pull a private image. + +Create this Secret, naming it regcred: + +kubectl create secret docker-registry regcred --docker-server= --docker-username= --docker-password= --docker-email= +where: + + is your Private Docker Registry FQDN. (https://index.docker.io/v1/ for DockerHub) + is your Docker username. + is your Docker password. + is your Docker email. +You have successfully set your Docker credentials in the cluster as a Secret called regcred. + diff --git a/charts/t4/.helmignore b/charts/t4/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/t4/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/t4/Chart.yaml b/charts/t4/Chart.yaml new file mode 100644 index 00000000..300bdea0 --- /dev/null +++ b/charts/t4/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: t4 +version: 0.1.0 diff --git a/charts/t4/charts/elasticsearch-6.4.2.tgz b/charts/t4/charts/elasticsearch-6.4.2.tgz new file mode 100644 index 00000000..480a8b40 Binary files /dev/null and b/charts/t4/charts/elasticsearch-6.4.2.tgz differ diff --git a/charts/t4/charts/minio-2.3.5.tgz b/charts/t4/charts/minio-2.3.5.tgz new file mode 100644 index 00000000..9eaec8f7 Binary files /dev/null and b/charts/t4/charts/minio-2.3.5.tgz differ diff --git a/charts/t4/requirements.lock b/charts/t4/requirements.lock new file mode 100644 index 00000000..995f3abc --- /dev/null +++ b/charts/t4/requirements.lock @@ -0,0 +1,9 @@ +dependencies: +- name: elasticsearch + repository: https://helm.elastic.co + version: 6.4.2 +- name: minio + repository: https://kubernetes-charts.storage.googleapis.com + version: 2.3.5 +digest: sha256:af3ac95048e13e3b42fc79b4e1fc808fb771037220fed621234dc63dc5c26be6 +generated: 2019-01-23T13:49:18.248497-08:00 diff --git a/charts/t4/requirements.yaml b/charts/t4/requirements.yaml new file mode 100644 index 00000000..a3285d11 --- /dev/null +++ b/charts/t4/requirements.yaml @@ -0,0 +1,13 @@ +dependencies: + - name: elasticsearch + repository: https://helm.elastic.co + version: '6.4.2' + - name: minio + repository: https://kubernetes-charts.storage.googleapis.com + version: '2.3.5' + +# knative installations? +# helm install riff-incubator/istio --name istio --namespace istio-system --values http://riff-incubator.cfapps.io/values/values-istio-knative.yaml --set ingressgateway.service.type=NodePort +# helm install riff-incubator/knative-istio --name knative-istio --namespace istio-system --set knative.ingressgateway.service.type=NodePort +# helm install riff-incubator/knative-build --name knative-build --namespace knative-build +# helm install riff-incubator/knative-serving --name knative-serving --namespace knative-serving \ No newline at end of file diff --git a/charts/t4/templates/NOTES.txt b/charts/t4/templates/NOTES.txt new file mode 100644 index 00000000..97bd8cdc --- /dev/null +++ b/charts/t4/templates/NOTES.txt @@ -0,0 +1,21 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range $.Values.ingress.paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host }}{{ . }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "t4.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ include "t4.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "t4.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "t4.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward $POD_NAME 8080:80 +{{- end }} diff --git a/charts/t4/templates/_helpers.tpl b/charts/t4/templates/_helpers.tpl new file mode 100644 index 00000000..d8d858f4 --- /dev/null +++ b/charts/t4/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "t4.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "t4.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "t4.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/charts/t4/templates/deployment.yaml b/charts/t4/templates/deployment.yaml new file mode 100644 index 00000000..b0907139 --- /dev/null +++ b/charts/t4/templates/deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "t4.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "t4.name" . }} + helm.sh/chart: {{ include "t4.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "t4.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "t4.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 3000 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/t4/templates/ingress.yaml b/charts/t4/templates/ingress.yaml new file mode 100644 index 00000000..243546a7 --- /dev/null +++ b/charts/t4/templates/ingress.yaml @@ -0,0 +1,40 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "t4.fullname" . -}} +{{- $ingressPaths := .Values.ingress.paths -}} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + app.kubernetes.io/name: {{ include "t4.name" . }} + helm.sh/chart: {{ include "t4.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ . | quote }} + http: + paths: + {{- range $ingressPaths }} + - path: {{ . }} + backend: + serviceName: {{ $fullName }} + servicePort: http + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/t4/templates/service.yaml b/charts/t4/templates/service.yaml new file mode 100644 index 00000000..2b2b7517 --- /dev/null +++ b/charts/t4/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "t4.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "t4.name" . }} + helm.sh/chart: {{ include "t4.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "t4.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/t4/templates/tests/test-connection.yaml b/charts/t4/templates/tests/test-connection.yaml new file mode 100644 index 00000000..11fb6931 --- /dev/null +++ b/charts/t4/templates/tests/test-connection.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "t4.fullname" . }}-test-connection" + labels: + app.kubernetes.io/name: {{ include "t4.name" . }} + helm.sh/chart: {{ include "t4.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "t4.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/t4/values.yaml b/charts/t4/values.yaml new file mode 100644 index 00000000..bbc3d5ed --- /dev/null +++ b/charts/t4/values.yaml @@ -0,0 +1,52 @@ +# Default values for t4. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: quiltdata/catalog + tag: minio-test + pullPolicy: IfNotPresent + imagePullSecrets: + - name: regcred + +nameOverride: "" +fullnameOverride: "" + +service: + type: NodePort + port: 3000 + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + paths: [] + hosts: + - chart-example.local + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# pre-install hook to setup PV for minio PVC? + +elasticsearch: + replicas: 1 + +minio: + accessKey: AKIAIOSFODNN7EXAMPLE + secretKey: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + defaultBucket: + enabled: True + name: config