From 837601c33c0a770faa9b77799c1905f2b4b0a326 Mon Sep 17 00:00:00 2001 From: georgedriver <36863151+georgedriver@users.noreply.github.com> Date: Wed, 24 Oct 2018 18:25:53 +0800 Subject: [PATCH 1/5] update --- cmd/root.go | 29 ++++++++-- cmd/util.go | 33 ++++++++++- pkg/kv/alicloud_kms/alicloud_kms.go | 86 +++++++++++++++++++++++++++ pkg/kv/alicloud_oss/alicloud_oss.go | 90 +++++++++++++++++++++++++++++ 4 files changed, 230 insertions(+), 8 deletions(-) create mode 100644 pkg/kv/alicloud_kms/alicloud_kms.go create mode 100644 pkg/kv/alicloud_oss/alicloud_oss.go diff --git a/cmd/root.go b/cmd/root.go index aa6fde1a..b1094425 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,12 +1,12 @@ package cmd import ( - "fmt" - "os" - "strings" +"fmt" +"os" +"strings" - "github.com/spf13/cobra" - "github.com/spf13/viper" +"github.com/spf13/cobra" +"github.com/spf13/viper" ) var appConfig *viper.Viper @@ -16,6 +16,7 @@ const cfgSecretShares = "secret-shares" const cfgSecretThreshold = "secret-threshold" const cfgMode = "mode" +const cfgModeValueAlicloudKMSOSS = "alicloud-kms-ssm" const cfgModeValueAWSKMSSSM = "aws-kms-ssm" const cfgModeValueGoogleCloudKMSGCS = "google-cloud-kms-gcs" const cfgModeValueLocal = "local" @@ -31,6 +32,13 @@ const cfgGoogleCloudStoragePrefix = "google-cloud-storage-prefix" const cfgAWSKMSKeyID = "aws-kms-key-id" const cfgAWSSSMKeyPrefix = "aws-ssm-key-prefix" +const cfgAlicloudKMSRegion = "alicloud-kms-region" +const cfgAlicloudKMSKeyID = "alicloud-kms-id" + +const cfgAlicloudStorageEndpoint = "alicloud-storage-endpoint" +const cfgAlicloudStorageBucket = "alicloud-storage-bucket" +const cfgAlicloudStoragePrefix = "alicloud-storage-prefix" + const cfgLocalKeyDir = "local-key-dir" // RootCmd represents the base command when called without any subcommands @@ -78,7 +86,7 @@ func init() { configStringVar( cfgMode, cfgModeValueGoogleCloudKMSGCS, - fmt.Sprintf("Select the mode to use '%s' => Google Cloud Storage with encryption using Google KMS; '%s' => AWS SSM parameter store using AWS KMS encryption; %s => Use local keys in path", cfgModeValueGoogleCloudKMSGCS, cfgModeValueAWSKMSSSM, cfgModeValueLocal), + fmt.Sprintf("Select the mode to use '%s' => Google Cloud Storage with encryption using Google KMS; '%s' => AWS SSM parameter store using AWS KMS encryption; '%s' => Alicloud KMS parameter store using Alicloud KMS encryption; %s => Use local keys in path", cfgModeValueGoogleCloudKMSGCS, cfgModeValueAWSKMSSSM, cfgModeValueAlicloudKMSOSS, cfgModeValueLocal), ) // Secret config @@ -101,5 +109,14 @@ func init() { // AWS SSM Parameter Storage flags configStringVar("aws-ssm-key-prefix", "", "The Key Prefix for SSM Parameter store") + // Alicloud KMS flags + configStringVar(cfgAlicloudKMSRegion, "", "The Region string of the Alicloud KMS to encrypt values") + configStringVar(cfgAlicloudKMSKeyID, "", "The Key ID string of the Alicloud KMS key to encrypt values") + + // Alicloud Storage flags + configStringVar(cfgAlicloudStorageEndpoint, "", "The endpoint of the Alicloud Bucket") + configStringVar(cfgAlicloudStorageBucket, "", "The Region string of the Alicloud KMS to encrypt values") + configStringVar(cfgAlicloudStoragePrefix, "", "The prefix to use for values store in Alicloud Storage") + configStringVar("local-key-dir", "", "Directory of key shares in path") } diff --git a/cmd/util.go b/cmd/util.go index 85e9c7a9..3ba89bea 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -2,8 +2,10 @@ package cmd import ( "fmt" - + "github.com/jetstack/vault-unsealer/pkg/kv/alicloud_kms" + "github.com/jetstack/vault-unsealer/pkg/kv/alicloud_oss" "github.com/spf13/viper" + "os" "github.com/jetstack/vault-unsealer/pkg/kv" "github.com/jetstack/vault-unsealer/pkg/kv/aws_kms" @@ -11,7 +13,6 @@ import ( "github.com/jetstack/vault-unsealer/pkg/kv/cloudkms" "github.com/jetstack/vault-unsealer/pkg/kv/gcs" "github.com/jetstack/vault-unsealer/pkg/kv/local" - "github.com/jetstack/vault-unsealer/pkg/vault" ) @@ -70,6 +71,34 @@ func kvStoreForConfig(cfg *viper.Viper) (kv.Service, error) { return kms, nil + case cfgModeValueAlicloudKMSOSS: + envAlicloudAccessKey := os.Getenv("ALICLOUD_ACCESS_KEY") + envAlicloudSecretKey := os.Getenv("ALICLOUD_SECRET_KEY") + + o, err := alicloud_oss.New( + cfg.GetString(cfgAlicloudStorageEndpoint), + cfg.GetString(cfgAlicloudStorageBucket), + cfg.GetString(cfgAlicloudStoragePrefix), + envAlicloudSecretKey, + envAlicloudSecretKey) + + if err != nil { + return nil, fmt.Errorf("error creating Alicloud storage kv store: %s", err.Error()) + } + + + kms, err := alicloud_kms.New(o, + cfg.GetString(cfgAlicloudKMSKeyID), + cfg.GetString(cfgAlicloudKMSRegion), + envAlicloudAccessKey, + envAlicloudSecretKey) + + if err != nil { + return nil, fmt.Errorf("error creating Alicloud KMS ID kv store: %s", err.Error()) + } + + return kms, nil + case cfgModeValueLocal: return local.New(cfg.GetString(cfgLocalKeyDir)) diff --git a/pkg/kv/alicloud_kms/alicloud_kms.go b/pkg/kv/alicloud_kms/alicloud_kms.go new file mode 100644 index 00000000..ff09b583 --- /dev/null +++ b/pkg/kv/alicloud_kms/alicloud_kms.go @@ -0,0 +1,86 @@ +package alicloud_kms + +import ( + "fmt" + "github.com/aliyun/alibaba-cloud-sdk-go/services/kms" + + "github.com/jetstack/vault-unsealer/pkg/kv" +) + +type alicloudKms struct { + kmsClient *kms.Client + store kv.Service + keyId string +} + +var _ kv.Service = &alicloudKms{} + +func New(store kv.Service, keyId, regionId, alicloudAccessKey, alicloudSecretKey string) (kv.Service, error) { + client, err := kms.NewClientWithAccessKey(regionId, alicloudAccessKey, alicloudSecretKey) + + if err != nil { + return nil, fmt.Errorf("error creating alicloud kms client: %s", err.Error()) + } + + return &alicloudKms{ + store: store, + kmsClient: client, + keyId: keyId, + }, nil +} + +func (a *alicloudKms) encrypt(s []byte) ([]byte, error) { + requestStruct := kms.CreateEncryptRequest() + requestStruct.KeyId = a.keyId + requestStruct.RpcRequest.SetScheme("HTTPS") + requestStruct.Plaintext = string(s[:]) + + fmt.Println("HGX: Planintext is ", string(s[:])) + genresp, err := a.kmsClient.Encrypt(requestStruct) + + if err != nil { + return nil, fmt.Errorf("error encrypting data: %s", err.Error()) + } + + fmt.Println("HGX: encrypted string is ", genresp.CiphertextBlob) + return []byte(genresp.CiphertextBlob), nil +} + +func (a *alicloudKms) decrypt(s []byte) ([]byte, error) { + derequestStruct := kms.CreateDecryptRequest() + derequestStruct.CiphertextBlob = string(s[:]) + derequestStruct.RpcRequest.SetScheme("HTTPS") + + degenresp, err := a.kmsClient.Decrypt(derequestStruct) + + if err != nil { + return nil, fmt.Errorf("error decrypting data: %s", err.Error()) + } + + return []byte(degenresp.Plaintext), nil +} + +func (a *alicloudKms) Get(key string) ([]byte, error) { + cipherText, err := a.store.Get(key) + + if err != nil { + return nil, err + } + + return a.decrypt(cipherText) +} + +func (a *alicloudKms) Set(key string, val []byte) error { + cipherText, err := a.encrypt(val) + + if err != nil { + return err + } + + return a.store.Set(key, cipherText) +} + +func (a *alicloudKms) Test(key string) error { + // TODO: Implement a test if a Set is likely to work, Alicloud doesn't seemt to provide a dry-run on the parameter store + return nil +} diff --git a/pkg/kv/alicloud_oss/alicloud_oss.go b/pkg/kv/alicloud_oss/alicloud_oss.go new file mode 100644 index 00000000..eb630acb --- /dev/null +++ b/pkg/kv/alicloud_oss/alicloud_oss.go @@ -0,0 +1,90 @@ +package alicloud_oss + +import ( + "bytes" + "fmt" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "io" + + "github.com/jetstack/vault-unsealer/pkg/kv" +) + +type ossStorage struct { + cl *oss.Client + endpoint string + bucket string + prefix string +} + +func New(endpoint, bucket, prefix, alicloudAccessKey, alicloudSecretKey string) (kv.Service, error) { + cl, err := oss.New(endpoint, alicloudAccessKey, alicloudSecretKey) + + lsRes, _ := cl.ListBuckets() + for _, bucket := range lsRes.Buckets { + fmt.Println("Buckets:", bucket.Name) + + } + + + if err != nil { + return nil, fmt.Errorf("error creating oss client: %s", err.Error()) + } + + return &ossStorage{cl, endpoint, bucket, prefix}, nil +} + +func (o *ossStorage) Set(key string, val []byte) error { + n := objectNameWithPrefix(o.prefix, key) + bucket, err := o.cl.Bucket(o.bucket) + if err != nil { + return fmt.Errorf("error writing key '%s' to oss bucket '%s'", n, o.bucket) + } + + err = bucket.PutObject(n, bytes.NewReader(val)) + if err != nil { + return fmt.Errorf("error writing key '%s' to oss bucket '%s'", n, o.bucket) + } + + return nil +} + +func (o *ossStorage) Get(key string) ([]byte, error) { + n := objectNameWithPrefix(o.prefix, key) + fmt.Println("HGX: n in Get is %s", n) + + b := new(bytes.Buffer) + + bucket, err := o.cl.Bucket(o.bucket) + if err != nil { + return nil, fmt.Errorf("error writing key '%s' to oss bucket '%s'", n, o.bucket) + } + + lsRes, _ := o.cl.ListBuckets() + for _, bucket := range lsRes.Buckets { + fmt.Println("Buckets:", bucket.Name) + + } + + + body, err := bucket.GetObject(n) + if err != nil { + if err != nil { + return nil, kv.NewNotFoundError("error getting object for key '%s': %s", n, err.Error()) + } + //return nil, fmt.Errorf("error writing key '%s' to oss bucket '%s'", n, o.bucket) + } + + io.Copy(b, body) + body.Close() + + return b.Bytes(), nil +} + +func objectNameWithPrefix(prefix, key string) string { + return fmt.Sprintf("%s%s", prefix, key) +} + +func (o *ossStorage) Test(key string) error { + // TODO: Implement me properly + return nil +} From 74ed7bd1f02090082f1e7ef8e4377e192ff51191 Mon Sep 17 00:00:00 2001 From: georgedriver <36863151+georgedriver@users.noreply.github.com> Date: Wed, 24 Oct 2018 18:29:23 +0800 Subject: [PATCH 2/5] format adjust --- cmd/root.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index b1094425..6b8d7e68 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,12 +1,12 @@ package cmd import ( -"fmt" -"os" -"strings" + "fmt" + "os" + "strings" -"github.com/spf13/cobra" -"github.com/spf13/viper" + "github.com/spf13/cobra" + "github.com/spf13/viper" ) var appConfig *viper.Viper From bc1e7b08046c4edacf37982178eeb391bc9723b1 Mon Sep 17 00:00:00 2001 From: georgedriver <36863151+georgedriver@users.noreply.github.com> Date: Wed, 24 Oct 2018 18:40:27 +0800 Subject: [PATCH 3/5] Remove the debug print --- pkg/kv/alicloud_kms/alicloud_kms.go | 2 -- pkg/kv/alicloud_oss/alicloud_oss.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/pkg/kv/alicloud_kms/alicloud_kms.go b/pkg/kv/alicloud_kms/alicloud_kms.go index ff09b583..a2987baa 100644 --- a/pkg/kv/alicloud_kms/alicloud_kms.go +++ b/pkg/kv/alicloud_kms/alicloud_kms.go @@ -35,14 +35,12 @@ func (a *alicloudKms) encrypt(s []byte) ([]byte, error) { requestStruct.RpcRequest.SetScheme("HTTPS") requestStruct.Plaintext = string(s[:]) - fmt.Println("HGX: Planintext is ", string(s[:])) genresp, err := a.kmsClient.Encrypt(requestStruct) if err != nil { return nil, fmt.Errorf("error encrypting data: %s", err.Error()) } - fmt.Println("HGX: encrypted string is ", genresp.CiphertextBlob) return []byte(genresp.CiphertextBlob), nil } diff --git a/pkg/kv/alicloud_oss/alicloud_oss.go b/pkg/kv/alicloud_oss/alicloud_oss.go index eb630acb..6d2fa776 100644 --- a/pkg/kv/alicloud_oss/alicloud_oss.go +++ b/pkg/kv/alicloud_oss/alicloud_oss.go @@ -50,8 +50,6 @@ func (o *ossStorage) Set(key string, val []byte) error { func (o *ossStorage) Get(key string) ([]byte, error) { n := objectNameWithPrefix(o.prefix, key) - fmt.Println("HGX: n in Get is %s", n) - b := new(bytes.Buffer) bucket, err := o.cl.Bucket(o.bucket) From 98ad53e1f358afb47f734f4003b1c12b45714514 Mon Sep 17 00:00:00 2001 From: George He <36863151+georgedriver@users.noreply.github.com> Date: Wed, 7 Nov 2018 15:56:28 +0800 Subject: [PATCH 4/5] Update CHANGELOG.md --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0531014b..249448aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [0.3.1] - 2018-11-07 + +### Added + +* Add alicloud support + ## [0.3.0] - 2017-09-03 ### Added From 086b04a6a848c432ddf2d0da5e8fbc4ab9f2a213 Mon Sep 17 00:00:00 2001 From: George He <36863151+georgedriver@users.noreply.github.com> Date: Thu, 8 Nov 2018 22:16:14 +0800 Subject: [PATCH 5/5] Update CHANGELOG.md --- CHANGELOG.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 249448aa..0531014b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.3.1] - 2018-11-07 - -### Added - -* Add alicloud support - ## [0.3.0] - 2017-09-03 ### Added