Skip to content
This repository was archived by the owner on Dec 2, 2020. It is now read-only.

Commit 38b084b

Browse files
clareliguoriclareliguori
committed
Try to match existing Lambda layers before publishing new ones
Fixes #3
1 parent 1d1aa83 commit 38b084b

File tree

2 files changed

+220
-41
lines changed

2 files changed

+220
-41
lines changed

‎img2lambda/publish/publish_layers.go‎

Lines changed: 67 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
package publish
44

55
import (
6+
"crypto/sha256"
7+
"encoding/base64"
68
"encoding/json"
79
"io/ioutil"
810
"log"
@@ -12,6 +14,7 @@ import (
1214

1315
"github.com/aws/aws-sdk-go/aws"
1416
"github.com/aws/aws-sdk-go/service/lambda"
17+
"github.com/aws/aws-sdk-go/service/lambda/lambdaiface"
1518
"github.com/awslabs/aws-lambda-container-image-converter/img2lambda/types"
1619
)
1720

@@ -26,26 +29,35 @@ func PublishLambdaLayers(opts *types.PublishOptions, layers []types.LambdaLayer)
2629
return "", err
2730
}
2831

29-
publishArgs := &lambda.PublishLayerVersionInput{
30-
CompatibleRuntimes: []*string{aws.String("provided")},
31-
Content: &lambda.LayerVersionContentInput{ZipFile: layerContents},
32-
Description: aws.String("created by img2lambda from image " + opts.SourceImageName),
33-
LayerName: aws.String(layerName),
34-
}
35-
36-
resp, err := opts.LambdaClient.PublishLayerVersion(publishArgs)
32+
found, existingArn, err := matchExistingLambdaLayer(layerName, layerContents, &opts.LambdaClient)
3733
if err != nil {
3834
return "", err
3935
}
4036

41-
layerArns = append(layerArns, *resp.LayerVersionArn)
37+
if found {
38+
layerArns = append(layerArns, existingArn)
39+
log.Printf("Matched Lambda layer file %s (image layer %s) to existing Lambda layer: %s", layer.File, layer.Digest, existingArn)
40+
} else {
41+
publishArgs := &lambda.PublishLayerVersionInput{
42+
CompatibleRuntimes: []*string{aws.String("provided")},
43+
Content: &lambda.LayerVersionContentInput{ZipFile: layerContents},
44+
Description: aws.String("created by img2lambda from image " + opts.SourceImageName),
45+
LayerName: aws.String(layerName),
46+
}
47+
48+
resp, err := opts.LambdaClient.PublishLayerVersion(publishArgs)
49+
if err != nil {
50+
return "", err
51+
}
52+
53+
layerArns = append(layerArns, *resp.LayerVersionArn)
54+
log.Printf("Published Lambda layer file %s (image layer %s) to Lambda: %s", layer.File, layer.Digest, *resp.LayerVersionArn)
55+
}
4256

4357
err = os.Remove(layer.File)
4458
if err != nil {
4559
return "", err
4660
}
47-
48-
log.Printf("Published Lambda layer file %s (image layer %s) to Lambda: %s", layer.File, layer.Digest, *resp.LayerVersionArn)
4961
}
5062

5163
jsonArns, err := json.MarshalIndent(layerArns, "", " ")
@@ -69,3 +81,47 @@ func PublishLambdaLayers(opts *types.PublishOptions, layers []types.LambdaLayer)
6981

7082
return resultsPath, nil
7183
}
84+
85+
func matchExistingLambdaLayer(layerName string, layerContents []byte, lambdaClient *lambdaiface.LambdaAPI) (bool, string, error) {
86+
hash := sha256.Sum256(layerContents)
87+
hashStr := base64.StdEncoding.EncodeToString(hash[:])
88+
89+
var marker *string
90+
client := *lambdaClient
91+
92+
for {
93+
listArgs := &lambda.ListLayerVersionsInput{
94+
LayerName: aws.String(layerName),
95+
Marker: marker,
96+
}
97+
98+
resp, err := client.ListLayerVersions(listArgs)
99+
if err != nil {
100+
return false, "", err
101+
}
102+
103+
for _, layerVersion := range resp.LayerVersions {
104+
getArgs := &lambda.GetLayerVersionInput{
105+
LayerName: aws.String(layerName),
106+
VersionNumber: layerVersion.Version,
107+
}
108+
109+
layerResp, err := client.GetLayerVersion(getArgs)
110+
if err != nil {
111+
return false, "", err
112+
}
113+
114+
if *layerResp.Content.CodeSha256 == hashStr && *layerResp.Content.CodeSize == int64(len(layerContents)) {
115+
return true, *layerResp.LayerVersionArn, nil
116+
}
117+
}
118+
119+
if resp.NextMarker == nil {
120+
break
121+
}
122+
123+
marker = resp.NextMarker
124+
}
125+
126+
return false, "", nil
127+
}

‎img2lambda/publish/publish_layers_test.go‎

Lines changed: 153 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package publish
55
import (
66
"encoding/json"
77
"errors"
8+
"fmt"
89
"io/ioutil"
910
"os"
1011
"strconv"
@@ -35,6 +36,7 @@ func mockLayer(t *testing.T, n int) types.LambdaLayer {
3536

3637
_, err = tmpFile.WriteString("hello world " + strconv.Itoa(n))
3738
assert.Nil(t, err)
39+
3840
return types.LambdaLayer{
3941
Digest: "sha256:" + strconv.Itoa(n),
4042
File: tmpFile.Name(),
@@ -46,10 +48,145 @@ func mockLayers(t *testing.T) []types.LambdaLayer {
4648

4749
layers = append(layers, mockLayer(t, 1))
4850
layers = append(layers, mockLayer(t, 2))
51+
layers = append(layers, mockLayer(t, 3))
4952

5053
return layers
5154
}
5255

56+
func mockPublishNoExistingLayers(t *testing.T, lambdaClient *mocks.MockLambdaAPI, n int) {
57+
layerName := aws.String(fmt.Sprintf("test-prefix-sha256-%d", n))
58+
59+
expectedPublishInput := &lambda.PublishLayerVersionInput{
60+
CompatibleRuntimes: []*string{aws.String("provided")},
61+
Content: &lambda.LayerVersionContentInput{ZipFile: []byte(fmt.Sprintf("hello world %d", n))},
62+
Description: aws.String("created by img2lambda from image test-image"),
63+
LayerName: layerName,
64+
}
65+
66+
expectedPublishOutput := &lambda.PublishLayerVersionOutput{
67+
LayerVersionArn: aws.String(fmt.Sprintf("arn:aws:lambda:us-east-2:123456789012:layer:example-layer-%d:1", n)),
68+
}
69+
70+
// Test out pagination
71+
expectedListInput1 := &lambda.ListLayerVersionsInput{
72+
LayerName: layerName,
73+
}
74+
75+
expectedListOutput1 := &lambda.ListLayerVersionsOutput{
76+
LayerVersions: []*lambda.LayerVersionsListItem{},
77+
NextMarker: aws.String("hello"),
78+
}
79+
80+
expectedListInput2 := &lambda.ListLayerVersionsInput{
81+
LayerName: layerName,
82+
Marker: aws.String("hello"),
83+
}
84+
85+
expectedListOutput2 := &lambda.ListLayerVersionsOutput{
86+
LayerVersions: []*lambda.LayerVersionsListItem{},
87+
}
88+
89+
gomock.InOrder(
90+
lambdaClient.EXPECT().ListLayerVersions(gomock.Eq(expectedListInput1)).Return(expectedListOutput1, nil),
91+
lambdaClient.EXPECT().ListLayerVersions(gomock.Eq(expectedListInput2)).Return(expectedListOutput2, nil),
92+
lambdaClient.EXPECT().PublishLayerVersion(gomock.Eq(expectedPublishInput)).Return(expectedPublishOutput, nil),
93+
)
94+
}
95+
96+
func mockPublishNoMatchingLayers(t *testing.T, lambdaClient *mocks.MockLambdaAPI, n int) {
97+
layerName := aws.String(fmt.Sprintf("test-prefix-sha256-%d", n))
98+
99+
expectedPublishInput := &lambda.PublishLayerVersionInput{
100+
CompatibleRuntimes: []*string{aws.String("provided")},
101+
Content: &lambda.LayerVersionContentInput{ZipFile: []byte(fmt.Sprintf("hello world %d", n))},
102+
Description: aws.String("created by img2lambda from image test-image"),
103+
LayerName: layerName,
104+
}
105+
106+
expectedPublishOutput := &lambda.PublishLayerVersionOutput{
107+
LayerVersionArn: aws.String(fmt.Sprintf("arn:aws:lambda:us-east-2:123456789012:layer:example-layer-%d:1", n)),
108+
}
109+
110+
expectedListInput := &lambda.ListLayerVersionsInput{
111+
LayerName: layerName,
112+
}
113+
114+
var existingVersions []*lambda.LayerVersionsListItem
115+
existingVersionNumber := int64(0)
116+
existingVersionListItem := &lambda.LayerVersionsListItem{
117+
LayerVersionArn: aws.String(fmt.Sprintf("arn:aws:lambda:us-east-2:123456789012:layer:example-layer-%d:0", n)),
118+
Version: &existingVersionNumber,
119+
}
120+
existingVersions = append(existingVersions, existingVersionListItem)
121+
expectedListOutput := &lambda.ListLayerVersionsOutput{
122+
LayerVersions: existingVersions,
123+
}
124+
125+
expectedGetInput := &lambda.GetLayerVersionInput{
126+
LayerName: layerName,
127+
VersionNumber: &existingVersionNumber,
128+
}
129+
130+
size := int64(0)
131+
expectedContentOutput := &lambda.LayerVersionContentOutput{
132+
CodeSha256: aws.String("kjsdflkjfd"),
133+
CodeSize: &size,
134+
}
135+
136+
expectedGetOutput := &lambda.GetLayerVersionOutput{
137+
Version: &existingVersionNumber,
138+
Content: expectedContentOutput,
139+
LayerVersionArn: aws.String(fmt.Sprintf("arn:aws:lambda:us-east-2:123456789012:layer:example-layer-%d:0", n)),
140+
}
141+
142+
gomock.InOrder(
143+
lambdaClient.EXPECT().ListLayerVersions(gomock.Eq(expectedListInput)).Return(expectedListOutput, nil),
144+
lambdaClient.EXPECT().GetLayerVersion(gomock.Eq(expectedGetInput)).Return(expectedGetOutput, nil),
145+
lambdaClient.EXPECT().PublishLayerVersion(gomock.Eq(expectedPublishInput)).Return(expectedPublishOutput, nil),
146+
)
147+
}
148+
149+
func mockMatchingLayer(t *testing.T, lambdaClient *mocks.MockLambdaAPI, n int) {
150+
layerName := aws.String(fmt.Sprintf("test-prefix-sha256-%d", n))
151+
152+
expectedListInput := &lambda.ListLayerVersionsInput{
153+
LayerName: layerName,
154+
}
155+
156+
var existingVersions []*lambda.LayerVersionsListItem
157+
existingVersionNumber := int64(0)
158+
existingVersionListItem := &lambda.LayerVersionsListItem{
159+
LayerVersionArn: aws.String(fmt.Sprintf("arn:aws:lambda:us-east-2:123456789012:layer:example-layer-%d:1", n)),
160+
Version: &existingVersionNumber,
161+
}
162+
existingVersions = append(existingVersions, existingVersionListItem)
163+
expectedListOutput := &lambda.ListLayerVersionsOutput{
164+
LayerVersions: existingVersions,
165+
}
166+
167+
expectedGetInput := &lambda.GetLayerVersionInput{
168+
LayerName: layerName,
169+
VersionNumber: &existingVersionNumber,
170+
}
171+
172+
size := int64(13)
173+
expectedContentOutput := &lambda.LayerVersionContentOutput{
174+
CodeSha256: aws.String("T/q7q052MgJGLfH1mBGUQSFYjwVn9VvOWBoOmevPZgY="),
175+
CodeSize: &size,
176+
}
177+
178+
expectedGetOutput := &lambda.GetLayerVersionOutput{
179+
Version: &existingVersionNumber,
180+
Content: expectedContentOutput,
181+
LayerVersionArn: aws.String(fmt.Sprintf("arn:aws:lambda:us-east-2:123456789012:layer:example-layer-%d:1", n)),
182+
}
183+
184+
gomock.InOrder(
185+
lambdaClient.EXPECT().ListLayerVersions(gomock.Eq(expectedListInput)).Return(expectedListOutput, nil),
186+
lambdaClient.EXPECT().GetLayerVersion(gomock.Eq(expectedGetInput)).Return(expectedGetOutput, nil),
187+
)
188+
}
189+
53190
func TestNoLayers(t *testing.T) {
54191
ctrl := gomock.NewController(t)
55192
defer ctrl.Finish()
@@ -95,43 +232,18 @@ func TestPublishSuccess(t *testing.T) {
95232

96233
layers := mockLayers(t)
97234

98-
expectedInput1 := &lambda.PublishLayerVersionInput{
99-
CompatibleRuntimes: []*string{aws.String("provided")},
100-
Content: &lambda.LayerVersionContentInput{ZipFile: []byte("hello world 1")},
101-
Description: aws.String("created by img2lambda from image test-image"),
102-
LayerName: aws.String("test-prefix-sha256-1"),
103-
}
104-
105-
expectedOutput1 := &lambda.PublishLayerVersionOutput{
106-
LayerVersionArn: aws.String("arn:aws:lambda:us-east-2:123456789012:layer:example-layer-1:1"),
107-
}
108-
109-
expectedInput2 := &lambda.PublishLayerVersionInput{
110-
CompatibleRuntimes: []*string{aws.String("provided")},
111-
Content: &lambda.LayerVersionContentInput{ZipFile: []byte("hello world 2")},
112-
Description: aws.String("created by img2lambda from image test-image"),
113-
LayerName: aws.String("test-prefix-sha256-2"),
114-
}
115-
116-
expectedOutput2 := &lambda.PublishLayerVersionOutput{
117-
LayerVersionArn: aws.String("arn:aws:lambda:us-east-2:123456789012:layer:example-layer-2:1"),
118-
}
119-
120-
lambdaClient.EXPECT().
121-
PublishLayerVersion(gomock.Eq(expectedInput1)).
122-
Return(expectedOutput1, nil)
123-
124-
lambdaClient.EXPECT().
125-
PublishLayerVersion(gomock.Eq(expectedInput2)).
126-
Return(expectedOutput2, nil)
235+
mockPublishNoExistingLayers(t, lambdaClient, 1)
236+
mockPublishNoMatchingLayers(t, lambdaClient, 2)
237+
mockMatchingLayer(t, lambdaClient, 3)
127238

128239
resultsFilename, err := PublishLambdaLayers(opts, layers)
129240
assert.Nil(t, err)
130241

131242
resultArns := parseResult(t, resultsFilename)
132-
assert.Len(t, resultArns, 2)
243+
assert.Len(t, resultArns, 3)
133244
assert.Equal(t, "arn:aws:lambda:us-east-2:123456789012:layer:example-layer-1:1", resultArns[0])
134245
assert.Equal(t, "arn:aws:lambda:us-east-2:123456789012:layer:example-layer-2:1", resultArns[1])
246+
assert.Equal(t, "arn:aws:lambda:us-east-2:123456789012:layer:example-layer-3:1", resultArns[2])
135247

136248
os.Remove(dir)
137249
}
@@ -154,6 +266,16 @@ func TestPublishError(t *testing.T) {
154266

155267
layers := mockLayers(t)
156268

269+
expectedListInput := &lambda.ListLayerVersionsInput{
270+
LayerName: aws.String("test-prefix-sha256-1"),
271+
}
272+
273+
expectedListOutput := &lambda.ListLayerVersionsOutput{
274+
LayerVersions: []*lambda.LayerVersionsListItem{},
275+
}
276+
277+
lambdaClient.EXPECT().ListLayerVersions(gomock.Eq(expectedListInput)).Return(expectedListOutput, nil)
278+
157279
expectedInput1 := &lambda.PublishLayerVersionInput{
158280
CompatibleRuntimes: []*string{aws.String("provided")},
159281
Content: &lambda.LayerVersionContentInput{ZipFile: []byte("hello world 1")},
@@ -171,6 +293,7 @@ func TestPublishError(t *testing.T) {
171293

172294
os.Remove(layers[0].File)
173295
os.Remove(layers[1].File)
296+
os.Remove(layers[2].File)
174297

175298
os.Remove(dir)
176299
}

0 commit comments

Comments
 (0)