Skip to content

Commit 6db3143

Browse files
kudjkudj
committed
added tests - s3 and abs tables staging
1 parent c0d29a3 commit 6db3143

File tree

5 files changed

+204
-0
lines changed

5 files changed

+204
-0
lines changed

‎tests/data_examples/data_storage_staging_abs/config.json‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
"storage": {
3+
"input": {
4+
"files": [
5+
],
6+
"tables": [
7+
{
8+
"source": "in.c-main.test",
9+
"destination": "sample.csv",
10+
"columns": [],
11+
"where_values": [],
12+
"where_operator": "eq"
13+
}
14+
]
15+
},
16+
"output": {
17+
"tables": [
18+
],
19+
"files": [
20+
]
21+
}
22+
},
23+
"parameters": {
24+
"fooBar": {
25+
"foo": 42,
26+
"bar": 24
27+
},
28+
"baz": "bazBar"
29+
},
30+
"action": "run",
31+
"authorization": {
32+
"oauth_api": {
33+
"id": "123456",
34+
"credentials": {
35+
"id": "main",
36+
"authorizedFor": "Myself",
37+
"creator": {
38+
"id": "1234",
39+
"description": "me@keboola.com"
40+
},
41+
"created": "2016-01-31 00:13:30",
42+
"oauthVersion": "2.0",
43+
"appKey": "myappkey",
44+
"#data": "{\"mykey\":\"myval\"}",
45+
"#appSecret": "myappsecret"
46+
}
47+
}
48+
}
49+
}

‎tests/data_examples/data_storage_staging_abs/in/tables/sample.csv.manifest‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"id": "in.c-main.test",
3+
"uri": "https:\/\/connection.keboola.com\/\/v2\/storage\/tables\/in.c-main.test",
4+
"name": "sample.csv",
5+
"primary_key": [],
6+
"indexed_columns": [],
7+
"created": "2015-11-02T09:11:37+0100",
8+
"last_change_date": "2015-11-02T09:11:37+0100",
9+
"last_import_date": "2015-11-02T09:11:37+0100",
10+
"rows_count": 400,
11+
"data_size_bytes": 81920,
12+
"is_alias": false,
13+
"columns": [
14+
"x",
15+
"Sales",
16+
"CompPrice",
17+
"Income",
18+
"Advertising",
19+
"Population",
20+
"Price",
21+
"ShelveLoc",
22+
"Age",
23+
"Education",
24+
"Urban",
25+
"US",
26+
"High"
27+
],
28+
"attributes": [],
29+
"abs": {
30+
"is_sliced": true,
31+
"region": "us-east-1",
32+
"container": "exp-2-export-test-test",
33+
"name": "12345.csv.gzmanifest",
34+
"credentials": {
35+
"sas_connection_string": "BlobEndpoint=https://asdf.blob.core.windows.net;SharedAccessSignature=sv=2017-11-09&sr=c&st=2020-08-27T08:42:08Z&se=2020-08-27T20:42:08Z&sp=rl&sig=UJW4DPh%2Baaaaaaaaaa",
36+
"expiration": "2020-08-27T22:42:08+0200"
37+
}
38+
}
39+
}

‎tests/data_examples/data_storage_staging_s3/config.json‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
"storage": {
3+
"input": {
4+
"files": [
5+
],
6+
"tables": [
7+
{
8+
"source": "in.c-main.test",
9+
"destination": "sample.csv",
10+
"columns": [],
11+
"where_values": [],
12+
"where_operator": "eq"
13+
}
14+
]
15+
},
16+
"output": {
17+
"tables": [
18+
],
19+
"files": [
20+
]
21+
}
22+
},
23+
"parameters": {
24+
"fooBar": {
25+
"foo": 42,
26+
"bar": 24
27+
},
28+
"baz": "bazBar"
29+
},
30+
"action": "run",
31+
"authorization": {
32+
"oauth_api": {
33+
"id": "123456",
34+
"credentials": {
35+
"id": "main",
36+
"authorizedFor": "Myself",
37+
"creator": {
38+
"id": "1234",
39+
"description": "me@keboola.com"
40+
},
41+
"created": "2016-01-31 00:13:30",
42+
"oauthVersion": "2.0",
43+
"appKey": "myappkey",
44+
"#data": "{\"mykey\":\"myval\"}",
45+
"#appSecret": "myappsecret"
46+
}
47+
}
48+
}
49+
}

‎tests/data_examples/data_storage_staging_s3/in/tables/sample.csv.manifest‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"id": "in.c-main.test",
3+
"uri": "https:\/\/connection.keboola.com\/\/v2\/storage\/tables\/in.c-main.test",
4+
"name": "sample.csv",
5+
"primary_key": [],
6+
"indexed_columns": [],
7+
"created": "2015-11-02T09:11:37+0100",
8+
"last_change_date": "2015-11-02T09:11:37+0100",
9+
"last_import_date": "2015-11-02T09:11:37+0100",
10+
"rows_count": 400,
11+
"data_size_bytes": 81920,
12+
"is_alias": false,
13+
"columns": [
14+
"x",
15+
"Sales",
16+
"CompPrice",
17+
"Income",
18+
"Advertising",
19+
"Population",
20+
"Price",
21+
"ShelveLoc",
22+
"Age",
23+
"Education",
24+
"Urban",
25+
"US",
26+
"High"
27+
],
28+
"attributes": [],
29+
"s3": {
30+
"isSliced": true,
31+
"region": "eu-central-1",
32+
"bucket": "test",
33+
"key": "test\/asdf\/12345.csv.gzmanifest",
34+
"credentials": {
35+
"access_key_id": "ASDF",
36+
"secret_access_key": "1234",
37+
"session_token": "abcd1234"
38+
}
39+
}
40+
}

‎tests/test_dao.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -808,3 +808,30 @@ def test_all_tags(self):
808808
)
809809

810810
self.assertEqual(all_tags, file_def.tags)
811+
812+
def test_build_from_manifest_s3_staging(self):
813+
sample_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
814+
'data_examples', 'data_storage_staging_s3', 'in', 'tables')
815+
816+
table_def = TableDefinition.build_from_manifest(os.path.join(sample_path, 'sample.csv.manifest'))
817+
818+
self.assertEqual(table_def.s3_staging.bucket, "test")
819+
self.assertEqual(table_def.s3_staging.credentials_access_key_id, "ASDF")
820+
self.assertEqual(table_def.s3_staging.credentials_secret_access_key,"1234")
821+
self.assertEqual(table_def.s3_staging.credentials_session_token,"abcd1234")
822+
self.assertEqual(table_def.s3_staging.is_sliced,True)
823+
self.assertEqual(table_def.s3_staging.key,"test/asdf/12345.csv.gzmanifest")
824+
self.assertEqual(table_def.s3_staging.region,"eu-central-1")
825+
826+
def test_build_from_manifest_abs_staging(self):
827+
sample_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
828+
'data_examples', 'data_storage_staging_abs', 'in', 'tables')
829+
830+
table_def = TableDefinition.build_from_manifest(os.path.join(sample_path, 'sample.csv.manifest'))
831+
832+
self.assertEqual(table_def.abs_staging.container, "exp-2-export-test-test")
833+
self.assertEqual(table_def.abs_staging.credentials_expiration, "2020-08-27T22:42:08+0200")
834+
self.assertEqual(table_def.abs_staging.credentials_sas_connection_string, "BlobEndpoint=https://asdf.blob.core.windows.net;SharedAccessSignature=sv=2017-11-09&sr=c&st=2020-08-27T08:42:08Z&se=2020-08-27T20:42:08Z&sp=rl&sig=UJW4DPh%2Baaaaaaaaaa")
835+
self.assertEqual(table_def.abs_staging.is_sliced, True)
836+
self.assertEqual(table_def.abs_staging.name, "12345.csv.gzmanifest")
837+
self.assertEqual(table_def.abs_staging.region, "us-east-1")

0 commit comments

Comments
 (0)