Skip to content

Commit 422896d

Browse files
kudjkudj
committed
added s3 and abs staging for table definitions
1 parent 037aa22 commit 422896d

File tree

6 files changed

+222
-0
lines changed

6 files changed

+222
-0
lines changed

‎src/keboola/component/dao.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -799,6 +799,8 @@ def __init__(self, name: str,
799799
has_header: Optional[bool] = None,
800800
description: Optional[str] = None,
801801
# input
802+
s3: Optional[dict] = None,
803+
abs: Optional[dict] = None,
802804
**kwargs
803805
):
804806
"""
@@ -890,6 +892,10 @@ def __init__(self, name: str,
890892
self.stage = stage
891893
self.has_header = has_header or self._has_header_in_file()
892894

895+
# storage staging
896+
self._s3 = s3
897+
self._abs = abs
898+
893899
def __get_stage_inferred(self):
894900
if self._uri:
895901
return 'in'
@@ -974,6 +980,10 @@ def build_input_definition(cls, name: str,
974980
created: Optional[str] = None,
975981
last_change_date: Optional[str] = None,
976982
last_import_date: Optional[str] = None,
983+
984+
# storage staging
985+
s3: Optional[dict] = None,
986+
abs: Optional[dict] = None,
977987
**kwargs
978988
):
979989
"""
@@ -1005,6 +1015,8 @@ def build_input_definition(cls, name: str,
10051015
created (Optional[str]): The creation timestamp of the table. Defaults to None.
10061016
last_change_date (Optional[str]): The last modification timestamp of the table. Defaults to None.
10071017
last_import_date (Optional[str]): The last import timestamp of the table. Defaults to None.
1018+
s3 (Optional[dict]): A dictionary containing Amazon S3 storage details. Defaults to None.
1019+
abs (Optional[dict]): A dictionary containing Azure Blob Storage details. Defaults to None.
10081020
10091021
Returns:
10101022
TableDefinition: An instance of TableDefinition configured for input tables.
@@ -1031,6 +1043,8 @@ def build_input_definition(cls, name: str,
10311043
created=created,
10321044
last_change_date=last_change_date,
10331045
last_import_date=last_import_date,
1046+
s3=s3,
1047+
abs=abs,
10341048
**kwargs
10351049
)
10361050

@@ -1156,6 +1170,10 @@ def build_from_manifest(cls,
11561170
is_alias=manifest.get('is_alias'),
11571171
attributes=manifest.get('attributes'),
11581172
indexed_columns=manifest.get('indexed_columns'),
1173+
1174+
# storage staging
1175+
s3=manifest.get('s3'),
1176+
abs=manifest.get('abs'),
11591177
)
11601178

11611179
else:

‎tests/data_examples/data_storage_staging_abs/config.json‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
"storage": {
3+
"input": {
4+
"files": [
5+
],
6+
"tables": [
7+
{
8+
"source": "in.c-main.test",
9+
"destination": "sample.csv",
10+
"columns": [],
11+
"where_values": [],
12+
"where_operator": "eq"
13+
}
14+
]
15+
},
16+
"output": {
17+
"tables": [
18+
],
19+
"files": [
20+
]
21+
}
22+
},
23+
"parameters": {
24+
"fooBar": {
25+
"foo": 42,
26+
"bar": 24
27+
},
28+
"baz": "bazBar"
29+
},
30+
"action": "run",
31+
"authorization": {
32+
"oauth_api": {
33+
"id": "123456",
34+
"credentials": {
35+
"id": "main",
36+
"authorizedFor": "Myself",
37+
"creator": {
38+
"id": "1234",
39+
"description": "me@keboola.com"
40+
},
41+
"created": "2016-01-31 00:13:30",
42+
"oauthVersion": "2.0",
43+
"appKey": "myappkey",
44+
"#data": "{\"mykey\":\"myval\"}",
45+
"#appSecret": "myappsecret"
46+
}
47+
}
48+
}
49+
}

‎tests/data_examples/data_storage_staging_abs/in/tables/sample.csv.manifest‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"id": "in.c-main.test",
3+
"uri": "https:\/\/connection.keboola.com\/\/v2\/storage\/tables\/in.c-main.test",
4+
"name": "sample.csv",
5+
"primary_key": [],
6+
"indexed_columns": [],
7+
"created": "2015-11-02T09:11:37+0100",
8+
"last_change_date": "2015-11-02T09:11:37+0100",
9+
"last_import_date": "2015-11-02T09:11:37+0100",
10+
"rows_count": 400,
11+
"data_size_bytes": 81920,
12+
"is_alias": false,
13+
"columns": [
14+
"x",
15+
"Sales",
16+
"CompPrice",
17+
"Income",
18+
"Advertising",
19+
"Population",
20+
"Price",
21+
"ShelveLoc",
22+
"Age",
23+
"Education",
24+
"Urban",
25+
"US",
26+
"High"
27+
],
28+
"attributes": [],
29+
"abs": {
30+
"is_sliced": true,
31+
"region": "us-east-1",
32+
"container": "exp-2-export-test-test",
33+
"name": "12345.csv.gzmanifest",
34+
"credentials": {
35+
"sas_connection_string": "BlobEndpoint=https://asdf.blob.core.windows.net;SharedAccessSignature=sv=2017-11-09&sr=c&st=2020-08-27T08:42:08Z&se=2020-08-27T20:42:08Z&sp=rl&sig=UJW4DPh%2Baaaaaaaaaa",
36+
"expiration": "2020-08-27T22:42:08+0200"
37+
}
38+
}
39+
}

‎tests/data_examples/data_storage_staging_s3/config.json‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
"storage": {
3+
"input": {
4+
"files": [
5+
],
6+
"tables": [
7+
{
8+
"source": "in.c-main.test",
9+
"destination": "sample.csv",
10+
"columns": [],
11+
"where_values": [],
12+
"where_operator": "eq"
13+
}
14+
]
15+
},
16+
"output": {
17+
"tables": [
18+
],
19+
"files": [
20+
]
21+
}
22+
},
23+
"parameters": {
24+
"fooBar": {
25+
"foo": 42,
26+
"bar": 24
27+
},
28+
"baz": "bazBar"
29+
},
30+
"action": "run",
31+
"authorization": {
32+
"oauth_api": {
33+
"id": "123456",
34+
"credentials": {
35+
"id": "main",
36+
"authorizedFor": "Myself",
37+
"creator": {
38+
"id": "1234",
39+
"description": "me@keboola.com"
40+
},
41+
"created": "2016-01-31 00:13:30",
42+
"oauthVersion": "2.0",
43+
"appKey": "myappkey",
44+
"#data": "{\"mykey\":\"myval\"}",
45+
"#appSecret": "myappsecret"
46+
}
47+
}
48+
}
49+
}

‎tests/data_examples/data_storage_staging_s3/in/tables/sample.csv.manifest‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"id": "in.c-main.test",
3+
"uri": "https:\/\/connection.keboola.com\/\/v2\/storage\/tables\/in.c-main.test",
4+
"name": "sample.csv",
5+
"primary_key": [],
6+
"indexed_columns": [],
7+
"created": "2015-11-02T09:11:37+0100",
8+
"last_change_date": "2015-11-02T09:11:37+0100",
9+
"last_import_date": "2015-11-02T09:11:37+0100",
10+
"rows_count": 400,
11+
"data_size_bytes": 81920,
12+
"is_alias": false,
13+
"columns": [
14+
"x",
15+
"Sales",
16+
"CompPrice",
17+
"Income",
18+
"Advertising",
19+
"Population",
20+
"Price",
21+
"ShelveLoc",
22+
"Age",
23+
"Education",
24+
"Urban",
25+
"US",
26+
"High"
27+
],
28+
"attributes": [],
29+
"s3": {
30+
"isSliced": true,
31+
"region": "eu-central-1",
32+
"bucket": "test",
33+
"key": "test\/asdf\/12345.csv.gzmanifest",
34+
"credentials": {
35+
"access_key_id": "ASDF",
36+
"secret_access_key": "1234",
37+
"session_token": "abcd1234"
38+
}
39+
}
40+
}

‎tests/test_dao.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -808,3 +808,30 @@ def test_all_tags(self):
808808
)
809809

810810
self.assertEqual(all_tags, file_def.tags)
811+
812+
def test_build_from_manifest_s3_staging(self):
813+
sample_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
814+
'data_examples', 'data_storage_staging_s3', 'in', 'tables')
815+
816+
table_def = TableDefinition.build_from_manifest(os.path.join(sample_path, 'sample.csv.manifest'))
817+
818+
self.assertEqual(table_def.s3_staging.bucket, "test")
819+
self.assertEqual(table_def.s3_staging.credentials_access_key_id, "ASDF")
820+
self.assertEqual(table_def.s3_staging.credentials_secret_access_key,"1234")
821+
self.assertEqual(table_def.s3_staging.credentials_session_token,"abcd1234")
822+
self.assertEqual(table_def.s3_staging.is_sliced,True)
823+
self.assertEqual(table_def.s3_staging.key,"test/asdf/12345.csv.gzmanifest")
824+
self.assertEqual(table_def.s3_staging.region,"eu-central-1")
825+
826+
def test_build_from_manifest_abs_staging(self):
827+
sample_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
828+
'data_examples', 'data_storage_staging_abs', 'in', 'tables')
829+
830+
table_def = TableDefinition.build_from_manifest(os.path.join(sample_path, 'sample.csv.manifest'))
831+
832+
self.assertEqual(table_def.abs_staging.container, "exp-2-export-test-test")
833+
self.assertEqual(table_def.abs_staging.credentials_expiration, "2020-08-27T22:42:08+0200")
834+
self.assertEqual(table_def.abs_staging.credentials_sas_connection_string, "BlobEndpoint=https://asdf.blob.core.windows.net;SharedAccessSignature=sv=2017-11-09&sr=c&st=2020-08-27T08:42:08Z&se=2020-08-27T20:42:08Z&sp=rl&sig=UJW4DPh%2Baaaaaaaaaa")
835+
self.assertEqual(table_def.abs_staging.is_sliced, True)
836+
self.assertEqual(table_def.abs_staging.name, "12345.csv.gzmanifest")
837+
self.assertEqual(table_def.abs_staging.region, "us-east-1")

0 commit comments

Comments
 (0)