teckel4s/docs/etl/example.yaml at master · eff3ct0/teckel4s · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
## Inputs

input:
  - name: table1
    format: csv
    path: 'prefix://bucket1/path/path1'
    options:
      - header: true
      - sep: ','

  - name: table2
    format: parquet
    select:
      - t2pk1
      - t2pk2
      - field2
    path: 'prefix://bucket1/path/path2'

  - name: table5
    format: json
    path: 'prefix://bucket1/path/path5'

  - name: table6
    format: csv
    select:
      - t6pk1
      - t6pk2
      - field6
    path: 'prefix://bucket1/path/path6'
    options:
      - header: true
      - sep: '|'

  - name: table8
    format: parquet
    path: 'prefix://bucket1/path/path8'


## Transformations

transformation:
  - name: tableFinal
    join:
      joinType: left
      relation:
        left: table1
        right:
          - name: table2
            fields:
              - t1pk1:t2pk1 # left : right
              - t1pk2:t2pk2 # table1 : table2

          - name: table3
            fields:
              - t1pk1:t3pk1 # table1 : table3
              - t1pk2:t3pk2 # table1 : table3

          - name: table4
            fields:
              - t1pk1:t4pk1 # table1 : table4

  - name: table3
    group:
      name: table8
      by:
        - f1
        - f2
      agg:
        - expr1

  - name: table4
    join:
      joinType: inner
      relation:
        left: table6
        right:
          - name: table7
            fields:
              - t6pk1:t7pk1 # left : right
              - t6pk2:t7pk2 # table6 : table7

## Output

output:
  - name: tableFinal
    format: parquet
    path: 'prefix://bucket2/path/path1'

  - name: table4
    format: parquet
    path: 'prefix://bucket2/path/path2'