dabs/bundle.yml at main · dlevy324/dabs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
# yaml-language-server: $schema=bundle_config_schema.json
bundle:
  name: medium_post_report

workspace:
  host: https://e2-demo-field-eng.cloud.databricks.com/

resources:

  pipelines:
    # Our DLT pipeline
    medium_metrics_pipeline:
      name: "[${bundle.environment}] FE Medium Metrics Pipeline2"
      target: "medium_post_report_${bundle.environment}"
      libraries:
        - file:
            path: ./ingest.py
        - file:
            path: ./get_metrics.py
      channel: preview
      configuration:
        "bundle.file_path": "/Workspace/${workspace.file_path}"

  jobs:
    # A two-task Databricks Workflow - dlt + notebook report
    fe_medium_metrics:
      name: "[${bundle.environment}] Metrics for FE Medium Posts2"
      tasks:
        - task_key: dlt_medium_pipeline
          pipeline_task:
            pipeline_id: ${resources.pipelines.medium_metrics_pipeline.id}
        - task_key: "${bundle.environment}_medium_notebook_report"
          depends_on:
            - task_key: dlt_medium_pipeline
          notebook_task:
            base_parameters:
              dbname: "medium_post_report_${bundle.environment}"
            notebook_path: ./fe_medium_report.py
          new_cluster:
            spark_version: 13.1.x-scala2.12
            num_workers: 1
            node_type_id: i3.xlarge

environments:
  development:
    default: true
    resources:
      pipelines:
        medium_metrics_pipeline:
          development: true

  qa: # This environment is when deploying test runs from a pull request on GitHub..
    workspace:
      host: https://e2-demo-field-eng.cloud.databricks.com/
    resources:
      pipelines:
        medium_metrics_pipeline:
          development: true
          permissions:
            - level: CAN_VIEW
              group_name: users

  production:
    workspace:
      host: https://e2-demo-field-eng.cloud.databricks.com/
    resources:
      pipelines:
        medium_metrics_pipeline:
          permissions:
          - level: CAN_VIEW
            group_name: users
          development: false
          photon: true
          clusters:
            - autoscale:
                min_workers: 2
                max_workers: 8