Skip to content

Commit 58b2111

Browse files
andre-salvatiandre-salvati
committed
fix
1 parent c65ce59 commit 58b2111

File tree

8 files changed

+60
-129
lines changed

8 files changed

+60
-129
lines changed

‎.github/workflows/onpush.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@ jobs:
3333
pipenv run curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh
3434
pipenv run databricks --version
3535
36-
# - name: Deploy on staging
37-
# run: |
38-
# make deploy-staging
36+
- name: Deploy on staging
37+
run: |
38+
make deploy-staging
3939
40-
# - name: Run on staging
41-
# run: |
42-
# make run-staging
40+
- name: Run on staging
41+
run: |
42+
make run-staging
4343
4444
- name: Deploy on prod
4545
run: |

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11

2-
# Project Template for Spark/Databricks with Python packaging and CI/CD automation
2+
# Databricks template project with Asset Bundles, Python packaging and CI/CD automation
33

44
This project template provides a structured approach to enhance your productivity when delivering ETL pipelines on Databricks. Feel free to customize it based on your project's specific nuances and the audience you are targeting.
55

‎conf/workflow.yml‎

Lines changed: 0 additions & 89 deletions
This file was deleted.

‎databricks.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ targets:
2727
default: true
2828
workspace:
2929
profile: dev
30+
run_as:
31+
user_name: [email protected]
3032

3133
# Optionally, there could be a 'staging' target here.
3234
# (See Databricks docs on CI/CD at https://docs.databricks.com/dev-tools/bundles/index.html.)

‎docs/ci_cd.drawio‎

Lines changed: 49 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,77 @@
11
<mxfile host="65bd71144e">
22
<diagram id="mtFdcSvoKdh9C-5KIGSu" name="Page-1">
3-
<mxGraphModel dx="1048" dy="638" grid="0" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" background="#ffffff" math="0" shadow="0">
3+
<mxGraphModel dx="1105" dy="694" grid="0" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" background="#ffffff" math="0" shadow="0">
44
<root>
55
<mxCell id="0"/>
66
<mxCell id="1" parent="0"/>
7-
<mxCell id="2" value="VS Code and notebooks" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" vertex="1" parent="1">
8-
<mxGeometry x="440" y="190" width="120" height="60" as="geometry"/>
7+
<mxCell id="2" value="VS Code and notebooks" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" parent="1" vertex="1">
8+
<mxGeometry x="407" y="185" width="120" height="60" as="geometry"/>
99
</mxCell>
10-
<mxCell id="3" value="prototype" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontColor=#333333;" vertex="1" parent="1">
11-
<mxGeometry x="269" y="235" width="60" height="30" as="geometry"/>
10+
<mxCell id="3" value="prototype" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontColor=#333333;" parent="1" vertex="1">
11+
<mxGeometry x="236" y="230" width="60" height="30" as="geometry"/>
1212
</mxCell>
13-
<mxCell id="5" value="dev catalog&lt;br&gt;(dev workspace)" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" vertex="1" parent="1">
14-
<mxGeometry x="440" y="310" width="120" height="60" as="geometry"/>
13+
<mxCell id="5" value="dev catalog&lt;br&gt;(dev workspace)" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" parent="1" vertex="1">
14+
<mxGeometry x="407" y="305" width="120" height="60" as="geometry"/>
1515
</mxCell>
16-
<mxCell id="6" value="stage catalog&lt;br&gt;(dev workspace)" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" vertex="1" parent="1">
17-
<mxGeometry x="440" y="428" width="120" height="60" as="geometry"/>
16+
<mxCell id="6" value="stage catalog&lt;br&gt;(dev workspace)" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" parent="1" vertex="1">
17+
<mxGeometry x="407" y="423" width="120" height="60" as="geometry"/>
1818
</mxCell>
19-
<mxCell id="8" value="prod catalog&lt;br&gt;(prod workspace)" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" vertex="1" parent="1">
20-
<mxGeometry x="440" y="545" width="120" height="60" as="geometry"/>
19+
<mxCell id="8" value="prod catalog&lt;br&gt;(prod workspace)" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" parent="1" vertex="1">
20+
<mxGeometry x="407" y="540" width="120" height="60" as="geometry"/>
2121
</mxCell>
22-
<mxCell id="9" value="" style="endArrow=classic;html=1;fillColor=#f5f5f5;strokeColor=#666666;" edge="1" parent="1">
22+
<mxCell id="9" value="" style="endArrow=classic;html=1;fillColor=#f5f5f5;strokeColor=#666666;" parent="1" edge="1">
2323
<mxGeometry width="50" height="50" relative="1" as="geometry">
24-
<mxPoint x="255" y="441" as="sourcePoint"/>
25-
<mxPoint x="405" y="461" as="targetPoint"/>
24+
<mxPoint x="222" y="436" as="sourcePoint"/>
25+
<mxPoint x="372" y="456" as="targetPoint"/>
2626
</mxGeometry>
2727
</mxCell>
28-
<mxCell id="10" value="" style="endArrow=classic;html=1;fillColor=#f5f5f5;strokeColor=#666666;" edge="1" parent="1">
28+
<mxCell id="10" value="" style="endArrow=classic;html=1;fillColor=#f5f5f5;strokeColor=#666666;" parent="1" edge="1">
2929
<mxGeometry width="50" height="50" relative="1" as="geometry">
30-
<mxPoint x="260" y="370" as="sourcePoint"/>
31-
<mxPoint x="420" y="345" as="targetPoint"/>
30+
<mxPoint x="227" y="365" as="sourcePoint"/>
31+
<mxPoint x="387" y="340" as="targetPoint"/>
3232
</mxGeometry>
3333
</mxCell>
34-
<mxCell id="11" value="" style="endArrow=classic;html=1;fillColor=#f5f5f5;strokeColor=#666666;" edge="1" parent="1">
34+
<mxCell id="11" value="" style="endArrow=classic;html=1;fillColor=#f5f5f5;strokeColor=#666666;" parent="1" edge="1">
3535
<mxGeometry width="50" height="50" relative="1" as="geometry">
36-
<mxPoint x="250" y="315" as="sourcePoint"/>
37-
<mxPoint x="410" y="225" as="targetPoint"/>
36+
<mxPoint x="217" y="310" as="sourcePoint"/>
37+
<mxPoint x="377" y="220" as="targetPoint"/>
3838
</mxGeometry>
3939
</mxCell>
40-
<mxCell id="12" value="make &lt;br&gt;deploy-dev" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;fontColor=#333333;" vertex="1" parent="1">
41-
<mxGeometry x="290" y="310" width="80" height="40" as="geometry"/>
40+
<mxCell id="12" value="make &lt;br&gt;deploy-dev" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;fontColor=#333333;" parent="1" vertex="1">
41+
<mxGeometry x="257" y="305" width="80" height="40" as="geometry"/>
4242
</mxCell>
43-
<mxCell id="13" value="&lt;div style=&quot;text-align: left;&quot;&gt;&lt;span style=&quot;background-color: initial;&quot;&gt;open PR&lt;/span&gt;&lt;/div&gt;&lt;div style=&quot;text-align: left;&quot;&gt;&lt;span style=&quot;background-color: initial;&quot;&gt;for every push:&lt;/span&gt;&lt;/div&gt;&lt;div style=&quot;text-align: left;&quot;&gt;&lt;span style=&quot;background-color: initial;&quot;&gt;&amp;nbsp; - run unit tests&lt;/span&gt;&lt;/div&gt;&lt;div style=&quot;text-align: left;&quot;&gt;&lt;span style=&quot;background-color: initial;&quot;&gt;&amp;nbsp; - deploy and run workflow&lt;/span&gt;&lt;/div&gt;" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;fontColor=#333333;" vertex="1" parent="1">
44-
<mxGeometry x="271" y="379" width="170" height="70" as="geometry"/>
43+
<mxCell id="13" value="&lt;div style=&quot;text-align: left;&quot;&gt;&lt;span style=&quot;background-color: initial;&quot;&gt;open PR&lt;/span&gt;&lt;/div&gt;&lt;div style=&quot;text-align: left;&quot;&gt;&lt;span style=&quot;background-color: initial;&quot;&gt;for every push:&lt;/span&gt;&lt;/div&gt;&lt;div style=&quot;text-align: left;&quot;&gt;&lt;span style=&quot;background-color: initial;&quot;&gt;&amp;nbsp; - run unit tests&lt;/span&gt;&lt;/div&gt;&lt;div style=&quot;text-align: left;&quot;&gt;&lt;span style=&quot;background-color: initial;&quot;&gt;&amp;nbsp; - deploy and run workflow&lt;/span&gt;&lt;/div&gt;" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;fontColor=#333333;" parent="1" vertex="1">
44+
<mxGeometry x="238" y="374" width="170" height="70" as="geometry"/>
4545
</mxCell>
46-
<mxCell id="14" value="&lt;div style=&quot;text-align: left;&quot;&gt;PR approved&lt;/div&gt;" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;fontColor=#333333;" vertex="1" parent="1">
47-
<mxGeometry x="290" y="488" width="90" height="30" as="geometry"/>
46+
<mxCell id="14" value="&lt;div style=&quot;text-align: left;&quot;&gt;PR approved&lt;/div&gt;" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;fontColor=#333333;" parent="1" vertex="1">
47+
<mxGeometry x="257" y="483" width="90" height="30" as="geometry"/>
4848
</mxCell>
49-
<mxCell id="15" value="" style="endArrow=classic;html=1;fillColor=#f5f5f5;strokeColor=#666666;" edge="1" parent="1">
49+
<mxCell id="15" value="" style="endArrow=classic;html=1;fillColor=#f5f5f5;strokeColor=#666666;" parent="1" edge="1">
5050
<mxGeometry width="50" height="50" relative="1" as="geometry">
51-
<mxPoint x="250" y="495" as="sourcePoint"/>
52-
<mxPoint x="400" y="565" as="targetPoint"/>
51+
<mxPoint x="217" y="490" as="sourcePoint"/>
52+
<mxPoint x="367" y="560" as="targetPoint"/>
5353
</mxGeometry>
5454
</mxCell>
55-
<mxCell id="18" value="" style="outlineConnect=0;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;shape=mxgraph.aws3.user;fillColor=#D2D3D3;gradientColor=none;" vertex="1" parent="1">
56-
<mxGeometry x="150" y="360" width="45" height="63" as="geometry"/>
55+
<mxCell id="18" value="" style="outlineConnect=0;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;shape=mxgraph.aws3.user;fillColor=#D2D3D3;gradientColor=none;" parent="1" vertex="1">
56+
<mxGeometry x="117" y="355" width="45" height="63" as="geometry"/>
57+
</mxCell>
58+
<mxCell id="19" value="" style="shape=curlyBracket;whiteSpace=wrap;html=1;rounded=1;flipH=1;labelPosition=right;verticalLabelPosition=middle;align=left;verticalAlign=middle;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" vertex="1" parent="1">
59+
<mxGeometry x="582" y="305" width="20" height="79" as="geometry"/>
60+
</mxCell>
61+
<mxCell id="20" value="" style="shape=curlyBracket;whiteSpace=wrap;html=1;rounded=1;flipH=1;labelPosition=right;verticalLabelPosition=middle;align=left;verticalAlign=middle;fillColor=#60a917;fontColor=#ffffff;strokeColor=#2D7600;" vertex="1" parent="1">
62+
<mxGeometry x="582" y="428" width="20" height="158" as="geometry"/>
63+
</mxCell>
64+
<mxCell id="21" value="Text" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
65+
<mxGeometry x="361" y="336" width="60" height="30" as="geometry"/>
66+
</mxCell>
67+
<mxCell id="22" value="Text" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
68+
<mxGeometry x="453" y="336" width="60" height="30" as="geometry"/>
69+
</mxCell>
70+
<mxCell id="23" value="Selective run setting on config.ini&lt;br&gt;&amp;nbsp;the tasks that should run&amp;nbsp;" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;fontColor=#000000;" vertex="1" parent="1">
71+
<mxGeometry x="611" y="328" width="193" height="41" as="geometry"/>
72+
</mxCell>
73+
<mxCell id="24" value="Selective run setting on system.config &lt;br&gt;table the tasks that should run&amp;nbsp;" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;fontColor=#000000;" vertex="1" parent="1">
74+
<mxGeometry x="608" y="495" width="219" height="41" as="geometry"/>
5775
</mxCell>
5876
</root>
5977
</mxGraphModel>

‎docs/ci_cd.png‎

100755100644
2.25 KB
Loading

‎src/template/config.ini‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
[andre.f.salvati@gmail.com]
1+
[user.two@domain.com]
22
extract_source1 = true
33
extract_source2 = true
44
generate_orders = true

‎src/template/config.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ def skip_task(self):
7575
):
7676
print("Skipped with config file for 'dev' and 'ci' envs.")
7777
return True
78-
elif self.params["env"] in ("stag", "prod") and self.in_table_for_skip(self.params["task"]):
78+
elif self.params["env"] in ("staging", "prod") and self.in_table_for_skip(self.params["task"]):
7979
print("Skipped with config table for 'prod' env.")
8080
return True
8181

0 commit comments

Comments
 (0)