forked from kedro-org/kedro
-
Notifications
You must be signed in to change notification settings - Fork 0
/
config.yml
223 lines (196 loc) · 4.95 KB
/
config.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
version: 2.1
executors:
py36:
docker:
- image: 350138855857.dkr.ecr.eu-west-2.amazonaws.com/kedro-builder:3.6
py37:
docker:
- image: 350138855857.dkr.ecr.eu-west-2.amazonaws.com/kedro-builder:3.7
py38:
docker:
- image: 350138855857.dkr.ecr.eu-west-2.amazonaws.com/kedro-builder:3.8
commands:
setup_conda:
description: Activate conda environment
steps:
- run:
name: Run conda.sh
command: echo ". /home/circleci/miniconda/etc/profile.d/conda.sh" >> $BASH_ENV
- run:
name: Activate conda environment
command: echo "conda deactivate; conda activate kedro_builder" >> $BASH_ENV
setup_requirements:
description: Install PIP dependencies
steps:
- run:
name: Install pip setuptools
command: make install-pip-setuptools
- run:
name: Install requirements
command: pip install -r requirements.txt -U
- run:
name: Install venv for some pre-commit hooks
command: conda install -y virtualenv
- run:
name: Install test requirements
command: pip install -r test_requirements.txt -U
- run:
# Since recently Spark installation for some reason does not have enough permissions to execute
# /home/circleci/miniconda/envs/kedro_builder/lib/python3.X/site-packages/pyspark/bin/spark-class.
# So fixing it manually here.
name: Fix Spark permissions
command: sudo chmod -R u+x /home/circleci/miniconda/envs/kedro_builder/lib/
- run:
name: Print Python environment
command: make print-python-env
- run:
name: Pip freeze
command: pip freeze
setup_pre_commit:
description: Install pre-commit hooks
steps:
- run:
name: Install pre-commit hooks
command: pre-commit install --install-hooks
- run:
name: Run pre-commit hooks
command: pre-commit install --hook-type pre-push
unit_tests:
description: Run unit tests
steps:
- checkout
- setup_conda
- setup_requirements
- run:
name: Run unit tests
command: make test
lint:
description: Run linters
steps:
- checkout
- setup_conda
- setup_requirements
- setup_pre_commit
- run:
name: Run linters
command: make lint
- run:
name: Check legal headers
command: make legal
e2e_tests:
description: Run all end to end tests
steps:
- checkout
- setup_conda
- setup_requirements
- run:
name: Run e2e tests
command: make e2e-tests
build_docs:
description: Build docs
steps:
- checkout
- setup_conda
- setup_requirements
- run:
name: Build docs
command: make build-docs
pip_compile:
description: Pip-compile requirements file
steps:
- checkout
- setup_conda
- setup_requirements
- run:
name: Pip-compile requirements file
command: make pip-compile
jobs:
unit_tests_36:
executor: py36
steps: [unit_tests]
linters_36:
executor: py36
steps: [lint]
e2e_tests_36:
executor: py36
steps: [e2e_tests]
docs_36:
executor: py36
steps: [build_docs]
unit_tests_37:
executor: py37
steps: [unit_tests]
linters_37:
executor: py37
steps: [lint]
e2e_tests_37:
executor: py37
steps: [e2e_tests]
docs_37:
executor: py37
steps: [build_docs]
unit_tests_38:
executor: py38
steps:
- checkout
- setup_conda
- setup_requirements
- run:
name: Run unit tests without Spark
command: make test-no-spark
linters_38:
executor: py38
steps: [lint]
e2e_tests_38:
executor: py38
steps: [e2e_tests]
pip_compile_36:
executor: py36
steps: [pip_compile]
pip_compile_37:
executor: py37
steps: [pip_compile]
pip_compile_38:
executor: py38
steps: [pip_compile]
all_circleci_checks_succeeded:
docker:
- image: circleci/python # any light-weight image
steps:
- run:
name: Success!
command: echo "All checks passed"
workflows:
version: 2
regular:
jobs:
- unit_tests_36
- linters_36
- e2e_tests_36
- docs_36
- unit_tests_37
- linters_37
- e2e_tests_37
- docs_37
- unit_tests_38
- linters_38
- e2e_tests_38
- pip_compile_36
- pip_compile_37
- pip_compile_38
- all_circleci_checks_succeeded:
requires:
- unit_tests_36
- linters_36
- e2e_tests_36
- docs_36
- unit_tests_37
- linters_37
- e2e_tests_37
- docs_37
- unit_tests_38
- linters_38
- e2e_tests_38
- pip_compile_36
- pip_compile_37
- pip_compile_38