-
-
Notifications
You must be signed in to change notification settings - Fork 229
/
Makefile
318 lines (259 loc) · 10.8 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
#
# Makefile
#
# this is horrible magic, we use it to open a nice welcome message for
# the user in tmux in the shell that they normally use (e.g. bash, zsh)
# https://unix.stackexchange.com/questions/352316/finding-out-the-default-shell-of-a-user-within-a-shell-script
LOGIN_SHELL = $(shell finger $(USER) | grep 'Shell:*' | cut -f3 -d ":")
# Check for the environment variable OWID_SCRIPT_SHELL and fall back to LOGIN_SHELL if not set
SCRIPT_SHELL ?= $(or $(shell echo $$OWID_SCRIPT_SHELL),$(LOGIN_SHELL))
# setting .env variables as Make variables for validate.env targets
# https://lithic.tech/blog/2020-05/makefile-dot-env/
ifneq (,$(wildcard ./.env))
include .env
endif
.PHONY: help up up.full down refresh refresh.wp refresh.full migrate svgtest itsJustJavascript
help:
@echo 'Available commands:'
@echo
@echo ' GRAPHER ONLY'
@echo ' make up start dev environment via docker-compose and tmux'
@echo ' make down stop any services still running'
@echo ' make refresh (while up) download a new grapher snapshot and update MySQL'
@echo ' make refresh.pageviews (while up) download and load pageviews from the private datasette instance'
@echo ' make refresh.full (while up) run refresh and refresh.pageviews'
@echo ' make migrate (while up) run any outstanding db migrations'
@echo ' make test run full suite (except db tests) of CI checks including unit tests'
@echo ' make dbtest run db test suite that needs a running mysql db'
@echo ' make svgtest compare current rendering against reference SVGs'
@echo ' make local-bake do a full local site bake'
@echo
@echo ' GRAPHER + CLOUDFLARE (staff-only)'
@echo ' make up.full start dev environment via docker-compose and tmux'
@echo ' make update.chart-entities update the charts_x_entities join table'
@echo ' make reindex reindex (or initialise) search in Algolia'
@echo ' make bench.search run search benchmarks'
@echo ' make sync-cloudflare-images sync Cloudflare Images with local DB'
up: export DEBUG = 'knex:query'
up: require create-if-missing.env ../owid-content tmp-downloads/owid_metadata.sql.gz node_modules
@make validate.env
@make check-port-3306
@if tmux has-session -t grapher 2>/dev/null; then \
echo '==> Killing existing tmux session'; \
tmux kill-session -t grapher; \
fi
@echo '==> Building grapher'
yarn lerna run build
@echo '==> Starting dev environment'
@mkdir -p logs
tmux new-session -s grapher \
-n docker 'docker compose -f docker-compose.grapher.yml up' \; \
set remain-on-exit on \; \
set-option -g default-shell $(SCRIPT_SHELL) \; \
new-window -n admin \
'devTools/docker/wait-for-mysql.sh && yarn startAdminDevServer' \; \
set remain-on-exit on \; \
new-window -n vite 'yarn run startSiteFront' \; \
set remain-on-exit on \; \
new-window -n lerna 'yarn startLernaWatcher' \; \
set remain-on-exit on \; \
new-window -n welcome 'devTools/docker/banner.sh; exec $(LOGIN_SHELL)' \; \
bind R respawn-pane -k \; \
bind X kill-pane \; \
bind Q kill-server \; \
set -g mouse on \
|| make down
up.devcontainer: create-if-missing.env.devcontainer tmp-downloads/owid_metadata.sql.gz node_modules
@make validate.env
@make check-port-3306
@echo '==> Building grapher'
yarn lerna run build
@echo '==> Starting dev environment'
@mkdir -p logs
tmux new-session -s grapher \
-n admin \
'devTools/docker/wait-for-mysql.sh && yarn startAdminDevServer' \; \
set remain-on-exit on \; \
new-window -n vite 'yarn run startSiteFront' \; \
set remain-on-exit on \; \
new-window -n lerna 'yarn startLernaWatcher' \; \
set remain-on-exit on \; \
new-window -n welcome 'devTools/docker/banner.sh; exec $(LOGIN_SHELL)' \; \
bind R respawn-pane -k \; \
bind X kill-pane \; \
bind Q kill-server
up.full: export DEBUG = 'knex:query'
up.full: require create-if-missing.env.full ../owid-content tmp-downloads/owid_metadata.sql.gz node_modules
@make validate.env.full
@make check-port-3306
@if tmux has-session -t grapher 2>/dev/null; then \
echo '==> Killing existing tmux session'; \
tmux kill-session -t grapher; \
fi
@echo '==> Building grapher'
yarn lerna run build
@echo '==> Starting dev environment'
tmux new-session -s grapher \
-n docker 'docker compose -f docker-compose.grapher.yml up' \; \
set remain-on-exit on \; \
set-option -g default-shell $(SCRIPT_SHELL) \; \
new-window -n admin \
'devTools/docker/wait-for-mysql.sh && yarn startAdminDevServer' \; \
set remain-on-exit on \; \
new-window -n vite 'yarn run startSiteFront' \; \
set remain-on-exit on \; \
new-window -n lerna 'yarn startLernaWatcher' \; \
set remain-on-exit on \; \
new-window -n functions 'yarn startLocalCloudflareFunctions' \; \
set remain-on-exit on \; \
new-window -n welcome 'devTools/docker/banner.sh; exec $(LOGIN_SHELL)' \; \
bind R respawn-pane -k \; \
bind X kill-pane \; \
bind Q kill-server \; \
set -g mouse on \
|| make down
migrate: node_modules
@echo '==> Running DB migrations'
yarn buildLerna && yarn runDbMigrations
refresh:
@echo '==> Downloading chart data'
./devTools/docker/download-grapher-metadata-mysql.sh
@echo '==> Updating grapher database'
DATA_FOLDER=tmp-downloads ./devTools/docker/refresh-grapher-data.sh
@echo '!!! If you use ETL, wipe indicators from your R2 staging with `rclone delete r2:owid-api-staging/[yourname]/ ' \
'--fast-list --transfers 32 --checkers 32 --verbose`'
refresh.pageviews: node_modules
@echo '==> Refreshing pageviews'
yarn refreshPageviews
sync-images:
@echo 'Task has been deprecated.'
bake-images:
@echo 'Task has been deprecated.'
# Only needed to run once to seed the prod DB with initially
sync-cloudflare-images: node_modules
@echo '==> Syncing images table with Cloudflare Images'
@yarn syncCloudflareImages
refresh.full: refresh refresh.pageviews
@echo '==> Full refresh completed'
down:
@echo '==> Stopping services'
docker compose -f docker-compose.grapher.yml down
require:
@echo '==> Checking your local environment has the necessary commands...'
@which docker >/dev/null 2>&1 || (echo "ERROR: docker compose is required."; exit 1)
@which yarn >/dev/null 2>&1 || (echo "ERROR: yarn is required."; exit 1)
@which tmux >/dev/null 2>&1 || (echo "ERROR: tmux is required."; exit 1)
@which finger >/dev/null 2>&1 || (echo "ERROR: finger is required."; exit 1)
guard-%:
@if [ -z "${${*}}" ]; then echo 'ERROR: .env variable $* not set' && exit 1; fi
create-if-missing.env:
@if test ! -f .env; then \
echo 'Copying .env.example-grapher --> .env'; \
cp .env.example-grapher .env; \
fi
create-if-missing.env.devcontainer:
@if test ! -f .env; then \
echo 'Copying .env.devcontainer --> .env'; \
cp .env.devcontainer .env; \
fi
validate.env:
@echo '==> Validating your .env file for make up'
@grep '=' .env.example-grapher | grep -v optional | sed 's/=.*//' | while read variable; \
do make guard-$$variable 2>/dev/null || exit 1; \
done
@echo '.env file valid for make up'
create-if-missing.env.full:
@if test ! -f .env; then \
echo 'Copying .env.example-full --> .env'; \
sed "s/IMAGE_HOSTING_R2_BUCKET_PATH=.*/IMAGE_HOSTING_R2_BUCKET_PATH=owid-image-upload-staging\/dev-$(USER)/g" <.env.example-full >.env; \
fi
validate.env.full:
@echo '==> Validating your .env file for make up.full'
@grep '=' .env.example-full | grep -v optional | sed 's/=.*//' | while read variable; \
do make guard-$$variable 2>/dev/null || exit 1; \
done
@echo '.env file valid for make up.full'
check-port-3306:
@echo "==> Checking port"
@if [ "${GRAPHER_DB_PORT}" = "3306" ]; then \
echo "Your database port is set to 3306.\
\nThis will likely conflict with any pre-existing MySQL instances you have running.\
\nWe recommend using a different port (like 3307)";\
fi
tmp-downloads/owid_metadata.sql.gz:
@echo '==> Downloading metadata'
./devTools/docker/download-grapher-metadata-mysql.sh
test: node_modules
@echo '==> Linting'
yarn run eslint
yarn lerna run build
yarn lerna run buildTests
@echo '==> Checking formatting'
yarn testPrettierAll
@echo '==> Running tests'
yarn run jest
dbtest: node_modules
@echo '==> Building'
yarn buildTsc
@echo '==> Running db test script'
./db/tests/run-db-tests.sh
lint: node_modules
@echo '==> Linting'
yarn run eslint
check-formatting: node_modules
@echo '==> Checking formatting'
yarn testPrettierAll
format: node_modules
@echo '==> Fixing formatting'
yarn fixPrettierAll
unittest: node_modules
@echo '==> Running tests'
yarn run jest --all
../owid-grapher-svgs:
cd .. && git clone [email protected]:owid/owid-grapher-svgs
svgtest: ../owid-grapher-svgs
@echo '==> Comparing against reference SVGs'
@# get ../owid-grapher-svgs reliably to a base state at origin/master
cd ../owid-grapher-svgs && git fetch && git checkout -f master && git reset --hard origin/master && git clean -fd
@# generate a full new set of svgs and create an HTML report if there are differences
node --enable-source-maps itsJustJavascript/devTools/svgTester/verify-graphs.js \
|| node --enable-source-maps itsJustJavascript/devTools/svgTester/create-compare-view.js
../owid-content:
@echo '==> Cloning owid-content to ../owid-content'
cd .. && git clone [email protected]:owid/owid-content
node_modules: package.json yarn.lock yarn.config.cjs
@echo '==> Installing packages'
yarn install
touch -m $@
itsJustJavascript: node_modules
@echo '==> Compiling TS'
yarn lerna run build
yarn run tsc -b
touch $@
update.chart-entities: itsJustJavascript
@echo '==> Updating chart entities table'
node --enable-source-maps itsJustJavascript/baker/updateChartEntities.js --all
reindex: itsJustJavascript
@echo '==> Reindexing search in Algolia'
@echo '--- Running configureAlgolia...'
node --enable-source-maps itsJustJavascript/baker/algolia/configureAlgolia.js
@echo '--- Running indexPagesToAlgolia...'
node --enable-source-maps itsJustJavascript/baker/algolia/indexPagesToAlgolia.js
@echo '--- Running indexChartsToAlgolia...'
node --enable-source-maps itsJustJavascript/baker/algolia/indexChartsToAlgolia.js
@echo '--- Running indexPostsToAlgolia...'
node --enable-source-maps itsJustJavascript/baker/algolia/indexExplorerViewsToAlgolia.js
@echo '--- Running indexExplorerViewsAndChartsToAlgolia...'
node --enable-source-maps itsJustJavascript/baker/algolia/indexExplorerViewsAndChartsToAlgolia.js
delete-algolia-index: itsJustJavascript
@echo '==> Deleting Algolia index'
node --enable-source-maps itsJustJavascript/baker/algolia/deleteAlgoliaIndex.js
bench.search: itsJustJavascript
@echo '==> Running search benchmarks'
@node --enable-source-maps itsJustJavascript/site/search/evaluateSearch.js
local-bake: itsJustJavascript
@echo '==> Baking site'
yarn buildVite
yarn buildLocalBake
clean:
rm -rf node_modules itsJustJavascript