-
Notifications
You must be signed in to change notification settings - Fork 26
147 lines (147 loc) · 4.83 KB
/
createUploadBlocklistFilter.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
name: MakeTrie
on:
# docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows#schedule
schedule:
# at 21:53 on 1st, 8th, 16th, 24th of every month
- cron: '53 21 1,8,16,24 * *'
workflow_dispatch:
inputs:
publish:
description: 'publish blocklists?'
required: true
default: 'test'
type: choice
options:
# "yes" / "no" convert to bools, hence use "yeah" / "nope" instead.
# bools are bools when accessed with input-context, but are
# stringified when accessed with github.events context. And so:
# for the sake of everyone's sanity, stick to non-boolean inputs.
# ref: docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#providing-inputs
- yeah
- nope
- test
env:
INDIR: "blocklistfiles"
OUTDIR: "result"
S3DIR: "blocklists"
S3_TEST_DIR: "testblocklists"
BLCONFIG: "config.json"
PREFER_S3_OVER_R2: "false"
jobs:
version:
runs-on: ubuntu-latest
timeout-minutes: 1
# docs.github.com/en/actions/learn-github-actions/contexts#example-usage-of-the-needs-context
outputs:
epoch: ${{ steps.v.outputs.epoch }}
steps:
- name: Gen version
id: v
run: |
# stackoverflow.com/a/69400542
v=$(date --utc +"%s.%3N")
echo "epoch=${v}" >> $GITHUB_OUTPUT
echo "::notice::Version ${v} / ${SCHED}"
shell: bash
env:
SCHED: ${{ github.event.schedule }}
make:
needs: version
strategy:
matrix:
codec: ["u8", "u6"]
runs-on: ubuntu-latest
timeout-minutes: 100
steps:
- name: 📃 Checkout repo
uses: actions/[email protected]
- name: 🐉 Setup python
uses: actions/[email protected]
with:
python-version: '3.x'
- name: ⭕ Setup node
uses: actions/[email protected]
with:
node-version: '19'
- name: ⏳ Make swap
run: |
free -hm
df -h
# make swap: community.fly.io/t/6782/10
sudo fallocate -l 16G /tmp/swapfile
# unix.stackexchange.com/a/297155
sudo chown root:root /tmp/swapfile
sudo mkswap /tmp/swapfile
# stackoverflow.com/a/550808
sudo cat /proc/sys/vm/swappiness
echo "20" | sudo tee /proc/sys/vm/swappiness > /dev/null
# github.com/pierotofy/set-swap-space/blob/master/action.yml
sudo swapon /tmp/swapfile
free -hm
df -h
shell: bash
- name: 🎨 Setup
id: setup
run: |
# install deps
python -m pip install aiohttp
npm install
# determine config
# all output vars are strings
# docs.github.com/en/actions/learn-github-actions/contexts#steps-context
echo "upload=y" >> $GITHUB_OUTPUT
if [ "$PUBLISH_BLOCKLISTS" = "test" ]; then
echo "S3DIR=$S3_TEST_DIR" >> $GITHUB_ENV
elif [ "$PUBLISH_BLOCKLISTS" = "nope" ]; then
echo "S3DIR=" >> $GITHUB_ENV
# do not upload
echo "upload=n" >> $GITHUB_OUTPUT
fi
echo "UNIX_EPOCH_SEC=${V}" >> $GITHUB_ENV
echo "::notice::Building ${GIT_REF}/${COMMIT_SHA} @ ${V}"
shell: bash
env:
COMMIT_SHA: ${{ github.sha }}
GIT_REF: ${{ github.ref }}
V: ${{ needs.version.outputs.epoch }}
PUBLISH_BLOCKLISTS: ${{ inputs.publish }}
- name: 📥 Download blocklists
run: |
python download.py
env:
BLCONFIG: ${{ env.BLCONFIG }}
- name: 🌲 Make compact trie
run: |
mkdir -p "$OUTDIR"
# 32768/1024 => 32G RAM
node --max-old-space-size=32768 --expose-gc ./src/build.js
env:
BLCONFIG: ${{ env.BLCONFIG }}
OUTDIR: ${{ env.OUTDIR }}
INDIR: ${{ env.INDIR }}
UNIX_EPOCH_SEC: ${{ env.UNIX_EPOCH_SEC }}
CODEC: ${{ matrix.codec }}
- name: 🪄 Show artifacts
run: |
# note: split now happens in serverless-dns/trie:trie.js
# creates td00.txt, td01.txt, ... , td98.txt, td99.txt, td100.txt, ...
# cd "$OUTDIR" && split --bytes=30M -d --additional-suffix=.txt td.txt td
cd "$OUTDIR" && ls -lhtr
env:
OUTDIR: ${{ env.OUTDIR }}
- name: 🏆 Upload?
# stackoverflow.com/a/68317611
# github.com/actions/runner/issues/866
if: ${{ steps.setup.outputs.upload == 'y' }}
run: |
node ./src/upload.js
env:
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
PREFER_S3_OVER_R2: ${{ env.PREFER_S3_OVER_R2 }}
UNIX_EPOCH_SEC: ${{ env.UNIX_EPOCH_SEC }}
CODEC: ${{ matrix.codec }}
S3DIR: ${{ env.S3DIR }}
OUTDIR: ${{ env.OUTDIR }}