forked from xarantolus/filtrite
-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.go
124 lines (103 loc) · 3.04 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
package main
import (
"fmt"
"log"
"os"
"path/filepath"
"strings"
"unicode"
"xarantolus/filtrite/util"
)
const (
tmpDir = "tmp"
listDir = "lists"
distDir = "dist"
logDir = "logs"
// This is the "kMaxBodySize" from https://github.com/bromite/bromite/blob/6f40f8341ab3fbcab458c10fe7b6bbcb8f881404/build/patches/Bromite-subresource-adblocker.patch#L1160-L1161
bromiteMaxFilterSize = 1024 * 1024 * 20
)
// generateFilterList generates a filter list from the listTextFile
func generateFilterList(listTextFile string) (err error) {
var listName = strings.Map(
func(r rune) rune {
if unicode.IsSpace(r) {
return '_'
}
return r
},
strings.TrimSuffix(filepath.Base(listTextFile), ".txt"),
)
fmt.Printf("::group::List: %s\n", listName)
defer fmt.Println("::endgroup::")
var (
outputFile = filepath.Join(distDir, listName+".dat")
logFile = filepath.Join(logDir, listName+".log")
)
// Load all URLs
filterListURLs, err := util.ReadListFile(listTextFile)
if err != nil {
return fmt.Errorf("reading list file: %w", err)
}
// Create temporary directory and make sure we remove it afterwards
err = os.MkdirAll(tmpDir, os.ModePerm)
if err != nil {
return fmt.Errorf("creating temp directory for filter lists: %w", err)
}
defer os.RemoveAll(tmpDir)
log.Printf("Downloading %d filter lists...\n", len(filterListURLs))
// Actually download lists
paths, err := util.DownloadURLs(filterListURLs, tmpDir)
if err != nil {
return fmt.Errorf("downloading filter lists: %w", err)
}
log.Printf("Got %d/%d\n", len(paths), len(filterListURLs))
if len(paths) == 0 {
return fmt.Errorf("all lists failed to download")
}
// make sure dir exists
err = os.MkdirAll(filepath.Dir(outputFile), 0664)
if err != nil {
return fmt.Errorf("creating distribution directory: %w", err)
}
log.Println("Converting ruleset...")
err = util.GenerateDistributableList(paths, outputFile, logFile)
if err != nil {
return fmt.Errorf("generating distributable list: %w", err)
}
// Check if output file is larger than 10mb
fileInfo, err := os.Stat(outputFile)
if err != nil {
return fmt.Errorf("getting filter output file info: %w", err)
}
if fileInfo.Size() > bromiteMaxFilterSize {
return fmt.Errorf("filter list is too large for Bromite (%d bytes > %d bytes)", fileInfo.Size(), bromiteMaxFilterSize)
}
err = util.AppendReleaseList(listTextFile, len(paths), len(filterListURLs))
if err != nil {
return fmt.Errorf("generating release list: %w", err)
}
return nil
}
func main() {
err := filepath.Walk(listDir, func(path string, d os.FileInfo, err error) error {
if err != nil {
log.Printf("Walk: Error: %s\n", err.Error())
return nil
}
if d.IsDir() {
return nil
}
if !strings.HasSuffix(path, ".txt") {
log.Printf("File %q was not processed because it does not end with \".txt\"\n", path)
return nil
}
err = generateFilterList(path)
if err != nil {
log.Printf("Error while generating filter for %q: %s\n", path, err.Error())
}
return err
})
if err != nil {
panic("error while walking: " + err.Error())
}
}