diff --git a/README.md b/README.md new file mode 100644 index 0000000..4c91de3 --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +# chia-garden + +chia-garden is a utility to handle the migration of plot files for the Chia +blockchain. + +It can be used to transfer plot files from dedicated plotting nodes to available +harvester nodes, optimizing for even file distribution and network conjestion. +It can work across a set of many plotters and many harvesters by communicating +over a dedicated message bus utilizing NATS. + +### How does it handle distribution? + +When a new plot file is found on a harvester, it will publish a message to NATS +notifying harvesters of the file being available and noting its size. + +All the available harvesters will receive this message and generate a response +with a URL for the plotter to send the file to the harvester over HTTP. + +Whichever harvester responds first to the plotter is the one which will have the +file sent to it. + +Since the plotter only takes the first response, even distribution can be +naturally achieved by the harvesters responding fast or intentionally responding +slow. For instance: + +* If the current harvester is already transferring a few plots, its network + traffic is going to be pegged, so it would be less ideal to receive it. It can + delay responding by 10ms for each active transfer it already has. +* If the current harvester's disk is getting closer to filling, it might be less + ideal to plot compared to a harvester with a completely empty disk. So it can + add a 5-10ms delay to when it responds. +* If the harvester has no transfers and plenty of storage, no delay, respond + immediately! + +### Optimizations Applied + +A number of optimizations are instrumented in how the transfers are performed. + +1. The harvester nodes will prioritize disks with more free space. With this, + disks will typically fill at an even rate rather than one at a time. +1. It will only allow one plot to be written to a disk at a time. This is to + avoid fragmentation on the disk caused by concurrent plots being written. +1. Harvesters with more transfers will be deprioritized. This is to limit + network conjestion in cases where multiple plotters are are transferring at + the same time. \ No newline at end of file diff --git a/cmd/harvester/main.go b/cmd/harvester/main.go new file mode 100644 index 0000000..0fa24b0 --- /dev/null +++ b/cmd/harvester/main.go @@ -0,0 +1,49 @@ +package main + +import ( + "flag" + "log" + + "github.com/krobertson/chia-garden/pkg/rpc" + "github.com/krobertson/chia-garden/pkg/utils" + + "github.com/nats-io/nats.go" + + _ "net/http/pprof" +) + +var ( + maxTransfers int64 = 5 +) + +func main() { + var plotPaths utils.ArrayFlags + + natsUrl := flag.String("nats", nats.DefaultURL, "NATS connection string") + flag.Var(&plotPaths, "plot", "Plots directories") + flag.Int64Var(&maxTransfers, "max-transfers", 5, "max concurrent transfers") + flag.Parse() + + log.Print("Starting harvester-client...") + + conn, err := nats.Connect(*natsUrl, nats.MaxReconnects(-1)) + if err != nil { + log.Fatal("Failed to connect to NATS: ", err) + } + defer conn.Close() + + server, err := newHarvester(plotPaths) + if err != nil { + log.Fatal("Failed to initialize harvester: ", err) + } + + // initialize the rpc + _, err = rpc.NewNatsHarvesterListener(conn, server) + if err != nil { + log.Fatal("Failed to initialize NATS listener: ", err) + } + + // Block main goroutine forever. + log.Print("Ready") + <-make(chan struct{}) +} diff --git a/cmd/harvester/plotpath.go b/cmd/harvester/plotpath.go new file mode 100644 index 0000000..8e64f8e --- /dev/null +++ b/cmd/harvester/plotpath.go @@ -0,0 +1,27 @@ +package main + +import ( + "sync" + "sync/atomic" + + "golang.org/x/sys/unix" +) + +type plotPath struct { + path string + busy atomic.Bool + freeSpace uint64 + totalSpace uint64 + mutex sync.Mutex +} + +// updateFreeSpace will get the filesystem stats and update the free and total +// space on the plotPath. This primarily should be done with the plotPath mutex +// locked. +func (p *plotPath) updateFreeSpace() { + var stat unix.Statfs_t + unix.Statfs(p.path, &stat) + + p.freeSpace = stat.Bavail * uint64(stat.Bsize) + p.totalSpace = stat.Blocks * uint64(stat.Bsize) +} diff --git a/cmd/harvester/server.go b/cmd/harvester/server.go new file mode 100644 index 0000000..a61bfed --- /dev/null +++ b/cmd/harvester/server.go @@ -0,0 +1,245 @@ +package main + +import ( + "cmp" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "slices" + "sync" + "sync/atomic" + "time" + + "github.com/krobertson/chia-garden/pkg/types" + "github.com/krobertson/chia-garden/pkg/utils" + + "github.com/dustin/go-humanize" +) + +const ( + taintTransfers = 20 * time.Millisecond + taintFreeSpace = 20 * time.Millisecond +) + +type harvester struct { + plots map[string]*plotPath + sortedPlots []*plotPath + sortMutex sync.Mutex + hostPort string + transfers atomic.Int64 +} + +// newHarvester will create a the harvester server process and validate all of +// the provided plot paths. It will return an error if any of the paths do not +// exist, or are not a directory. +func newHarvester(paths []string) (*harvester, error) { + h := &harvester{ + plots: make(map[string]*plotPath), + sortedPlots: make([]*plotPath, len(paths)), + hostPort: fmt.Sprintf("%s:3434", utils.GetHostIP().String()), + } + + // ensure we have at least one + if len(paths) == 0 { + return nil, fmt.Errorf("at least one plot path must be specified") + } + + // validate the plots exist and add them in + for i, p := range paths { + p, err := filepath.Abs(p) + if err != nil { + return nil, fmt.Errorf("path %s failed expansion: %v", p, err) + } + + fi, err := os.Stat(p) + if err != nil { + return nil, fmt.Errorf("path %s failed validation: %v", p, err) + } + + if !fi.IsDir() { + return nil, fmt.Errorf("path %s is not a directory", p) + } + + pp := &plotPath{path: p} + pp.updateFreeSpace() + h.plots[p] = pp + h.sortedPlots[i] = pp + } + + // sort the paths + h.sortPaths() + + // FIXME ideally handle graceful shutdown of existing transfers + http.HandleFunc("/", h.httpHandler) + go http.ListenAndServe(":3434", nil) + + return h, nil +} + +// PlotReady processes a request from a plotter to transfer a plot to the +// harvester. It will generate a response, but then momentarily sleep as a +// slight taint to allow the most ideal system to originally respond the +// fastest. +func (h *harvester) PlotReady(req *types.PlotRequest) (*types.PlotResponse, error) { + // pick a plot. This should return the one with the most free space that + // isn't busy. + plot := h.pickPlot() + if plot == nil { + return nil, fmt.Errorf("no paths available") + } + + // check if we have enough free space + if plot.freeSpace <= req.Size { + return nil, nil + } + + // generate response + resp := &types.PlotResponse{ + Url: fmt.Sprintf("http://%s%s", h.hostPort, filepath.Join(plot.path, req.Name)), + } + + // generate and handle the taint + d := h.generateTaint(plot) + time.Sleep(d) + return resp, nil +} + +// httpHandler faciliates the transfer of plot files from the plotters to the +// harvesters. It encapculates a single request and is ran within its own +// goroutine. It will respond with a 201 on success and a relevant error code on +// failure. A failure should trigger the plotter to re-request storage. +func (h *harvester) httpHandler(w http.ResponseWriter, req *http.Request) { + defer req.Body.Close() + + // get the plot path and ensure it exists + base := filepath.Dir(req.URL.Path) + plotPath, exists := h.plots[base] + if !exists { + log.Printf("Request to store in %s, but does not exist", base) + w.WriteHeader(404) + return + } + + // check if we're maxed on concurrent transfers + if h.transfers.Load() >= maxTransfers { + log.Printf("Request to store in %s, but at max transfers", base) + w.WriteHeader(503) + return + } + + // make sure the disk isn't already being written to. this helps to avoid + // file fragmentation + if plotPath.busy.Load() { + log.Printf("Request to store %s, but already trasnferring", req.URL.Path) + w.WriteHeader(503) + return + } + + // make sure we have the content length + if req.ContentLength == 0 { + w.WriteHeader(411) + return + } + + // lock the file path + plotPath.mutex.Lock() + defer plotPath.mutex.Unlock() + plotPath.busy.Store(true) + defer plotPath.busy.Store(false) + h.transfers.Add(1) + defer h.transfers.Add(-1) + + // check if we have enough free space + if plotPath.freeSpace <= uint64(req.ContentLength) { + log.Printf("Request to store %s, but not enough space (%s / %s)", + req.URL.Path, humanize.Bytes(uint64(req.ContentLength)), humanize.Bytes(plotPath.freeSpace)) + w.WriteHeader(413) + } + + // validate the file doesn't already exist, as a safeguard + fi, _ := os.Stat(req.URL.Path) + if fi != nil { + log.Printf("File at %s already exists", req.URL.Path) + w.WriteHeader(409) + return + } + + // open the file and transfer + f, err := os.Create(req.URL.Path) + if err != nil { + log.Printf("Failed to open file at %s: %v", req.URL.Path, err) + w.WriteHeader(500) + return + } + defer f.Close() + + // perform the copy + start := time.Now() + bytes, err := io.Copy(f, req.Body) + if err != nil { + log.Printf("Failure while writing plot %s: %v", req.URL.Path, err) + f.Close() + os.Remove(req.URL.Path) + w.WriteHeader(500) + return + } + + // update free space + plotPath.updateFreeSpace() + h.sortPaths() + + // log successful and some metrics + seconds := time.Since(start).Seconds() + log.Printf("Successfully stored %s (%s, %f secs, %s/sec)", + req.URL.Path, humanize.IBytes(uint64(bytes)), seconds, humanize.Bytes(uint64(float64(bytes)/seconds))) + w.WriteHeader(201) +} + +// sortPaths will update the order of the plotPaths inside the harvester's +// sortedPaths slice. This should be done after every file transfer when the +// free space is updated. +func (h *harvester) sortPaths() { + h.sortMutex.Lock() + defer h.sortMutex.Unlock() + + slices.SortStableFunc(h.sortedPlots, func(a, b *plotPath) int { + return cmp.Compare(a.freeSpace, b.freeSpace) + }) +} + +// pickPlot will return which plot path would be most ideal for the current +// request. It will order the one with the most free space that doesn't already +// have an active transfer. +func (h *harvester) pickPlot() *plotPath { + h.sortMutex.Lock() + defer h.sortMutex.Unlock() + + for _, v := range h.sortedPlots { + if v.busy.Load() { + continue + } + return v + } + return nil +} + +// generateTaint will calculate how long to delay the response based on current +// system pressure. This can be used to organically load balance in a cluster, +// allowing more preferencial hosts to respond faster. +func (h *harvester) generateTaint(plot *plotPath) time.Duration { + d := time.Duration(0) + + // apply per current transfer going on. this helps prefer harvesters with + // less busy networks + d += time.Duration(h.transfers.Load()) * taintTransfers + + // apply for ratio of free disk space. this prefers harvesters with emptier + // disks + percent := 100 * plot.freeSpace / plot.totalSpace + d += time.Duration(percent) * taintFreeSpace / 1000 + + return d +} diff --git a/cmd/plotter/handler.go b/cmd/plotter/handler.go new file mode 100644 index 0000000..a0665c2 --- /dev/null +++ b/cmd/plotter/handler.go @@ -0,0 +1,96 @@ +package main + +import ( + "log" + "net/http" + "os" + "path/filepath" + "sync" + "time" + + "github.com/dustin/go-humanize" + "github.com/krobertson/chia-garden/pkg/rpc" + "github.com/krobertson/chia-garden/pkg/types" +) + +var ( + failedPlots = []string{} + failedPlotMutex = sync.Mutex{} +) + +func handlePlot(client *rpc.NatsPlotterClient, plot string) { + // gather info + fi, err := os.Stat(plot) + if err != nil { + log.Print("Failed to stat plot file", plot, err) + return + } + req := &types.PlotRequest{ + Name: filepath.Base(plot), + Size: uint64(fi.Size()), + } + + for i := 0; i < 10; i++ { + resp, err := client.PlotReady(req) + if err != nil { + log.Print("Received error on plot ready request", err) + time.Sleep(time.Minute) + continue + } + + // if we did not get a response, sleep and try again + if resp == nil { + log.Print("Received no response") + time.Sleep(time.Minute) + continue + } + + // open the file + f, err := os.Open(plot) + if err != nil { + log.Print("Failed to open plot file, bailing", err) + return + } + + // if we got a response, dispatch the transfer + httpreq, err := http.NewRequest("POST", resp.Url, f) + if err != nil { + log.Print("Failed to open http request", err) + f.Close() + time.Sleep(time.Minute) + continue + } + httpreq.ContentLength = int64(req.Size) + + start := time.Now() + httpresp, err := http.DefaultTransport.RoundTrip(httpreq) + if err != nil { + log.Print("HTTP transfer failed", err) + f.Close() + time.Sleep(time.Minute) + continue + } + + // FIXME better handle various error codes + if httpresp.StatusCode != 201 { + log.Print("Received bad response", httpresp.StatusCode, httpresp.Status) + f.Close() + time.Sleep(time.Minute) + continue + } + + // all done now + f.Close() + seconds := time.Since(start).Seconds() + log.Printf("Finished transfering plot %s (%s, %f secs, %s/sec)", + plot, humanize.IBytes(uint64(req.Size)), seconds, humanize.Bytes(uint64(float64(req.Size)/seconds))) + os.Remove(plot) + return + } + + // Too many retries, log and continue + log.Printf("Timed out transferring plot file %s, will retry later or on next restart", plot) + failedPlotMutex.Lock() + failedPlots = append(failedPlots, plot) + failedPlotMutex.Unlock() +} diff --git a/cmd/plotter/main.go b/cmd/plotter/main.go new file mode 100644 index 0000000..47b0629 --- /dev/null +++ b/cmd/plotter/main.go @@ -0,0 +1,79 @@ +package main + +import ( + "flag" + "log" + "strings" + + "github.com/krobertson/chia-garden/pkg/rpc" + + "github.com/fsnotify/fsnotify" + "github.com/nats-io/nats.go" +) + +func main() { + natsUrl := flag.String("nats", nats.DefaultURL, "NATS connection string") + plotDir := flag.String("plot", "", "Plots directory") + flag.Parse() + + log.Print("Starting plotter-client...") + + conn, err := nats.Connect(*natsUrl, nats.MaxReconnects(-1)) + if err != nil { + log.Fatal("Failed to connect to NATS: ", err) + } + defer conn.Close() + + client := rpc.NewNatsPlotterClient(conn) + + // begin watching the plots directory + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal("Failed to initialize watcher", err) + } + defer watcher.Close() + + // watch loop + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + log.Print("Leaving watch loop") + return + } + + // filter to create events + if event.Op != fsnotify.Create { + continue + } + + // filter to only the *.plot files + if !strings.HasSuffix(event.Name, ".plot") { + continue + } + + // found new plot + log.Printf("New plot created %s", event.Name) + go handlePlot(client, event.Name) + + case err, ok := <-watcher.Errors: + if !ok { + log.Print("Leaving watch loop") + return + } + log.Println("error:", err) + } + } + }() + + // Add the plots path to the watcher + err = watcher.Add(*plotDir) + if err != nil { + log.Fatal("Failed to watch plots path", err) + } + + // Block main goroutine forever. + log.Print("Ready") + <-make(chan struct{}) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..245963d --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module github.com/krobertson/chia-garden + +go 1.21.7 + +require ( + github.com/dustin/go-humanize v1.0.1 + github.com/fsnotify/fsnotify v1.7.0 + github.com/nats-io/nats.go v1.34.0 + golang.org/x/sys v0.18.0 +) + +require ( + github.com/klauspost/compress v1.17.2 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.18.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e60fb3c --- /dev/null +++ b/go.sum @@ -0,0 +1,16 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/nats-io/nats.go v1.34.0 h1:fnxnPCNiwIG5w08rlMcEKTUw4AV/nKyGCOJE8TdhSPk= +github.com/nats-io/nats.go v1.34.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/pkg/rpc/client_plotter.go b/pkg/rpc/client_plotter.go new file mode 100644 index 0000000..fb72f19 --- /dev/null +++ b/pkg/rpc/client_plotter.go @@ -0,0 +1,24 @@ +package rpc + +import ( + "github.com/krobertson/chia-garden/pkg/types" + "github.com/nats-io/nats.go" +) + +type NatsPlotterClient struct { + client *nats.Conn +} + +func NewNatsPlotterClient(conn *nats.Conn) *NatsPlotterClient { + return &NatsPlotterClient{ + client: conn, + } +} + +func (d *NatsPlotterClient) PlotReady(plot *types.PlotRequest) (*types.PlotResponse, error) { + var resp *types.PlotResponse + if err := request(d.client, subjPlotReady, plot, &resp); err != nil { + return nil, err + } + return resp, nil +} diff --git a/pkg/rpc/listener_harvester.go b/pkg/rpc/listener_harvester.go new file mode 100644 index 0000000..22837bc --- /dev/null +++ b/pkg/rpc/listener_harvester.go @@ -0,0 +1,79 @@ +package rpc + +import ( + "encoding/json" + "log" + + "github.com/krobertson/chia-garden/pkg/types" + "github.com/nats-io/nats.go" +) + +type NatsHarvesterListener struct { + client *nats.Conn + handler Harvester +} + +func NewNatsHarvesterListener(client *nats.Conn, handler Harvester) (*NatsHarvesterListener, error) { + w := &NatsHarvesterListener{ + client: client, + handler: handler, + } + return w, w.RegisterHandlers() +} + +func (w *NatsHarvesterListener) RegisterHandlers() error { + _, err := w.client.Subscribe(subjPlotReady, w.handlerPlot) + if err != nil { + return err + } + + return w.client.Flush() +} + +func (d *NatsHarvesterListener) handlerPlot(msg *nats.Msg) { + var req *types.PlotRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + log.Println("Failed to unmarshal rig") + return + } + + resp, err := d.handler.PlotReady(req) + if resp != nil || err != nil { + d.respond(msg, resp, err) + } +} + +func (d *NatsHarvesterListener) respond(msg *nats.Msg, v interface{}, err error) { + // if there is no reply subject, just bail, we have no option + if msg.Reply == "" { + if err != nil { + log.Printf("Error returned from call: %v", err) + } + return + } + + resp := &natsResponse{} + if err != nil { + s := err.Error() + resp.Error = &s + } else { + data, err := json.Marshal(v) + if err != nil { + s := err.Error() + resp.Error = &s + } else { + resp.Result = data + } + } + + data, err := json.Marshal(resp) + if err != nil { + log.Printf("Failed to generate reply message: %v", err) + return + } + + if err = d.client.Publish(msg.Reply, data); err != nil { + log.Printf("Failed to publish reply message: %v", err) + return + } +} diff --git a/pkg/rpc/rpc.go b/pkg/rpc/rpc.go new file mode 100644 index 0000000..b568eab --- /dev/null +++ b/pkg/rpc/rpc.go @@ -0,0 +1,9 @@ +package rpc + +import ( + "github.com/krobertson/chia-garden/pkg/types" +) + +type Harvester interface { + PlotReady(*types.PlotRequest) (*types.PlotResponse, error) +} diff --git a/pkg/rpc/utils.go b/pkg/rpc/utils.go new file mode 100644 index 0000000..bb6637f --- /dev/null +++ b/pkg/rpc/utils.go @@ -0,0 +1,45 @@ +package rpc + +import ( + "encoding/json" + "errors" + "time" + + "github.com/nats-io/nats.go" +) + +const ( + subjPlotReady = "b4s.plot.ready" + subjPlotLocate = "b4s.plot.locate" +) + +type natsResponse struct { + Result json.RawMessage `json:"result"` + Error *string `json:"error"` +} + +func request(client *nats.Conn, subj string, in interface{}, out interface{}) error { + data, err := json.Marshal(in) + if err != nil { + return err + } + + msg, err := client.Request(subj, data, time.Second*5) + if err != nil { + return err + } + + var resp *natsResponse + if err := json.Unmarshal(msg.Data, &resp); err != nil { + return err + } + + if resp.Error != nil { + return errors.New(*resp.Error) + } + + if out != nil { + return json.Unmarshal(resp.Result, &out) + } + return nil +} diff --git a/pkg/types/types.go b/pkg/types/types.go new file mode 100644 index 0000000..a78e022 --- /dev/null +++ b/pkg/types/types.go @@ -0,0 +1,20 @@ +package types + +type PlotRequest struct { + Name string `json:"name"` + Size uint64 `json:"size"` +} + +type PlotResponse struct { + Url string `json:"url"` +} + +type PlotLocateRequest struct { + Name string `json:"name"` +} + +type PlotLocateResponse struct { + Name string `json:"name"` + Size uint64 `json:"size"` + Sha1 string `json:"sha1sum"` +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go new file mode 100644 index 0000000..91be325 --- /dev/null +++ b/pkg/utils/utils.go @@ -0,0 +1,35 @@ +package utils + +import ( + "net" + "strings" +) + +// GetHostIP returns the net.IP of the default network interface on the machine. +func GetHostIP() net.IP { + // Attempt a UDP connection to a dummy IP, which will cause the local end + // of the connection to be the interface with the default route + addr := &net.UDPAddr{IP: net.IP{1, 2, 3, 4}, Port: 1} + conn, err := net.DialUDP("udp", nil, addr) + if err != nil { + return nil + } + defer conn.Close() + + return conn.LocalAddr().(*net.UDPAddr).IP +} + +// ArrayFlags can be used with flags.Var to specify the a command line argument +// multiple timmes. +type ArrayFlags []string + +// String returns a basic string concationation of all values. +func (i *ArrayFlags) String() string { + return strings.Join(*i, ", ") +} + +// Set is used to append a new value to the array by flags.Var. +func (i *ArrayFlags) Set(value string) error { + *i = append(*i, value) + return nil +}