From c792a9d14c65f746794829995e0bd171a7619128 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Sun, 26 Sep 2021 11:21:35 +0100 Subject: [PATCH] kubernetes: cache downloaded dependencies Signed-off-by: Abiola Ibrahim --- config/config.go | 7 +- .../container/kubernetes/containerd.go | 9 +-- .../container/kubernetes/kubernetes.go | 4 +- environment/container/kubernetes/minikube.go | 11 +-- environment/vm/lima/lima.go | 5 +- util/downloader/download.go | 68 +++++++++++++++++++ util/template.go | 11 --- util/yamlutil/yaml.go | 17 +++++ 8 files changed, 106 insertions(+), 26 deletions(-) create mode 100644 util/downloader/download.go create mode 100644 util/yamlutil/yaml.go diff --git a/config/config.go b/config/config.go index b3e9bbd6..bf453d45 100644 --- a/config/config.go +++ b/config/config.go @@ -2,7 +2,7 @@ package config import ( "fmt" - "github.com/abiosoft/colima/util" + "github.com/abiosoft/colima/util/yamlutil" "gopkg.in/yaml.v3" "log" "net" @@ -32,6 +32,9 @@ func SSHPort() int { return sshPort } // Dir returns the configuration directory. func Dir() string { return configDir } +// CacheDir returns the cache directory. +func CacheDir() string { return cacheDir } + // LogFile returns the path the command log output. func LogFile() string { return filepath.Join(cacheDir, "out.log") @@ -71,7 +74,7 @@ func configFile() string { // Save saves the config. func Save(c Config) error { - return util.WriteYAML(c, configFile()) + return yamlutil.WriteYAML(c, configFile()) } // Load loads the config. diff --git a/environment/container/kubernetes/containerd.go b/environment/container/kubernetes/containerd.go index cdeeb305..7ebd3ffb 100644 --- a/environment/container/kubernetes/containerd.go +++ b/environment/container/kubernetes/containerd.go @@ -4,14 +4,15 @@ import ( "fmt" "github.com/abiosoft/colima/cli" "github.com/abiosoft/colima/environment" + "github.com/abiosoft/colima/util/downloader" "os" "path/filepath" "runtime" ) -func installContainerdDeps(guest environment.GuestActions, r *cli.ActiveCommandChain) { +func installContainerdDeps(host environment.HostActions, guest environment.GuestActions, r *cli.ActiveCommandChain) { // install crictl - installCrictl(guest, r) + installCrictl(host, guest, r) // minikube with containerd still needs docker :( https://github.com/kubernetes/minikube/issues/10908 // the good news is we can spoof it. @@ -47,7 +48,7 @@ func installContainerdDeps(guest environment.GuestActions, r *cli.ActiveCommandC }) } -func installCrictl(guest environment.GuestActions, r *cli.ActiveCommandChain) { +func installCrictl(host environment.Host, guest environment.GuestActions, r *cli.ActiveCommandChain) { // TODO figure a way to keep up to date. version := "v1.22.0" downloadPath := "/tmp/crictl.tar.gz" @@ -63,7 +64,7 @@ func installCrictl(guest environment.GuestActions, r *cli.ActiveCommandChain) { }) r.Add(func() error { - return guest.RunInteractive("curl", "-L", "-#", "-o", downloadPath, url) + return downloader.Download(host, guest, url, downloadPath) }) r.Add(func() error { return guest.Run("sudo", "tar", "xvfz", downloadPath, "-C", "/usr/local/bin") diff --git a/environment/container/kubernetes/kubernetes.go b/environment/container/kubernetes/kubernetes.go index aedfe650..cad3c0e2 100644 --- a/environment/container/kubernetes/kubernetes.go +++ b/environment/container/kubernetes/kubernetes.go @@ -74,7 +74,7 @@ func (c *kubernetesRuntime) Provision() error { case containerd.Name: r.Stage("installing " + containerRuntime + " dependencies") - installContainerdDeps(c.guest, r) + installContainerdDeps(c.host, c.guest, r) case docker.Name: // no known dependencies for now @@ -82,7 +82,7 @@ func (c *kubernetesRuntime) Provision() error { // minikube r.Stage("installing minikube") - installMinikube(c.guest, r, c.kubernetesVersion()) + installMinikube(c.host, c.guest, r, c.kubernetesVersion()) // adding to chain to ensure it executes after successful provision r.Add(func() error { diff --git a/environment/container/kubernetes/minikube.go b/environment/container/kubernetes/minikube.go index 0aa5d074..a7fbcb68 100644 --- a/environment/container/kubernetes/minikube.go +++ b/environment/container/kubernetes/minikube.go @@ -3,28 +3,29 @@ package kubernetes import ( "github.com/abiosoft/colima/cli" "github.com/abiosoft/colima/environment" + "github.com/abiosoft/colima/util/downloader" "runtime" ) -func installMinikube(guest environment.GuestActions, r *cli.ActiveCommandChain, kubeVersion string) { - installMinikubeCache(guest, r, kubeVersion) +func installMinikube(host environment.HostActions, guest environment.GuestActions, r *cli.ActiveCommandChain, kubeVersion string) { + installMinikubeCache(host, guest, r, kubeVersion) // install minikube last to ensure it is the last step downloadPath := "/tmp/minikube" url := "https://storage.googleapis.com/minikube/releases/latest/minikube-linux-" + runtime.GOARCH r.Add(func() error { - return guest.RunInteractive("curl", "-L", "-#", "-o", downloadPath, url) + return downloader.Download(host, guest, url, downloadPath) }) r.Add(func() error { return guest.Run("sudo", "install", downloadPath, "/usr/local/bin/minikube") }) } -func installMinikubeCache(guest environment.GuestActions, r *cli.ActiveCommandChain, kubeVersion string) { +func installMinikubeCache(host environment.HostActions, guest environment.GuestActions, r *cli.ActiveCommandChain, kubeVersion string) { downloadPath := "/tmp/minikube-cache.tar.gz" url := "https://dl.k8s.io/" + kubeVersion + "/kubernetes-node-linux-" + runtime.GOARCH + ".tar.gz" r.Add(func() error { - return guest.RunInteractive("curl", "-L", "-#", "-o", downloadPath, url) + return downloader.Download(host, guest, url, downloadPath) }) r.Add(func() error { return guest.Run("tar", "xvfz", downloadPath, "-C", "/tmp") diff --git a/environment/vm/lima/lima.go b/environment/vm/lima/lima.go index d3d601a1..90082fee 100644 --- a/environment/vm/lima/lima.go +++ b/environment/vm/lima/lima.go @@ -8,6 +8,7 @@ import ( "github.com/abiosoft/colima/config" "github.com/abiosoft/colima/environment" "github.com/abiosoft/colima/util" + "github.com/abiosoft/colima/util/yamlutil" "os" "path/filepath" "strconv" @@ -65,7 +66,7 @@ func (l *limaVM) Start(conf config.Config) error { r.Add(func() error { limaConf := newConf(conf) - return util.WriteYAML(limaConf, configFile) + return yamlutil.WriteYAML(limaConf, configFile) }) r.Add(func() error { return l.host.Run(limactl, "start", "--tty=false", configFile) @@ -97,7 +98,7 @@ func (l limaVM) resume(conf config.Config) error { r.Add(func() error { limaConf := newConf(conf) - return util.WriteYAML(limaConf, configFile) + return yamlutil.WriteYAML(limaConf, configFile) }) r.Stage("starting") diff --git a/util/downloader/download.go b/util/downloader/download.go new file mode 100644 index 00000000..8e13785d --- /dev/null +++ b/util/downloader/download.go @@ -0,0 +1,68 @@ +package downloader + +import ( + "crypto/sha256" + "fmt" + "github.com/abiosoft/colima/config" + "github.com/abiosoft/colima/environment" + "os" + "path/filepath" +) + +// Download downloads file at url and saves it in the destination. +// +// In the implementation, the file is downloaded (and cached) on the host, but copied to the desired +// destination for the guest. +// fileName must be a directory on the guest that does not require root access. +func Download(host environment.HostActions, guest environment.GuestActions, url, fileName string) error { + d := downloader{ + host: host, + guest: guest, + } + + if !d.hasCache(url) { + if err := d.downloadFile(url); err != nil { + return fmt.Errorf("error downloading '%s': %w", url, err) + } + } + + return guest.Run("cp", d.cacheFileName(url), fileName) +} + +type downloader struct { + host environment.HostActions + guest environment.GuestActions +} + +func (d downloader) cacheFileName(url string) string { + return filepath.Join(config.CacheDir(), "caches", sha256Hash(url)) +} + +func (d downloader) cacheDownloadingFileName(url string) string { + return d.cacheFileName(url) + ".downloading" +} + +func (d downloader) downloadFile(url string) (err error) { + // save to a temporary file initially before renaming to the desired file after successful download + // this prevents having a corrupt file + cacheFileName := d.cacheDownloadingFileName(url) + if err := d.host.Run("mkdir", "-p", filepath.Dir(cacheFileName)); err != nil { + return fmt.Errorf("error preparing cache dir: %w", err) + } + // ask curl to resume previous download if possible + if err := d.host.RunInteractive("curl", "-L", "-#", "-C", "-", "-o", cacheFileName, url); err != nil { + return err + } + return d.host.Run("cp", d.cacheDownloadingFileName(url), d.cacheFileName(url)) + +} + +func (d downloader) hasCache(url string) bool { + _, err := os.Stat(d.cacheFileName(url)) + return err == nil +} + +func sha256Hash(s string) string { + sum := sha256.Sum256([]byte(s)) + return fmt.Sprintf("%x", sum) +} diff --git a/util/template.go b/util/template.go index 76fb7d21..d8d73f5e 100644 --- a/util/template.go +++ b/util/template.go @@ -3,7 +3,6 @@ package util import ( "bytes" "fmt" - "gopkg.in/yaml.v3" "os" "text/template" ) @@ -31,13 +30,3 @@ func ParseTemplate(body string, values interface{}) ([]byte, error) { return b.Bytes(), err } - -// WriteYAML encodes struct to file as YAML. -func WriteYAML(value interface{}, file string) error { - b, err := yaml.Marshal(value) - if err != nil { - return fmt.Errorf("error encoding YAML: %w", err) - } - - return os.WriteFile(file, b, 0644) -} diff --git a/util/yamlutil/yaml.go b/util/yamlutil/yaml.go new file mode 100644 index 00000000..ccda323a --- /dev/null +++ b/util/yamlutil/yaml.go @@ -0,0 +1,17 @@ +package yamlutil + +import ( + "fmt" + "gopkg.in/yaml.v3" + "os" +) + +// WriteYAML encodes struct to file as YAML. +func WriteYAML(value interface{}, file string) error { + b, err := yaml.Marshal(value) + if err != nil { + return fmt.Errorf("error encoding YAML: %w", err) + } + + return os.WriteFile(file, b, 0644) +}