core: fix disk image downloader

Signed-off-by: Abiola Ibrahim <[email protected]>

core/core.go

@@ -54,11 +54,10 @@ func SetupBinfmt(host hostActions, guest guestActions, arch environment.Arch) er
  // download
  url := baseURL + "binfmt-" + arch.Value().GoArch() + ".tar.gz"
  dest := "/tmp/binfmt.tar.gz"
- if err := downloader.Download(host, guest, downloader.Request{
+ if err := downloader.DownloadToGuest(host, guest, downloader.Request{
  URL: url,
  SHA: downloadSha(url),
- Filename: dest,
- }); err != nil {
+ }, dest); err != nil {
  return fmt.Errorf("error downloading binfmt: %w", err)
  }
 

environment/container/kubernetes/k3s.go

@@ -46,11 +46,10 @@ func installK3sBinary(
  }
  a.Add(func() error {
  r := downloader.Request{
- URL: url,
- Filename: downloadPath,
- SHA: &downloader.SHA{Size: 256, URL: shaURL},
+ URL: url,
+ SHA: &downloader.SHA{Size: 256, URL: shaURL},
  }
- return downloader.Download(host, guest, r)
+ return downloader.DownloadToGuest(host, guest, r, downloadPath)
  })
  a.Add(func() error {
  return guest.Run("sudo", "install", downloadPath, "/usr/local/bin/k3s")
@@ -75,11 +74,10 @@ func installK3sCache(
  shaURL := baseURL + shaSumTxt
  a.Add(func() error {
  r := downloader.Request{
- URL: url,
- Filename: downloadPathTarGz,
- SHA: &downloader.SHA{Size: 256, URL: shaURL},
+ URL: url,
+ SHA: &downloader.SHA{Size: 256, URL: shaURL},
  }
- return downloader.Download(host, guest, r)
+ return downloader.DownloadToGuest(host, guest, r, downloadPathTarGz)
  })
  a.Add(func() error {
  return guest.Run("gzip", "-f", "-d", downloadPathTarGz)
@@ -129,8 +127,8 @@ func installK3sCluster(
  downloadPath := "/tmp/k3s-install.sh"
  url := "https://raw.githubusercontent.com/k3s-io/k3s/" + k3sVersion + "/install.sh"
  a.Add(func() error {
- r := downloader.Request{URL: url, Filename: downloadPath}
- return downloader.Download(host, guest, r)
+ r := downloader.Request{URL: url}
+ return downloader.DownloadToGuest(host, guest, r, downloadPath)
  })
  a.Add(func() error {
  return guest.Run("sudo", "install", downloadPath, "/usr/local/bin/k3s-install.sh")

environment/vm/lima/lima.go

@@ -90,21 +90,28 @@ func (l *limaVM) Start(ctx context.Context, conf config.Config) error {
  })
 
  a.Stage("creating and starting")
- configFile := filepath.Join(os.TempDir(), config.CurrentProfile().ID+".yaml")
+ confFile := filepath.Join(os.TempDir(), config.CurrentProfile().ID+".yaml")
 
  a.Add(func() (err error) {
  l.limaConf, err = newConf(ctx, conf)
- if err != nil {
- return err
- }
- return yamlutil.WriteYAML(l.limaConf, configFile)
+ return err
+ })
+
+ a.Stage("downloading disk image")
+ a.Add(func() error {
+ return l.downloadDiskImage(conf)
  })
+
+ a.Add(func() error {
+ return yamlutil.WriteYAML(l.limaConf, confFile)
+ })
+
  a.Add(l.writeNetworkFile)
  a.Add(func() error {
- return l.host.Run(limactl, "start", "--tty=false", configFile)
+ return l.host.Run(limactl, "start", "--tty=false", confFile)
  })
  a.Add(func() error {
- return os.Remove(configFile)
+ return os.Remove(confFile)
  })
 
  // adding it to command chain to execute only after successful startup.
@@ -137,10 +144,16 @@ func (l *limaVM) resume(ctx context.Context, conf config.Config) error {
  conf = l.syncDiskSize(ctx, conf)
 
  l.limaConf, err = newConf(ctx, conf)
- if err != nil {
- return err
- }
- return yamlutil.WriteYAML(l.limaConf, l.limaConfFile())
+ return err
+ })
+
+ a.Add(func() error {
+ return l.downloadDiskImage(conf)
+ })
+
+ a.Add(func() error {
+ err := yamlutil.WriteYAML(l.limaConf, l.limaConfFile())
+ return err
  })
 
  a.Add(l.writeNetworkFile)
@@ -257,6 +270,16 @@ func (l limaVM) Arch() environment.Arch {
  return environment.Arch(a)
 }
 
+func (l *limaVM) downloadDiskImage(conf config.Config) error {
+ image, err := limautil.Image(l.limaConf.Arch, conf.Runtime)
+ if err != nil {
+ return fmt.Errorf("error getting qcow image: %w", err)
+ }
+
+ l.limaConf.Images = append(l.limaConf.Images, image)
+ return nil
+}
+
 func (l *limaVM) syncDiskSize(ctx context.Context, conf config.Config) config.Config {
  log := l.Logger(ctx)
  instance, err := limautil.InstanceConfig()

environment/vm/lima/limautil/image.go

@@ -5,10 +5,13 @@ import (
  "bytes"
  "fmt"
  "io"
+ "os"
 
  "github.com/abiosoft/colima/embedded"
  "github.com/abiosoft/colima/environment"
+ "github.com/abiosoft/colima/environment/host"
  "github.com/abiosoft/colima/environment/vm/lima/limaconfig"
+ "github.com/abiosoft/colima/util/downloader"
  "github.com/sirupsen/logrus"
 )
 
@@ -19,18 +22,39 @@ func init() {
 }
 
 func Image(arch environment.Arch, runtime string) (f limaconfig.File, err error) {
- if imgFile, ok := images[runtime]; ok {
- if img, ok := imgFile[arch.GoArch()]; ok {
- return img, nil
- }
+ err = fmt.Errorf("cannot find %s image for %s runtime", arch, runtime)
+
+ imgFile, ok := diskImageMap[runtime]
+ if !ok {
+ return
+ }
+ img, ok := imgFile[arch.GoArch()]
+ if !ok {
+ return
+ }
+
+ host := host.New()
+ // download image
+ qcow2, err := downloadImage(host, img)
+ if err != nil {
+ return f, err
+ }
+ // convert from qcow2 to raw
+ raw, err := qcow2ToRaw(host, qcow2)
+ if err != nil {
+ return f, err
  }
 
- return f, fmt.Errorf("cannot find %s image for %s runtime", arch, runtime)
+ img.Location = raw
+ img.Digest = "" // remove digest
+ return img, nil
 }
 
-var images = map[string]imageFiles{}
+// map of runtime to disk images.
+var diskImageMap = map[string]diskImages{}
 
-type imageFiles map[string]limaconfig.File
+// map of architecture to disk image
+type diskImages map[string]limaconfig.File
 
 func loadImages() error {
  filename := "images/images.txt"
@@ -60,14 +84,54 @@ func loadImagesFromBytes(b []byte) error {
  file.Digest = "sha512:" + sha
  }
 
- var files = imageFiles{}
- if m, ok := images[runtime]; ok {
+ var files = diskImages{}
+ if m, ok := diskImageMap[runtime]; ok {
  files = m
  }
  files[arch.GoArch()] = file
 
- images[runtime] = files
+ diskImageMap[runtime] = files
  }
 
  return nil
 }
+
+// downloadImage downloads the file and returns the location of the downloaded file.
+func downloadImage(host environment.HostActions, file limaconfig.File) (string, error) {
+ // download image
+ request := downloader.Request{URL: file.Location}
+ if file.Digest != "" {
+ request.SHA = &downloader.SHA{Size: 512, Digest: file.Digest}
+ }
+ location, err := downloader.Download(host, request)
+ if err != nil {
+ return "", fmt.Errorf("error during image download: %w", err)
+ }
+
+ return location, nil
+}
+
+// qcow2ToRaw uses qemu-img to conver the image from qcow to raw.
+// Returns the filename of the raw file and an error (if any).
+func qcow2ToRaw(host environment.Host, image string) (string, error) {
+ f := diskImageFile(image)
+
+ if _, err := os.Stat(f.Raw()); err == nil {
+ // already exists, return
+ return f.Raw(), nil
+ }
+
+ err := host.Run("qemu-img", "convert", "-f", "qcow2", "-O", "raw", f.String(), f.Raw())
+ if err != nil {
+ // remove the incomplete raw file
+ _ = host.RunQuiet("rm", "-f", f.Raw())
+ return "", err
+ }
+
+ return f.Raw(), err
+}
+
+type diskImageFile string
+
+func (d diskImageFile) String() string { return string(d) }
+func (d diskImageFile) Raw() string { return string(d) + ".raw" }

environment/vm/lima/yaml.go

@@ -17,7 +17,6 @@ import (
  "github.com/abiosoft/colima/environment/container/docker"
  "github.com/abiosoft/colima/environment/container/incus"
  "github.com/abiosoft/colima/environment/vm/lima/limaconfig"
- "github.com/abiosoft/colima/environment/vm/lima/limautil"
  "github.com/abiosoft/colima/util"
  "github.com/sirupsen/logrus"
 )
@@ -52,13 +51,6 @@ func newConf(ctx context.Context, conf config.Config) (l limaconfig.Config, err
  }
  }
 
- image, err := limautil.Image(l.Arch, conf.Runtime)
- if err != nil {
- return l, fmt.Errorf("error getting qcow image: %w", err)
- }
-
- l.Images = append(l.Images, image)
-
  if conf.CPU > 0 {
  l.CPUs = &conf.CPU
  }