Draft: LXC support for self-hosted runners

pkg/container/executions_environment.go

@@ -5,6 +5,8 @@ import "context"
 type ExecutionsEnvironment interface {
  Container
  ToContainerPath(string) string
+ GetName() string
+ GetRoot() string
  GetActPath() string
  GetPathVariableName() string
  DefaultPathVariable() string

pkg/container/host_environment.go

@@ -25,16 +25,18 @@ import (
 )
 
 type HostEnvironment struct {
+ Name string
  Path string
  TmpDir string
  ToolCache string
  Workdir string
  ActPath string
+ Root string
  CleanUp func()
  StdOut io.Writer
 }
 
-func (e *HostEnvironment) Create(capAdd []string, capDrop []string) common.Executor {
+func (e *HostEnvironment) Create(capAdd, capDrop []string) common.Executor {
  return func(ctx context.Context) error {
  return nil
  }
@@ -60,7 +62,7 @@ func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Exec
  }
 }
 
-func (e *HostEnvironment) CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor {
+func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor {
  return func(ctx context.Context) error {
  logger := common.Logger(ctx)
  srcPrefix := filepath.Dir(srcPath)
@@ -254,7 +256,7 @@ func getEnvListFromMap(env map[string]string) []string {
  return envList
 }
 
-func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, user, workdir string) error {
+func (e *HostEnvironment) exec(ctx context.Context, commandparam []string, cmdline string, env map[string]string, user, workdir string) error {
  envList := getEnvListFromMap(env)
  var wd string
  if workdir != "" {
@@ -266,6 +268,15 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
  } else {
  wd = e.Path
  }
+ command := make([]string, len(commandparam))
+ copy(command, commandparam)
+ if user == "root" {
+ command = append([]string{"/usr/bin/sudo"}, command...)
+ } else {
+ common.Logger(ctx).Debugf("lxc-attach --name %v %v", e.Name, command)
+ command = append([]string{"/usr/bin/sudo", "--preserve-env", "--preserve-env=PATH", "/usr/bin/lxc-attach", "--keep-env", "--name", e.Name, "--"}, command...)
+ }
+
  f, err := lookupPathHost(command[0], env, e.StdOut)
  if err != nil {
  return err
@@ -308,7 +319,7 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
  }
  err = cmd.Run()
  if err != nil {
- return err
+ return fmt.Errorf("RUN %w", err)
  }
  if tty != nil {
  writer.AutoStop = true
@@ -361,6 +372,14 @@ func (e *HostEnvironment) ToContainerPath(path string) string {
  return path
 }
 
+func (e *HostEnvironment) GetName() string {
+ return e.Name
+}
+
+func (e *HostEnvironment) GetRoot() string {
+ return e.Root
+}
+
 func (e *HostEnvironment) GetActPath() string {
  return e.ActPath
 }
@@ -414,7 +433,7 @@ func (e *HostEnvironment) GetRunnerContext(ctx context.Context) map[string]inter
  }
 }
 
-func (e *HostEnvironment) ReplaceLogWriter(stdout io.Writer, stderr io.Writer) (io.Writer, io.Writer) {
+func (e *HostEnvironment) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) {
  org := e.StdOut
  e.StdOut = stdout
  return org, org

pkg/container/linux_container_environment_extensions.go

@@ -10,8 +10,7 @@ import (
  log "github.com/sirupsen/logrus"
 )
 
-type LinuxContainerEnvironmentExtensions struct {
-}
+type LinuxContainerEnvironmentExtensions struct{}
 
 // Resolves the equivalent host path inside the container
 // This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject
@@ -47,6 +46,14 @@ func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string
  return result
 }
 
+func (*LinuxContainerEnvironmentExtensions) GetName() string {
+ return "NAME"
+}
+
+func (*LinuxContainerEnvironmentExtensions) GetRoot() string {
+ return "/var/run"
+}
+
 func (*LinuxContainerEnvironmentExtensions) GetActPath() string {
  return "/var/run/act"
 }

pkg/runner/run_context.go

@@ -3,6 +3,7 @@ package runner
 import (
  "archive/tar"
  "bufio"
+ "bytes"
  "context"
  "crypto/rand"
  "crypto/sha256"
@@ -16,6 +17,7 @@ import (
  "regexp"
  "runtime"
  "strings"
+ "text/template"
 
  "github.com/mitchellh/go-homedir"
  "github.com/opencontainers/selinux/go-selinux"
@@ -139,6 +141,113 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) {
  return binds, mounts
 }
 
+var startTemplate = template.Must(template.New("start").Parse(`#!/bin/sh -xe
+lxc-create --name="{{.Name}}" --template={{.Template}} -- --release {{.Release}} $packages
+tee -a /var/lib/lxc/{{.Name}}/config <<'EOF'
+security.nesting = true
+lxc.cap.drop =
+lxc.apparmor.profile = unconfined
+#
+# /dev/net (docker won't work without /dev/net/tun)
+#
+lxc.cgroup2.devices.allow = c 10:200 rwm
+lxc.mount.entry = /dev/net dev/net none bind,create=dir 0 0
+#
+# /dev/kvm (libvirt / kvm won't work without /dev/kvm)
+#
+lxc.cgroup2.devices.allow = c 10:232 rwm
+lxc.mount.entry = /dev/kvm dev/kvm none bind,create=file 0 0
+#
+# /dev/loop
+#
+lxc.cgroup2.devices.allow = c 10:237 rwm
+lxc.cgroup2.devices.allow = b 7:* rwm
+lxc.mount.entry = /dev/loop-control dev/loop-control none bind,create=file 0 0
+#
+# /dev/mapper
+#
+lxc.cgroup2.devices.allow = c 10:236 rwm
+lxc.mount.entry = /dev/mapper dev/mapper none bind,create=dir 0 0
+#
+# /dev/fuse
+#
+lxc.cgroup2.devices.allow = b 10:229 rwm
+lxc.mount.entry = /dev/fuse dev/fuse none bind,create=file 0 0
+EOF
+
+mkdir -p /var/lib/lxc/{{.Name}}/rootfs/{{ .Root }}
+mount --bind {{ .Root }} /var/lib/lxc/{{.Name}}/rootfs/{{ .Root }}
+
+mkdir /var/lib/lxc/{{.Name}}/rootfs/tmpdir
+mount --bind {{.TmpDir}} /var/lib/lxc/{{.Name}}/rootfs/tmpdir
+
+lxc-start {{.Name}}
+lxc-wait --name {{.Name}} --state RUNNING
+
+#
+# Wait for the network to come up
+#
+cat > /var/lib/lxc/{{.Name}}/rootfs/tmpdir/networking.sh <<'EOF'
+#!/bin/sh -xe
+for d in $(seq 60); do
+ getent hosts wikipedia.org > /dev/null && break
+ sleep 1
+done
+getent hosts wikipedia.org
+EOF
+chmod +x /var/lib/lxc/{{.Name}}/rootfs/tmpdir/networking.sh
+
+lxc-attach --name {{.Name}} -- /tmpdir/networking.sh
+
+cat > /var/lib/lxc/{{.Name}}/rootfs/tmpdir/node.sh <<'EOF'
+#!/bin/sh -xe
+# https://github.com/nodesource/distributions#debinstall
+apt-get install -y curl git
+curl -fsSL https://deb.nodesource.com/setup_16.x | bash -
+apt-get install -y nodejs
+EOF
+chmod +x /var/lib/lxc/{{.Name}}/rootfs/tmpdir/node.sh
+
+lxc-attach --name {{.Name}} -- /tmpdir/node.sh
+
+`))
+
+var stopTemplate = template.Must(template.New("stop").Parse(`#!/bin/sh -x
+lxc-ls -1 --filter="^{{.Name}}" | while read container ; do
+ lxc-stop --kill --name="$container"
+ umount "/var/lib/lxc/$container/rootfs/{{ .Root }}"
+ umount "/var/lib/lxc/$container/rootfs/tmpdir"
+ lxc-destroy --force --name="$container"
+done
+`))
+
+func (rc *RunContext) stopHostEnvironment() common.Executor {
+ return func(ctx context.Context) error {
+ logger := common.Logger(ctx)
+ logger.Debugf("stopHostEnvironment")
+
+ var stopScript bytes.Buffer
+ if err := stopTemplate.Execute(&stopScript, struct {
+ Name string
+ Root string
+ }{
+ Name: rc.JobContainer.GetName(),
+ Root: rc.JobContainer.GetRoot(),
+ }); err != nil {
+ return err
+ }
+
+ return common.NewPipelineExecutor(
+ rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{
+ Name: "workflow/stop-lxc.sh",
+ Mode: 0755,
+ Body: stopScript.String(),
+ }),
+ rc.JobContainer.Exec([]string{rc.JobContainer.GetActPath() + "/workflow/stop-lxc.sh"}, map[string]string{}, "root", rc.Config.Workdir),
+ )(ctx)
+ }
+}
+
 func (rc *RunContext) startHostEnvironment() common.Executor {
  return func(ctx context.Context) error {
  logger := common.Logger(ctx)
@@ -154,7 +263,8 @@ func (rc *RunContext) startHostEnvironment() common.Executor {
  cacheDir := rc.ActionCacheDir()
  randBytes := make([]byte, 8)
  _, _ = rand.Read(randBytes)
- miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes))
+ randName := hex.EncodeToString(randBytes)
+ miscpath := filepath.Join(cacheDir, randName)
  actPath := filepath.Join(miscpath, "act")
  if err := os.MkdirAll(actPath, 0o777); err != nil {
  return err
@@ -169,6 +279,8 @@ func (rc *RunContext) startHostEnvironment() common.Executor {
  }
  toolCache := filepath.Join(cacheDir, "tool_cache")
  rc.JobContainer = &container.HostEnvironment{
+ Name: randName,
+ Root: miscpath,
  Path: path,
  TmpDir: runnerTmp,
  ToolCache: toolCache,
@@ -194,7 +306,34 @@ func (rc *RunContext) startHostEnvironment() common.Executor {
  }
  }
 
+ var startScript bytes.Buffer
+ if err := startTemplate.Execute(&startScript, struct {
+ Name string
+ Template string
+ Release string
+ Repo string
+ Root string
+ TmpDir string
+ Script string
+ }{
+ Name: rc.JobContainer.GetName(),
+ Template: "debian",
+ Release: "bullseye",
+ Repo: "", // step.Environment["CI_REPO"],
+ Root: rc.JobContainer.GetRoot(),
+ TmpDir: runnerTmp,
+ Script: "", // "commands-" + step.Name,
+ }); err != nil {
+ return err
+ }
+
  return common.NewPipelineExecutor(
+ rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{
+ Name: "workflow/start-lxc.sh",
+ Mode: 0755,
+ Body: startScript.String(),
+ }),
+ rc.JobContainer.Exec([]string{rc.JobContainer.GetActPath() + "/workflow/start-lxc.sh"}, map[string]string{}, "root", rc.Config.Workdir),
  rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{
  Name: "workflow/event.json",
  Mode: 0o644,
@@ -397,12 +536,22 @@ func (rc *RunContext) IsHostEnv(ctx context.Context) bool {
 }
 
 func (rc *RunContext) stopContainer() common.Executor {
- return rc.stopJobContainer()
+ return func(ctx context.Context) error {
+ image := rc.platformImage(ctx)
+ if strings.EqualFold(image, "-self-hosted") {
+ return rc.stopHostEnvironment()(ctx)
+ }
+ return rc.stopJobContainer()(ctx)
+ }
 }
 
 func (rc *RunContext) closeContainer() common.Executor {
  return func(ctx context.Context) error {
  if rc.JobContainer != nil {
+ image := rc.platformImage(ctx)
+ if strings.EqualFold(image, "-self-hosted") {
+ return rc.stopHostEnvironment()(ctx)
+ }
  return rc.JobContainer.Close()(ctx)
  }
  return nil