[installer] Update the installer (#23902)

* * Add experiment unit file
* Setup and start the updater experiment

* * Point unit files to the bootstraped version of the updater
* Use the helper to setcap on the newly installed helper, with some validation
* Activate datadog-updater unit file on bootstrap

* Add file header

* WIP

* Cleanup installer systemd units

* Fix the helper script

* Fix installer helper setcap command

* Rename updater to installer

* Remove untrue commit

* Add the installer bootstrap in rpm

* Remove systemd startup from postinst

* Remove /usr/lib/systemd/system package file

* Fix linter + windows

* Fix fucntion comment

* Bootsrap datadog oci latest

* add windows shims

* Fix updater helper permissions on install

* fix windows lint

* * Fix updater helper permission check
* Fix systemdpath used by updater helper

* Add more error messages

* check file mode correctly

* fix error

* Enable installer unit before starting it

* fix rpmSystemdPath

* Invert systemd path search

* Print ls output in e2e tests

* skip TestInstallerUnitLoaded on rpm

cmd/updater/subcommands/bootstrap/command.go

@@ -47,7 +47,7 @@ func Commands(global *command.GlobalParams) []*cobra.Command {
  RunE: func(_ *cobra.Command, _ []string) error {
  ctx, cancel := context.WithTimeout(context.Background(), timeout)
  defer cancel()
- return boostrapFxWrapper(ctx, &cliParams{
+ return bootstrapFxWrapper(ctx, &cliParams{
  GlobalParams: *global,
  url: url,
  pkg: pkg,
@@ -62,7 +62,7 @@ func Commands(global *command.GlobalParams) []*cobra.Command {
  return []*cobra.Command{bootstrapCmd}
 }
 
-func boostrapFxWrapper(ctx context.Context, params *cliParams) error {
+func bootstrapFxWrapper(ctx context.Context, params *cliParams) error {
  return fxutil.OneShot(bootstrap,
  fx.Provide(func() context.Context { return ctx }),
  fx.Supply(params),

cmd/updater/subcommands/run/command.go

@@ -8,6 +8,9 @@ package run
 
 import (
  "context"
+ "os"
+ "os/signal"
+ "syscall"
 
  "github.com/spf13/cobra"
  "go.uber.org/fx"
@@ -25,6 +28,7 @@ import (
  "github.com/DataDog/datadog-agent/comp/remote-config/rctelemetryreporter/rctelemetryreporterimpl"
  "github.com/DataDog/datadog-agent/pkg/config/remote/service"
  "github.com/DataDog/datadog-agent/pkg/util/fxutil"
+ "github.com/DataDog/datadog-agent/pkg/util/log"
 
  "github.com/DataDog/datadog-agent/comp/updater/localapi"
  "github.com/DataDog/datadog-agent/comp/updater/localapi/localapiimpl"
@@ -77,6 +81,20 @@ func runFxWrapper(params *cliParams) error {
  )
 }
 
-func run(_ pid.Component, _ localapi.Component) error {
- select {}
+func run(shutdowner fx.Shutdowner, _ pid.Component, _ localapi.Component) error {
+ handleSignals(shutdowner)
+ return nil
+}
+
+func handleSignals(shutdowner fx.Shutdowner) {
+ sigChan := make(chan os.Signal, 1)
+ signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGPIPE)
+ for signo := range sigChan {
+ switch signo {
+ case syscall.SIGINT, syscall.SIGTERM:
+ log.Infof("Received signal %d (%v)", signo, signo)
+ _ = shutdowner.Shutdown()
+ return
+ }
+ }
 }

omnibus/config/projects/installer.rb

@@ -138,11 +138,6 @@
 # ------------------------------------
 
 if linux_target?
- systemd_directory = "/usr/lib/systemd/system"
- if debian_target?
- systemd_directory = "/lib/systemd/system"
- end
- extra_package_file "#{systemd_directory}/datadog-installer.service"
  extra_package_file '/etc/datadog-agent/'
  extra_package_file '/var/log/datadog/'
  extra_package_file '/var/run/datadog-packages/'

omnibus/config/software/installer.rb

@@ -46,30 +46,26 @@
 
  # Packages
  mkdir "/opt/datadog-packages"
-
  copy 'bin/installer', "#{install_dir}/bin/"
 
- # Add installer units
- systemdPath = "/lib/systemd/system/"
- if not debian_target?
- mkdir "/usr/lib/systemd/system/"
- systemdPath = "/usr/lib/systemd/system/"
- end
+ systemdPath = "#{install_dir}/systemd/"
  erb source: "datadog-installer.service.erb",
  dest: systemdPath + "datadog-installer.service",
  mode: 0644,
- vars: { install_dir: install_dir, etc_dir: etc_dir}
+ vars: { installer_dir: "/opt/datadog-packages/datadog-installer/stable", etc_dir: etc_dir}
 
+ erb source: "datadog-installer-exp.service.erb",
+ dest: systemdPath + "datadog-installer-exp.service",
+ mode: 0644,
+ vars: { installer_dir: "/opt/datadog-packages/datadog-installer/experiment", etc_dir: etc_dir}
 
- systemdPath = "#{install_dir}/systemd/"
  # Add stable agent units
  templateToFile = {
  "datadog-agent.service.erb" => "datadog-agent.service",
  "datadog-agent-trace.service.erb" => "datadog-agent-trace.service",
  "datadog-agent-process.service.erb" => "datadog-agent-process.service",
  "datadog-agent-security.service.erb" => "datadog-agent-security.service",
  "datadog-agent-sysprobe.service.erb" => "datadog-agent-sysprobe.service",
- "datadog-installer.service.erb" => "datadog-installer.service",
  }
  templateToFile.each do |template, file|
  agent_dir = "/opt/datadog-packages/datadog-agent/stable"

omnibus/config/templates/installer/datadog-installer-exp.service.erb

@@ -0,0 +1,20 @@
+[Unit]
+Description=Datadog Installer Experiment
+After=network.target
+OnFailure=datadog-installer.service
+Conflicts=datadog-installer.service
+JobTimeoutSec=3000 #50 minutes
+
+[Service]
+Type=oneshot
+PIDFile=<%= installer_dir %>/run/installer.pid
+User=dd-installer
+EnvironmentFile=-<%= etc_dir %>/environment
+ExecStart=<%= installer_dir %>/bin/installer/installer run -p <%= installer_dir %>/run/installer.pid
+ExecStart=<%= installer_dir %>/bin/installer/installer run -p <%= installer_dir %>/run/installer.pid
+ExecStart=<%= installer_dir %>/bin/installer/installer run -p <%= installer_dir %>/run/installer.pid
+ExecStart=/bin/false
+ExecStop=/bin/false
+
+[Install]
+WantedBy=multi-user.target

omnibus/config/templates/installer/datadog-installer.service.erb

@@ -1,14 +1,15 @@
 [Unit]
 Description=Datadog Installer
 After=network.target
+Conflicts=datadog-installer-exp.service
 
 [Service]
 Type=simple
-PIDFile=<%= install_dir %>/run/installer.pid
+PIDFile=<%= installer_dir %>/run/installer.pid
 User=dd-installer
 Restart=on-failure
 EnvironmentFile=-<%= etc_dir %>/environment
-ExecStart=<%= install_dir %>/bin/installer/installer run -p <%= install_dir %>/run/installer.pid
+ExecStart=<%= installer_dir %>/bin/installer/installer run -p <%= installer_dir %>/run/installer.pid
 # Since systemd 229, should be in [Unit] but in order to support systemd <229,
 # it is also supported to have it here.
 StartLimitInterval=10

omnibus/package-scripts/updater-deb/postinst

@@ -6,6 +6,7 @@
 
 readonly PACKAGES_DIR=/opt/datadog-packages
 readonly INSTALL_DIR=/opt/datadog-installer
+readonly BOOTSTRAP_INSTALLER=${INSTALL_DIR}/bin/installer/installer
 readonly HELPER=${INSTALL_DIR}/bin/installer/helper
 readonly LOG_DIR=/var/log/datadog
 readonly PACKAGES_LOCK_DIR=/var/run/datadog-packages
@@ -81,8 +82,6 @@ fi
 chmod 750 ${HELPER}
 setcap cap_setuid+ep ${HELPER}
 
-# start updater
-SYSTEMCTL_SKIP_SYSV=true systemctl enable datadog-installer || true
-SYSTEMCTL_SKIP_SYSV=true systemctl start datadog-installer || true
+$BOOTSTRAP_INSTALLER bootstrap --url "oci:https://docker.io/datadog/installer-package-dev:latest"
 
 exit 0

omnibus/package-scripts/updater-rpm/posttrans

@@ -65,9 +65,7 @@ fi
 chmod 750 ${HELPER}
 setcap cap_setuid+ep ${HELPER}
 
-# start updater
-SYSTEMCTL_SKIP_SYSV=true systemctl enable datadog-installer || true
-SYSTEMCTL_SKIP_SYSV=true systemctl start datadog-installer || true
+$BOOTSTRAP_INSTALLER bootstrap --url "oci:https://docker.io/datadog/installer-package-dev:latest"
 
 exit 0
 

pkg/updater/install.go

@@ -29,8 +29,9 @@ const (
  datadogPackageMaxSize = 3 << 30 // 3GiB
  defaultConfigsDir = "/etc"
 
- packageDatadogAgent = "datadog-agent"
- packageAPMInjector = "datadog-apm-inject"
+ packageDatadogAgent = "datadog-agent"
+ packageAPMInjector = "datadog-apm-inject"
+ packageDatadogInstaller = "datadog-installer"
 )
 
 type installer struct {
@@ -69,6 +70,8 @@ func (i *installer) installStable(pkg string, version string, image oci.Image) e
  return service.SetupAgentUnits()
  case packageAPMInjector:
  return service.SetupAPMInjector()
+ case packageDatadogInstaller:
+ return service.SetupInstallerUnit()
  default:
  return nil
  }
@@ -117,6 +120,8 @@ func (i *installer) startExperiment(pkg string) error {
  switch pkg {
  case packageDatadogAgent:
  return service.StartAgentExperiment()
+ case packageDatadogInstaller:
+ return service.StartInstallerExperiment()
  default:
  return nil
  }
@@ -128,6 +133,8 @@ func (i *installer) stopExperiment(pkg string) error {
  switch pkg {
  case packageDatadogAgent:
  return service.StopAgentExperiment()
+ case packageAPMInjector:
+ return service.StopInstallerExperiment()
  default:
  return nil
  }

pkg/updater/repository/repository.go

@@ -13,7 +13,6 @@ import (
  "os"
  "path/filepath"
  "strconv"
- "strings"
 
  "github.com/DataDog/gopsutil/process"
 
@@ -366,10 +365,19 @@ func movePackageFromSource(packageName string, rootPath string, lockedPackages m
  if err := os.Chmod(targetPath, 0755); err != nil {
  return "", fmt.Errorf("could not set permissions on package: %w", err)
  }
- if strings.HasSuffix(rootPath, "datadog-agent") {
+ switch filepath.Base(rootPath) {
+ case "datadog-agent":
  if err := service.ChownDDAgent(targetPath); err != nil {
  return "", err
  }
+ case "datadog-installer":
+ helperPath := filepath.Join(rootPath, packageName, "bin/installer/helper")
+ if err := os.Chmod(helperPath, 0750); err != nil {
+ return "", fmt.Errorf("could not set permissions on installer-helper: %w", err)
+ }
+ if err := service.SetCapHelper(helperPath); err != nil {
+ return "", fmt.Errorf("could not set capabilities on installer-helper: %w", err)
+ }
  }
 
  return targetPath, nil

pkg/updater/service/cmd_executor.go

@@ -44,6 +44,11 @@ func rmAgentSymlink() error {
  return executeCommand(`{"command":"rm-agent-symlink"}`)
 }
 
+// SetCapHelper sets cap setuid on the newly installed helper
+func SetCapHelper(path string) error {
+ return executeCommand(`{"command":"setcap cap_setuid+ep", "path":"` + path + `"}`)
+}
+
 func executeCommand(command string) error {
  cancelctx, cancelfunc := context.WithTimeout(context.Background(), execTimeout)
  defer cancelfunc()

pkg/updater/service/cmd_executor_windows.go

@@ -22,3 +22,8 @@ func RemoveAll(path string) error {
 func BuildHelperForTests(_, _ string, _ bool) error {
  return nil
 }
+
+// SetCapHelper sets cap setuid on the newly installed helper
+func SetCapHelper(_ string) error {
+ return nil
+}

pkg/updater/service/datadog_installer.go

@@ -0,0 +1,68 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+//go:build !windows
+
+package service
+
+import "github.com/DataDog/datadog-agent/pkg/util/log"
+
+const (
+ installerUnit = "datadog-installer.service"
+ installerUnitExp = "datadog-installer-exp.service"
+)
+
+var installerUnits = []string{installerUnit, installerUnitExp}
+
+// SetupInstallerUnit installs and starts the installer systemd units
+func SetupInstallerUnit() (err error) {
+ defer func() {
+ if err != nil {
+ log.Errorf("Failed to setup installer units: %s, reverting", err)
+ }
+ }()
+
+ for _, unit := range installerUnits {
+ if err = loadUnit(unit); err != nil {
+ return err
+ }
+ }
+
+ if err = systemdReload(); err != nil {
+ return err
+ }
+
+ if err = enableUnit(installerUnit); err != nil {
+ return err
+ }
+
+ if err = startUnit(installerUnit); err != nil {
+ return err
+ }
+ return nil
+}
+
+// RemoveInstallerUnit removes the installer systemd units
+func RemoveInstallerUnit() {
+ var err error
+ for _, unit := range installerUnits {
+ if err = disableUnit(unit); err != nil {
+ log.Warnf("Failed to disable %s: %s", unit, err)
+ }
+ if err = removeUnit(unit); err != nil {
+ log.Warnf("Failed to stop %s: %s", unit, err)
+ }
+ }
+}
+
+// StartInstallerExperiment installs the experimental systemd units for the installer
+func StartInstallerExperiment() error {
+ return startUnit(installerUnitExp)
+}
+
+// StopInstallerExperiment installs the stable systemd units for the installer
+func StopInstallerExperiment() error {
+ return startUnit(installerUnit)
+}

pkg/updater/service/datadog_installer_windows.go

@@ -0,0 +1,27 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+//go:build windows
+
+package service
+
+// SetupInstallerUnit noop
+func SetupInstallerUnit() (err error) {
+ return nil
+}
+
+// RemoveInstallerUnit noop
+func RemoveInstallerUnit() {
+}
+
+// StartInstallerExperiment noop
+func StartInstallerExperiment() error {
+ return nil
+}
+
+// StopInstallerExperiment noop
+func StopInstallerExperiment() error {
+ return nil
+}