fix(executor): windows output artifacts. Fixes #4082 (#4083)

Dockerfile.windows

@@ -19,7 +19,7 @@ RUN iex ((new-object net.webclient).DownloadString('https://chocolatey.org/insta
 
 # install golang, dep and other tools
 RUN choco install golang --version=$env:GOLANG_VERSION ; \
- choco install make dep docker-cli git.portable
+ choco install make dep docker-cli git.portable 7zip.portable
 
 ####################################################################################################
 # argoexec-base
@@ -41,6 +41,7 @@ RUN mkdir C:\app && \
 
 COPY --from=builder C:/ProgramData/chocolatey/lib/docker-cli/tools/docker.exe C:/app/docker.exe
 COPY --from=builder C:/tools/git C:/app/git
+COPY --from=builder C:/ProgramData/chocolatey/lib/7zip.portable/tools/7z-extra/x64/7za.exe C:/app/7za.exe
 
 # add binaries to path
 USER Administrator

USERS.md

@@ -85,3 +85,4 @@ Currently, the following organizations are **officially** using Argo Workflows:
 1. [Tulip](https://tulip.com/)
 1. [Wavefront](https://www.wavefront.com/)
 1. [Wellcome Trust](https://wellcome.ac.uk/)
+1. [Helio](https://helio.exchange)

workflow/common/util.go

@@ -8,6 +8,7 @@ import (
  "net/http"
  "os/exec"
  "regexp"
+ "runtime"
  "strconv"
  "strings"
  "time"
@@ -404,20 +405,33 @@ func Replace(fstTmpl *fasttemplate.Template, replaceMap map[string]string, allow
 }
 
 // RunCommand is a convenience function to run/log a command and log the stderr upon failure
-func RunCommand(name string, arg ...string) error {
+func RunCommand(name string, arg ...string) ([]byte, error) {
  cmd := exec.Command(name, arg...)
  cmdStr := strings.Join(cmd.Args, " ")
  log.Info(cmdStr)
- _, err := cmd.Output()
+ out, err := cmd.Output()
  if err != nil {
  if exErr, ok := err.(*exec.ExitError); ok {
  errOutput := string(exErr.Stderr)
  log.Errorf("`%s` failed: %s", cmdStr, errOutput)
- return errors.InternalError(strings.TrimSpace(errOutput))
+ return nil, errors.InternalError(strings.TrimSpace(errOutput))
  }
- return errors.InternalWrapError(err)
+ return nil, errors.InternalWrapError(err)
  }
- return nil
+ return out, nil
+}
+
+// RunShellCommand is a convenience function to use RunCommand for shell executions. It's os-specific
+// and runs `cmd` in windows.
+func RunShellCommand(arg ...string) ([]byte, error) {
+ name := "sh"
+ shellFlag := "-c"
+ if runtime.GOOS == "windows" {
+ name = "cmd"
+ shellFlag = "/c"
+ }
+ arg = append([]string{shellFlag}, arg...)
+ return RunCommand(name, arg...)
 }
 
 const patchRetries = 5

workflow/executor/docker/docker.go

@@ -8,6 +8,7 @@ import (
  "io/ioutil"
  "os"
  "os/exec"
+ "runtime"
  "strconv"
  "strings"
  "sync"
@@ -36,34 +37,18 @@ func (d *DockerExecutor) GetFileContents(containerID string, sourcePath string)
  // content from the tar archive and output into stdout. In this way, we do not need to
  // create and copy the content into a file from the wait container.
  dockerCpCmd := fmt.Sprintf("docker cp -a %s:%s - | tar -ax -O", containerID, sourcePath)
- cmd := exec.Command("sh", "-c", dockerCpCmd)
- log.Info(cmd.Args)
- out, err := cmd.Output()
+ out, err := common.RunShellCommand(dockerCpCmd)
  if err != nil {
- if exErr, ok := err.(*exec.ExitError); ok {
- log.Errorf("`%s` stderr:\n%s", cmd.Args, string(exErr.Stderr))
- }
- return "", errors.InternalWrapError(err)
+ return "", err
  }
  return string(out), nil
 }
 
 func (d *DockerExecutor) CopyFile(containerID string, sourcePath string, destPath string, compressionLevel int) error {
  log.Infof("Archiving %s:%s to %s", containerID, sourcePath, destPath)
- var levelFlag string
- switch compressionLevel {
- case gzip.NoCompression:
- // best we can do - if we skip gzip it's a different file
- levelFlag = "-1"
- case gzip.DefaultCompression:
- // use cmd default
- levelFlag = ""
- default:
- // -1 through -9 (or error)
- levelFlag = "-" + strconv.Itoa(compressionLevel)
- }
- dockerCpCmd := fmt.Sprintf("docker cp -a %s:%s - | gzip %s > %s", containerID, sourcePath, levelFlag, destPath)
- err := common.RunCommand("sh", "-c", dockerCpCmd)
+
+ dockerCpCmd := getDockerCpCmd(containerID, sourcePath, compressionLevel, destPath)
+ _, err := common.RunShellCommand(dockerCpCmd)
  if err != nil {
  return err
  }
@@ -182,15 +167,16 @@ func (d *DockerExecutor) WaitInit() error {
 
 // Wait for the container to complete
 func (d *DockerExecutor) Wait(containerID string) error {
- return common.RunCommand("docker", "wait", containerID)
+ _, err := common.RunCommand("docker", "wait", containerID)
+ return err
 }
 
 // killContainers kills a list of containerIDs first with a SIGTERM then with a SIGKILL after a grace period
 func (d *DockerExecutor) Kill(containerIDs []string) error {
  killArgs := append([]string{"kill", "--signal", "TERM"}, containerIDs...)
  // docker kill will return with an error if a container has terminated already, which is not an error in this case.
  // We therefore ignore any error. docker wait that follows will re-raise any other error with the container.
- err := common.RunCommand("docker", killArgs...)
+ _, err := common.RunCommand("docker", killArgs...)
  if err != nil {
  log.Warningf("Ignored error from 'docker kill --signal TERM': %s", err)
  }
@@ -219,3 +205,28 @@ func (d *DockerExecutor) Kill(containerIDs []string) error {
  log.Infof("Containers %s killed successfully", containerIDs)
  return nil
 }
+
+// getDockerCpCmd uses os-specific code to run `docker cp` and gzip/7zip to copy gzipped data from another
+// container.
+func getDockerCpCmd(containerID, sourcePath string, compressionLevel int, destPath string) string {
+ gzipCmd := "gzip %s > %s"
+ levelFlagParam := "-"
+ if runtime.GOOS == "windows" {
+ gzipCmd = "7za.exe a -tgzip -si %s %s"
+ levelFlagParam = "-mx"
+ }
+
+ var levelFlag string
+ switch compressionLevel {
+ case gzip.NoCompression:
+ // best we can do - if we skip gzip it's a different file
+ levelFlag = levelFlagParam + "1"
+ case gzip.DefaultCompression:
+ // use cmd default
+ levelFlag = ""
+ default:
+ // -1 through -9 (or error)
+ levelFlag = levelFlagParam + strconv.Itoa(compressionLevel)
+ }
+ return fmt.Sprintf("docker cp -a %s:%s - | %s", containerID, sourcePath, fmt.Sprintf(gzipCmd, levelFlag, destPath))
+}

workflow/executor/executor.go

@@ -854,7 +854,7 @@ func untar(tarPath string, destPath string) error {
  if err != nil {
  return errors.InternalWrapError(err)
  }
- err = common.RunCommand("tar", "-xf", tarPath, "-C", tmpDir)
+ _, err = common.RunCommand("tar", "-xf", tarPath, "-C", tmpDir)
  if err != nil {
  return err
  }