Server: Enable multi platform builds (amd64, armv7 and arm64) (lauren…

…t22#5338)

.github/workflows/github-actions-main.yml

@@ -35,6 +35,14 @@ jobs:
  sudo apt-get update || true
  sudo apt-get install -y docker-ce docker-ce-cli containerd.io
 
+ # the next line enables multi-architecture support for docker, it basically makes it use qemu for non native platforms
+ # See https://hub.docker.com/r/tonistiigi/binfmt for more info
+ docker run --privileged --rm tonistiigi/binfmt --install all
+
+ # this just prints the info about what platforms are supported in the builder (can help debugging if something isn't working right)
+ # and also proves the above worked properly
+ sudo docker buildx ls
+
  - uses: actions/checkout@v2
  - uses: olegtarasov/[email protected]
  - uses: actions/setup-node@v2

packages/tools/buildServerDocker.ts

@@ -1,6 +1,8 @@
 import { execCommand2, rootDir } from './tool-utils';
 import * as moment from 'moment';
 
+const DockerImageName = 'joplin/server';
+
 function getVersionFromTag(tagName: string, isPreRelease: boolean): string {
  if (tagName.indexOf('server-') !== 0) throw new Error(`Invalid tag: ${tagName}`);
  const s = tagName.split('-');
@@ -12,6 +14,10 @@ function getIsPreRelease(tagName: string): boolean {
  return tagName.indexOf('-beta') > 0;
 }
 
+function normalizePlatform(platform: string) {
+ return platform.replace(/\//g, '-');
+}
+
 async function main() {
  const argv = require('yargs').argv;
  if (!argv.tagName) throw new Error('--tag-name not provided');
@@ -27,7 +33,11 @@ async function main() {
  } catch (error) {
  console.info('Could not get git commit: metadata revision field will be empty');
  }
- const buildArgs = `--build-arg BUILD_DATE="${buildDate}" --build-arg REVISION="${revision}" --build-arg VERSION="${imageVersion}"`;
+ const buildArgs = [
+ `--build-arg BUILD_DATE="${buildDate}"`,
+ `--build-arg REVISION="${revision}"`,
+ `--build-arg VERSION="${imageVersion}"`,
+ ];
  const dockerTags: string[] = [];
  const versionPart = imageVersion.split('.');
  dockerTags.push(isPreRelease ? 'beta' : 'latest');
@@ -44,10 +54,48 @@ async function main() {
  console.info('isPreRelease:', isPreRelease);
  console.info('Docker tags:', dockerTags.join(', '));
 
- await execCommand2(`docker build -t "joplin/server:${imageVersion}" ${buildArgs} -f Dockerfile.server .`);
- for (const tag of dockerTags) {
- await execCommand2(`docker tag "joplin/server:${imageVersion}" "joplin/server:${tag}"`);
- if (pushImages) await execCommand2(`docker push joplin/server:${tag}`);
+ const platforms = [
+ 'linux/amd64',
+ 'linux/arm64',
+ 'linux/arm/v7',
+ ];
+
+ // this will build a bunch of local image tags named: ${imageVersion}-${platform} with the slashes replaced with dashes
+ for (const platform of platforms) {
+ const normalizedPlatform = normalizePlatform(platform);
+ await execCommand2([
+ 'docker', 'build',
+ '--platform', platform,
+ '-t', `${DockerImageName}:${imageVersion}-${normalizedPlatform}`,
+ ...buildArgs,
+ '-f', 'Dockerfile.server',
+ '.',
+ ]);
+ if (pushImages) {
+ await execCommand2([
+ 'docker', 'push', `${DockerImageName}:${imageVersion}-${normalizedPlatform}`,
+ ]);
+ }
+ }
+
+ // now we have to create the right manifests and push them
+ if (pushImages) {
+ for (const tag of dockerTags) {
+ // manifest create requires the tags being amended in to exist on the remote, so this all can only happen if pushImages is true
+ const platformArgs: string[] = [];
+ for (const platform in platforms) {
+ platformArgs.concat('--amend', `${DockerImageName}:${imageVersion}-${normalizePlatform(platform)}`);
+ }
+ await execCommand2([
+ 'docker', 'manifest', 'create',
+ `${DockerImageName}:${tag}`,
+ ...platformArgs,
+ ]);
+ await execCommand2([
+ 'docker', 'manifest', 'push',
+ `${DockerImageName}:${tag}`,
+ ]);
+ }
  }
 }
 

packages/tools/tool-utils.ts

@@ -147,6 +147,7 @@ export function execCommandVerbose(commandName: string, args: string[] = []) {
 interface ExecCommandOptions {
  showInput?: boolean;
  showOutput?: boolean;
+ showError?: boolean;
  quiet?: boolean;
 }
 
@@ -160,13 +161,15 @@ export async function execCommand2(command: string | string[], options: ExecComm
  options = {
  showInput: true,
  showOutput: true,
+ showError: true,
  quiet: false,
  ...options,
  };
 
  if (options.quiet) {
  options.showInput = false;
  options.showOutput = false;
+ options.showError = false;
  }
 
  if (options.showInput) {
@@ -182,6 +185,7 @@ export async function execCommand2(command: string | string[], options: ExecComm
  args.splice(0, 1);
  const promise = execa(executableName, args);
  if (options.showOutput) promise.stdout.pipe(process.stdout);
+ if (options.showError) promise.stderr.pipe(process.stderr);
  const result = await promise;
  return result.stdout.trim();
 }