From 1f22f9faf91b62d45b8cbc241deed39faa48c3e0 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Fri, 15 Mar 2024 14:53:33 +0000 Subject: [PATCH 01/72] adding debug for mac upload testing --- packages/artifact/package-lock.json | 28 ++++++++++++++++++- packages/artifact/package.json | 3 +- .../src/internal/upload/upload-artifact.ts | 8 ++++++ packages/artifact/src/internal/upload/zip.ts | 1 - 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index 4ce3c7fb7d..4222a676cd 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -22,7 +22,8 @@ "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", - "unzip-stream": "^0.3.1" + "unzip-stream": "^0.3.1", + "why-is-node-running": "^2.2.2" }, "devDependencies": { "@types/archiver": "^5.3.2", @@ -1240,6 +1241,11 @@ "vscode-textmate": "^8.0.0" } }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==" + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -1249,6 +1255,11 @@ "node": ">=0.10.0" } }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==" + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -1471,6 +1482,21 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/why-is-node-running": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.2.2.tgz", + "integrity": "sha512-6tSwToZxTOcotxHeA+qGCq1mVzKR3CwcJGmVcY+QE8SHy6TnpFnh8PAvPNHYr7EcuVeG0QSMxtYCuO1ta/G/oA==", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wordwrap": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", diff --git a/packages/artifact/package.json b/packages/artifact/package.json index c155708139..a246fdc79e 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -53,7 +53,8 @@ "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", - "unzip-stream": "^0.3.1" + "unzip-stream": "^0.3.1", + "why-is-node-running": "^2.2.2" }, "devDependencies": { "@types/archiver": "^5.3.2", diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index e880102fe5..a4f1ef6cb1 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -1,3 +1,4 @@ +import whyIsNodeRunning from 'why-is-node-running' import * as core from '@actions/core' import { UploadArtifactOptions, @@ -107,6 +108,13 @@ export async function uploadArtifact( core.info( `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}` ) + if (core.isDebug()) { + setTimeout(function () { + core.debug('Processes keeping upload stream running:') + whyIsNodeRunning() + }, 500) + } + // return { size: uploadResult.uploadSize, diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 5a9231193c..d7a748fec1 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -29,7 +29,6 @@ export async function createZipUploadStream( core.debug( `Creating Artifact archive with compressionLevel: ${compressionLevel}` ) - const zip = archiver.create('zip', { highWaterMark: getUploadChunkSize(), zlib: {level: compressionLevel} From 4617da254f11ae6052dcd3de6d4869f2bcc8b05d Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Fri, 15 Mar 2024 15:01:45 +0000 Subject: [PATCH 02/72] audit fix --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7575e42936..63f54a4de0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5781,9 +5781,9 @@ } }, "node_modules/follow-redirects": { - "version": "1.15.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", - "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "dev": true, "funding": [ { From 4090623397a8ef8e536aeff39f3184d6c2874b57 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 04:20:40 +0000 Subject: [PATCH 03/72] update debug statement --- packages/artifact/src/internal/upload/upload-artifact.ts | 4 ++-- packages/artifact/src/internal/upload/zip.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index a4f1ef6cb1..d406733fed 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -108,12 +108,12 @@ export async function uploadArtifact( core.info( `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}` ) - if (core.isDebug()) { + // if (core.isDebug()) { setTimeout(function () { core.debug('Processes keeping upload stream running:') whyIsNodeRunning() }, 500) - } + // } // return { diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index d7a748fec1..4b968dd446 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -29,6 +29,7 @@ export async function createZipUploadStream( core.debug( `Creating Artifact archive with compressionLevel: ${compressionLevel}` ) + const zip = archiver.create('zip', { highWaterMark: getUploadChunkSize(), zlib: {level: compressionLevel} From 13d981a863ca6e053e964ed8ec4da09f352e1861 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 14:32:03 +0000 Subject: [PATCH 04/72] Adding buffer and zlib options --- package-lock.json | 6 +- package.json | 2 +- packages/artifact/package-lock.json | 691 +++++++++++++++++- packages/artifact/package.json | 3 +- .../src/internal/upload/upload-artifact.ts | 8 +- packages/artifact/src/internal/upload/zip.ts | 22 +- packages/http-client/package-lock.json | 2 +- 7 files changed, 699 insertions(+), 35 deletions(-) diff --git a/package-lock.json b/package-lock.json index 63f54a4de0..d1395e3293 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7184,9 +7184,9 @@ } }, "node_modules/jackspeak": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.2.2.tgz", - "integrity": "sha512-mgNtVv4vUuaKA97yxUHoA3+FkuhtxkjdXEWOyB/N76fjy0FjezEt34oy3epBtvCvS+7DyKwqCFWx/oJLV5+kCg==", + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", "dev": true, "dependencies": { "@isaacs/cliui": "^8.0.2" diff --git a/package.json b/package.json index ea3ac2ed9d..d8c6012f34 100644 --- a/package.json +++ b/package.json @@ -33,4 +33,4 @@ "ts-jest": "^29.1.1", "typescript": "^5.2.2" } -} \ No newline at end of file +} diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index 4222a676cd..41b0c8d529 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -23,7 +23,8 @@ "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", "unzip-stream": "^0.3.1", - "why-is-node-running": "^2.2.2" + "why-is-node-running": "^2.2.2", + "zip-stream": "^6.0.1" }, "devDependencies": { "@types/archiver": "^5.3.2", @@ -187,6 +188,22 @@ "node": ">=14.0.0" } }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@octokit/auth-token": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz", @@ -349,6 +366,15 @@ "node": ">=8.0.0" } }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@protobuf-ts/plugin": { "version": "2.9.1", "resolved": "https://registry.npmjs.org/@protobuf-ts/plugin/-/plugin-2.9.1.tgz", @@ -481,12 +507,45 @@ "@types/node": "*" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, "node_modules/ansi-sequence-parser": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ansi-sequence-parser/-/ansi-sequence-parser-1.1.1.tgz", "integrity": "sha512-vJXt3yiaUL4UU546s3rPXlsry/RnM730G1+HkpKE012AN0sx1eOrxSu95oKDIonskeLTijMgqWZ3uDEe3NFvyg==", "dev": true }, + "node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/archiver": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.1.tgz", @@ -551,6 +610,65 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/archiver/node_modules/compress-commons": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.2.tgz", + "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", + "dependencies": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^4.0.2", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver/node_modules/crc32-stream": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.3.tgz", + "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^3.4.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver/node_modules/zip-stream": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", + "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", + "dependencies": { + "archiver-utils": "^3.0.4", + "compress-commons": "^4.1.2", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver/node_modules/zip-stream/node_modules/archiver-utils": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-3.0.4.tgz", + "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", + "dependencies": { + "glob": "^7.2.3", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/async": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", @@ -685,6 +803,22 @@ "node": "*" } }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -705,17 +839,56 @@ } }, "node_modules/compress-commons": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.1.tgz", - "integrity": "sha512-QLdDLCKNV2dtoTorqgxngQCMA+gWXkM/Nwu7FpeBhk/RdkzimqC3jueb/FDmaZeXh+uby1jkBqE3xArsLBE5wQ==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-6.0.2.tgz", + "integrity": "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==", "dependencies": { - "buffer-crc32": "^0.2.13", - "crc32-stream": "^4.0.2", + "crc-32": "^1.2.0", + "crc32-stream": "^6.0.0", + "is-stream": "^2.0.1", "normalize-path": "^3.0.0", - "readable-stream": "^3.6.0" + "readable-stream": "^4.0.0" }, "engines": { - "node": ">= 10" + "node": ">= 14" + } + }, + "node_modules/compress-commons/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/compress-commons/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, "node_modules/concat-map": { @@ -740,15 +913,66 @@ } }, "node_modules/crc32-stream": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.2.tgz", - "integrity": "sha512-DxFZ/Hk473b/muq1VJ///PMNLj0ZMnzye9thBpmjpJKCc5eMgB95aK8zCGrGfQ90cWo561Te6HK9D+j4KPdM6w==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-6.0.0.tgz", + "integrity": "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==", "dependencies": { "crc-32": "^1.2.0", - "readable-stream": "^3.4.0" + "readable-stream": "^4.0.0" }, "engines": { - "node": ">= 10" + "node": ">= 14" + } + }, + "node_modules/crc32-stream/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/crc32-stream/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" } }, "node_modules/crypto": { @@ -782,6 +1006,16 @@ "dot-object": "bin/dot-object" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -790,6 +1024,14 @@ "once": "^1.4.0" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, "node_modules/events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -798,6 +1040,21 @@ "node": ">=0.8.x" } }, + "node_modules/foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/form-data": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", @@ -899,6 +1156,14 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, "node_modules/is-plain-object": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", @@ -907,11 +1172,44 @@ "node": ">=0.10.0" } }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "node_modules/jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/jsonc-parser": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", @@ -999,6 +1297,14 @@ "tslib": "^2.0.3" } }, + "node_modules/lru-cache": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz", + "integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==", + "engines": { + "node": "14 || >=16.14" + } + }, "node_modules/lunr": { "version": "2.3.9", "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.9.tgz", @@ -1055,6 +1361,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/mkdirp": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", @@ -1133,6 +1447,29 @@ "node": ">=0.10.0" } }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", + "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/path-to-regexp": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.1.tgz", @@ -1229,6 +1566,25 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "engines": { + "node": ">=8" + } + }, "node_modules/shiki": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.14.5.tgz", @@ -1246,6 +1602,17 @@ "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==" }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -1268,6 +1635,94 @@ "safe-buffer": "~5.2.0" } }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, "node_modules/tar-stream": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", @@ -1482,6 +1937,20 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/why-is-node-running": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.2.2.tgz", @@ -1503,6 +1972,90 @@ "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", "dev": true }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -1537,16 +2090,114 @@ } }, "node_modules/zip-stream": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.0.tgz", - "integrity": "sha512-zshzwQW7gG7hjpBlgeQP9RuyPGNxvJdzR8SUM3QhxCnLjWN2E7j3dOvpeDcQoETfHx0urRS7EtmVToql7YpU4A==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-6.0.1.tgz", + "integrity": "sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==", "dependencies": { - "archiver-utils": "^2.1.0", - "compress-commons": "^4.1.0", - "readable-stream": "^3.6.0" + "archiver-utils": "^5.0.0", + "compress-commons": "^6.0.2", + "readable-stream": "^4.0.0" }, "engines": { - "node": ">= 10" + "node": ">= 14" + } + }, + "node_modules/zip-stream/node_modules/archiver-utils": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-5.0.2.tgz", + "integrity": "sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==", + "dependencies": { + "glob": "^10.0.0", + "graceful-fs": "^4.2.0", + "is-stream": "^2.0.1", + "lazystream": "^1.0.0", + "lodash": "^4.17.15", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/zip-stream/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/zip-stream/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/zip-stream/node_modules/glob": { + "version": "10.3.10", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", + "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.5", + "minimatch": "^9.0.1", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0", + "path-scurry": "^1.10.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/zip-stream/node_modules/minimatch": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", + "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/zip-stream/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } } } diff --git a/packages/artifact/package.json b/packages/artifact/package.json index a246fdc79e..94ca3d5d72 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -54,7 +54,8 @@ "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", "unzip-stream": "^0.3.1", - "why-is-node-running": "^2.2.2" + "why-is-node-running": "^2.2.2", + "zip-stream": "^6.0.1" }, "devDependencies": { "@types/archiver": "^5.3.2", diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index d406733fed..c15e35e7d6 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -109,10 +109,10 @@ export async function uploadArtifact( `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}` ) // if (core.isDebug()) { - setTimeout(function () { - core.debug('Processes keeping upload stream running:') - whyIsNodeRunning() - }, 500) + setTimeout(function () { + core.debug('Processes keeping upload stream running:') + whyIsNodeRunning() + }, 500) // } // diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 4b968dd446..8e15580a06 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -1,5 +1,7 @@ import * as stream from 'stream' import * as archiver from 'archiver' +import * as packer from 'zip-stream' + import * as core from '@actions/core' import {createReadStream} from 'fs' import {UploadZipSpecification} from './upload-zip-specification' @@ -29,12 +31,22 @@ export async function createZipUploadStream( core.debug( `Creating Artifact archive with compressionLevel: ${compressionLevel}` ) - - const zip = archiver.create('zip', { - highWaterMark: getUploadChunkSize(), - zlib: {level: compressionLevel} - }) + // const zip2 = archiver.create('zip', { + // highWaterMark: getUploadChunkSize(), + // zlib: {level: compressionLevel} + // }) + const zlibOptions = { + zlib: {level: compressionLevel, bufferSize: getUploadChunkSize()} + } + const zip = new packer(zlibOptions) + // zip.entry('string contents', {name: 'string.txt'}, function (err, entry) { + // if (err) throw err + // zip.entry(null, {name: 'directory/'}, function (err, entry) { + // if (err) throw err + // zip.finish() + // }) + // }) // register callbacks for various events during the zip lifecycle zip.on('error', zipErrorCallback) zip.on('warning', zipWarningCallback) diff --git a/packages/http-client/package-lock.json b/packages/http-client/package-lock.json index ea3d79510b..52038ad373 100644 --- a/packages/http-client/package-lock.json +++ b/packages/http-client/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "@actions/http-client", - "version": "2.2.0", + "version": "2.2.1", "license": "MIT", "dependencies": { "tunnel": "^0.0.6", From e71ddb9e5a78b34b509c6a6a19ada09f76b8f4df Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 14:44:20 +0000 Subject: [PATCH 05/72] updating append and directory zips --- packages/artifact/src/internal/upload/zip.ts | 22 ++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 8e15580a06..0a10b18ff8 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -56,13 +56,27 @@ export async function createZipUploadStream( for (const file of uploadSpecification) { if (file.sourcePath !== null) { // Add a normal file to the zip - zip.append(createReadStream(file.sourcePath), { - name: file.destinationPath - }) + // zip.append(createReadStream(file.sourcePath), { + // name: file.destinationPath + // }) + zip.entry( + createReadStream(file.sourcePath), + {name: file.destinationPath}, + function (err, entry) { + core.debug(`Entry is: ${entry}`) + if (err) throw err + } + ) } else { + zip.entry(null, {name: file.destinationPath}, function (err, entry) { + core.debug(`Entry is: ${entry}`) + if (err) throw err + }) // Add a directory to the zip - zip.append('', {name: file.destinationPath}) + // zip.append('', {name: file.destinationPath}) } + + zip.finish() } const bufferSize = getUploadChunkSize() From 14eee6f54a3b050d92bb24199b9002a399e53244 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 15:10:34 +0000 Subject: [PATCH 06/72] updating event handlers --- .../src/internal/upload/upload-artifact.ts | 8 +- packages/artifact/src/internal/upload/zip.ts | 100 +++++++++++------- 2 files changed, 67 insertions(+), 41 deletions(-) diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index c15e35e7d6..9549d0e6fa 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -109,10 +109,10 @@ export async function uploadArtifact( `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}` ) // if (core.isDebug()) { - setTimeout(function () { - core.debug('Processes keeping upload stream running:') - whyIsNodeRunning() - }, 500) + // setTimeout(function () { + // core.debug('Processes keeping upload stream running:') + // whyIsNodeRunning() + // }, 500) // } // diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 0a10b18ff8..483a863199 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -48,10 +48,37 @@ export async function createZipUploadStream( // }) // }) // register callbacks for various events during the zip lifecycle - zip.on('error', zipErrorCallback) - zip.on('warning', zipWarningCallback) - zip.on('finish', zipFinishCallback) - zip.on('end', zipEndCallback) + zip.on('error', err => { + core.error('An error has occurred while creating the zip file for upload') + core.info(err) + + throw new Error( + 'An error has occurred during zip creation for the artifact' + ) + }) + zip.on('warning', err => { + if (err.code === 'ENOENT') { + core.warning( + 'ENOENT warning during artifact zip creation. No such file or directory' + ) + core.info(err) + } else { + core.warning( + `A non-blocking warning has occurred during artifact zip creation: ${err.code}` + ) + core.info(err) + } + }) + + // zip.on('error', zipErrorCallback) + // zip.on('warning', zipWarningCallback) + zip.on('finish', () => { + core.debug('Zip stream for upload has finished.') + }) + zip.on('end', () => { + core.debug('Zip stream for upload has ended.') + }) + // zip.on('end', zipEndCallback) for (const file of uploadSpecification) { if (file.sourcePath !== null) { @@ -65,18 +92,18 @@ export async function createZipUploadStream( function (err, entry) { core.debug(`Entry is: ${entry}`) if (err) throw err + // zip.finish() } ) } else { zip.entry(null, {name: file.destinationPath}, function (err, entry) { core.debug(`Entry is: ${entry}`) if (err) throw err + // zip.finish() }) // Add a directory to the zip // zip.append('', {name: file.destinationPath}) } - - zip.finish() } const bufferSize = getUploadChunkSize() @@ -90,38 +117,37 @@ export async function createZipUploadStream( ) zip.pipe(zipUploadStream) + // zip.finalize() zip.finalize() - return zipUploadStream } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const zipErrorCallback = (error: any): void => { - core.error('An error has occurred while creating the zip file for upload') - core.info(error) - - throw new Error('An error has occurred during zip creation for the artifact') -} - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const zipWarningCallback = (error: any): void => { - if (error.code === 'ENOENT') { - core.warning( - 'ENOENT warning during artifact zip creation. No such file or directory' - ) - core.info(error) - } else { - core.warning( - `A non-blocking warning has occurred during artifact zip creation: ${error.code}` - ) - core.info(error) - } -} - -const zipFinishCallback = (): void => { - core.debug('Zip stream for upload has finished.') -} - -const zipEndCallback = (): void => { - core.debug('Zip stream for upload has ended.') -} +// const zipErrorCallback = (error: any): void => { +// core.error('An error has occurred while creating the zip file for upload') +// core.info(error) + +// throw new Error('An error has occurred during zip creation for the artifact') +// } + +// // eslint-disable-next-line @typescript-eslint/no-explicit-any +// const zipWarningCallback = (error: any): void => { +// if (error.code === 'ENOENT') { +// core.warning( +// 'ENOENT warning during artifact zip creation. No such file or directory' +// ) +// core.info(error) +// } else { +// core.warning( +// `A non-blocking warning has occurred during artifact zip creation: ${error.code}` +// ) +// core.info(error) +// } +// } + +// const zipFinishCallback = (): void => { +// core.debug('Zip stream for upload has finished.') +// } + +// const zipEndCallback = (): void => { +// core.debug('Zip stream for upload has ended.') +// } From 6e4aebbd4e9657999e9f18435a144f578d489b8d Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 15:17:40 +0000 Subject: [PATCH 07/72] cleaning up unused imports --- .../src/internal/upload/upload-artifact.ts | 1 - packages/artifact/src/internal/upload/zip.ts | 24 ------------------- 2 files changed, 25 deletions(-) diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index 9549d0e6fa..2e209348ca 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -1,4 +1,3 @@ -import whyIsNodeRunning from 'why-is-node-running' import * as core from '@actions/core' import { UploadArtifactOptions, diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 483a863199..cacfdce0d0 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -1,5 +1,4 @@ import * as stream from 'stream' -import * as archiver from 'archiver' import * as packer from 'zip-stream' import * as core from '@actions/core' @@ -31,22 +30,10 @@ export async function createZipUploadStream( core.debug( `Creating Artifact archive with compressionLevel: ${compressionLevel}` ) - // const zip2 = archiver.create('zip', { - // highWaterMark: getUploadChunkSize(), - // zlib: {level: compressionLevel} - // }) - const zlibOptions = { zlib: {level: compressionLevel, bufferSize: getUploadChunkSize()} } const zip = new packer(zlibOptions) - // zip.entry('string contents', {name: 'string.txt'}, function (err, entry) { - // if (err) throw err - // zip.entry(null, {name: 'directory/'}, function (err, entry) { - // if (err) throw err - // zip.finish() - // }) - // }) // register callbacks for various events during the zip lifecycle zip.on('error', err => { core.error('An error has occurred while creating the zip file for upload') @@ -70,39 +57,29 @@ export async function createZipUploadStream( } }) - // zip.on('error', zipErrorCallback) - // zip.on('warning', zipWarningCallback) zip.on('finish', () => { core.debug('Zip stream for upload has finished.') }) zip.on('end', () => { core.debug('Zip stream for upload has ended.') }) - // zip.on('end', zipEndCallback) for (const file of uploadSpecification) { if (file.sourcePath !== null) { // Add a normal file to the zip - // zip.append(createReadStream(file.sourcePath), { - // name: file.destinationPath - // }) zip.entry( createReadStream(file.sourcePath), {name: file.destinationPath}, function (err, entry) { core.debug(`Entry is: ${entry}`) if (err) throw err - // zip.finish() } ) } else { zip.entry(null, {name: file.destinationPath}, function (err, entry) { core.debug(`Entry is: ${entry}`) if (err) throw err - // zip.finish() }) - // Add a directory to the zip - // zip.append('', {name: file.destinationPath}) } } @@ -117,7 +94,6 @@ export async function createZipUploadStream( ) zip.pipe(zipUploadStream) - // zip.finalize() zip.finalize() return zipUploadStream } From 6defe19df1c32d8828489d699f401ca378dad99b Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 15:25:37 +0000 Subject: [PATCH 08/72] replacing constructor for zipstream --- packages/artifact/src/internal/upload/zip.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index cacfdce0d0..f80778d745 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -1,6 +1,5 @@ import * as stream from 'stream' -import * as packer from 'zip-stream' - +import {ZipStream} from 'zip-stream' import * as core from '@actions/core' import {createReadStream} from 'fs' import {UploadZipSpecification} from './upload-zip-specification' @@ -33,7 +32,7 @@ export async function createZipUploadStream( const zlibOptions = { zlib: {level: compressionLevel, bufferSize: getUploadChunkSize()} } - const zip = new packer(zlibOptions) + const zip = new ZipStream(zlibOptions) // register callbacks for various events during the zip lifecycle zip.on('error', err => { core.error('An error has occurred while creating the zip file for upload') From fda9d58d069c80eaad01a8bdc834ea6c96df8ff7 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 15:28:14 +0000 Subject: [PATCH 09/72] replacing constructor for zipstream with default --- packages/artifact/src/internal/upload/zip.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index f80778d745..4cb1da7c51 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -1,5 +1,5 @@ import * as stream from 'stream' -import {ZipStream} from 'zip-stream' +import * as ZipStream from 'zip-stream' import * as core from '@actions/core' import {createReadStream} from 'fs' import {UploadZipSpecification} from './upload-zip-specification' @@ -32,7 +32,7 @@ export async function createZipUploadStream( const zlibOptions = { zlib: {level: compressionLevel, bufferSize: getUploadChunkSize()} } - const zip = new ZipStream(zlibOptions) + const zip = new ZipStream.default(zlibOptions) // register callbacks for various events during the zip lifecycle zip.on('error', err => { core.error('An error has occurred while creating the zip file for upload') From 560ae4debd34087d83582ea1e3ceae9d2a7e23bc Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 15:41:52 +0000 Subject: [PATCH 10/72] updating archive import --- packages/artifact/package-lock.json | 282 ------------------- packages/artifact/package.json | 4 +- packages/artifact/src/internal/upload/zip.ts | 2 +- 3 files changed, 2 insertions(+), 286 deletions(-) diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index 41b0c8d529..dcfbda7987 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -18,7 +18,6 @@ "@octokit/plugin-retry": "^3.0.9", "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", - "archiver": "^5.3.1", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", @@ -27,7 +26,6 @@ "zip-stream": "^6.0.1" }, "devDependencies": { - "@types/archiver": "^5.3.2", "@types/unzip-stream": "^0.3.4", "typedoc": "^0.25.4", "typedoc-plugin-markdown": "^3.17.1", @@ -445,15 +443,6 @@ "@protobuf-ts/runtime": "^2.9.1" } }, - "node_modules/@types/archiver": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-5.3.2.tgz", - "integrity": "sha512-IctHreBuWE5dvBDz/0WeKtyVKVRs4h75IblxOACL92wU66v+HGAfEYAOyXkOFphvRJMhuXdI9huDXpX0FC6lCw==", - "dev": true, - "dependencies": { - "@types/readdir-glob": "*" - } - }, "node_modules/@types/node": { "version": "20.4.9", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.4.9.tgz", @@ -481,15 +470,6 @@ "node": ">= 6" } }, - "node_modules/@types/readdir-glob": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@types/readdir-glob/-/readdir-glob-1.1.1.tgz", - "integrity": "sha512-ImM6TmoF8bgOwvehGviEj3tRdRBbQujr1N+0ypaln/GWjaerOB26jb93vsRHmdMtvVQZQebOlqt2HROark87mQ==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/tunnel": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", @@ -546,134 +526,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/archiver": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.1.tgz", - "integrity": "sha512-8KyabkmbYrH+9ibcTScQ1xCJC/CGcugdVIwB+53f5sZziXgwUh3iXlAlANMxcZyDEfTHMe6+Z5FofV8nopXP7w==", - "dependencies": { - "archiver-utils": "^2.1.0", - "async": "^3.2.3", - "buffer-crc32": "^0.2.1", - "readable-stream": "^3.6.0", - "readdir-glob": "^1.0.0", - "tar-stream": "^2.2.0", - "zip-stream": "^4.1.0" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/archiver-utils": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", - "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", - "dependencies": { - "glob": "^7.1.4", - "graceful-fs": "^4.2.0", - "lazystream": "^1.0.0", - "lodash.defaults": "^4.2.0", - "lodash.difference": "^4.5.0", - "lodash.flatten": "^4.4.0", - "lodash.isplainobject": "^4.0.6", - "lodash.union": "^4.6.0", - "normalize-path": "^3.0.0", - "readable-stream": "^2.0.0" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/archiver-utils/node_modules/readable-stream": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", - "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/archiver-utils/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/archiver-utils/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/archiver/node_modules/compress-commons": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.2.tgz", - "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", - "dependencies": { - "buffer-crc32": "^0.2.13", - "crc32-stream": "^4.0.2", - "normalize-path": "^3.0.0", - "readable-stream": "^3.6.0" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/archiver/node_modules/crc32-stream": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.3.tgz", - "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", - "dependencies": { - "crc-32": "^1.2.0", - "readable-stream": "^3.4.0" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/archiver/node_modules/zip-stream": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", - "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", - "dependencies": { - "archiver-utils": "^3.0.4", - "compress-commons": "^4.1.2", - "readable-stream": "^3.6.0" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/archiver/node_modules/zip-stream/node_modules/archiver-utils": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-3.0.4.tgz", - "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", - "dependencies": { - "glob": "^7.2.3", - "graceful-fs": "^4.2.0", - "lazystream": "^1.0.0", - "lodash.defaults": "^4.2.0", - "lodash.difference": "^4.5.0", - "lodash.flatten": "^4.4.0", - "lodash.isplainobject": "^4.0.6", - "lodash.union": "^4.6.0", - "normalize-path": "^3.0.0", - "readable-stream": "^3.6.0" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/async": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -720,16 +572,6 @@ "node": "*" } }, - "node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, "node_modules/bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", @@ -744,37 +586,6 @@ "concat-map": "0.0.1" } }, - "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "engines": { - "node": "*" - } - }, "node_modules/buffers": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", @@ -1016,14 +827,6 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, - "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dependencies": { - "once": "^1.4.0" - } - }, "node_modules/event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -1068,11 +871,6 @@ "node": ">= 6" } }, - "node_modules/fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" - }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1264,31 +1062,6 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, - "node_modules/lodash.defaults": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", - "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" - }, - "node_modules/lodash.difference": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", - "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==" - }, - "node_modules/lodash.flatten": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", - "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==" - }, - "node_modules/lodash.isplainobject": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==" - }, - "node_modules/lodash.union": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", - "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==" - }, "node_modules/lower-case": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", @@ -1502,46 +1275,6 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, - "node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/readdir-glob": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", - "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", - "dependencies": { - "minimatch": "^5.1.0" - } - }, - "node_modules/readdir-glob/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/readdir-glob/node_modules/minimatch": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", - "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1723,21 +1456,6 @@ "node": ">=8" } }, - "node_modules/tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "dependencies": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", diff --git a/packages/artifact/package.json b/packages/artifact/package.json index 94ca3d5d72..222f1095ae 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -49,7 +49,6 @@ "@octokit/plugin-retry": "^3.0.9", "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", - "archiver": "^5.3.1", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", @@ -58,10 +57,9 @@ "zip-stream": "^6.0.1" }, "devDependencies": { - "@types/archiver": "^5.3.2", "@types/unzip-stream": "^0.3.4", "typedoc": "^0.25.4", "typedoc-plugin-markdown": "^3.17.1", "typescript": "^5.2.2" } -} +} \ No newline at end of file diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 4cb1da7c51..2d253544e4 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -32,7 +32,7 @@ export async function createZipUploadStream( const zlibOptions = { zlib: {level: compressionLevel, bufferSize: getUploadChunkSize()} } - const zip = new ZipStream.default(zlibOptions) + const zip = new ZipStream(zlibOptions) // register callbacks for various events during the zip lifecycle zip.on('error', err => { core.error('An error has occurred while creating the zip file for upload') From 8e29fb2e30e4637400c7414280ecd5c25026d685 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 15:44:27 +0000 Subject: [PATCH 11/72] adding default back to zipstream call --- packages/artifact/src/internal/upload/zip.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 2d253544e4..4cb1da7c51 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -32,7 +32,7 @@ export async function createZipUploadStream( const zlibOptions = { zlib: {level: compressionLevel, bufferSize: getUploadChunkSize()} } - const zip = new ZipStream(zlibOptions) + const zip = new ZipStream.default(zlibOptions) // register callbacks for various events during the zip lifecycle zip.on('error', err => { core.error('An error has occurred while creating the zip file for upload') From 97629b8767ed826bc51da6b7e87365a8afc8c602 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 15:47:34 +0000 Subject: [PATCH 12/72] adding archiver back for download --- packages/artifact/package-lock.json | 282 ++++++++++++++++++++++++++++ packages/artifact/package.json | 2 + 2 files changed, 284 insertions(+) diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index dcfbda7987..9710156d7b 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -18,6 +18,7 @@ "@octokit/plugin-retry": "^3.0.9", "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", + "archiver": "^5.3.1", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", @@ -26,6 +27,7 @@ "zip-stream": "^6.0.1" }, "devDependencies": { + "@types/archiver": "^5.3.2", "@types/unzip-stream": "^0.3.4", "typedoc": "^0.25.4", "typedoc-plugin-markdown": "^3.17.1", @@ -443,6 +445,15 @@ "@protobuf-ts/runtime": "^2.9.1" } }, + "node_modules/@types/archiver": { + "version": "5.3.4", + "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-5.3.4.tgz", + "integrity": "sha512-Lj7fLBIMwYFgViVVZHEdExZC3lVYsl+QL0VmdNdIzGZH544jHveYWij6qdnBgJQDnR7pMKliN9z2cPZFEbhyPw==", + "dev": true, + "dependencies": { + "@types/readdir-glob": "*" + } + }, "node_modules/@types/node": { "version": "20.4.9", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.4.9.tgz", @@ -470,6 +481,15 @@ "node": ">= 6" } }, + "node_modules/@types/readdir-glob": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@types/readdir-glob/-/readdir-glob-1.1.5.tgz", + "integrity": "sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/tunnel": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", @@ -526,6 +546,134 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/archiver": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.2.tgz", + "integrity": "sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw==", + "dependencies": { + "archiver-utils": "^2.1.0", + "async": "^3.2.4", + "buffer-crc32": "^0.2.1", + "readable-stream": "^3.6.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^2.2.0", + "zip-stream": "^4.1.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", + "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", + "dependencies": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/archiver-utils/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/archiver-utils/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/archiver-utils/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/archiver/node_modules/compress-commons": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.2.tgz", + "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", + "dependencies": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^4.0.2", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver/node_modules/crc32-stream": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.3.tgz", + "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^3.4.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver/node_modules/zip-stream": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", + "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", + "dependencies": { + "archiver-utils": "^3.0.4", + "compress-commons": "^4.1.2", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver/node_modules/zip-stream/node_modules/archiver-utils": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-3.0.4.tgz", + "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", + "dependencies": { + "glob": "^7.2.3", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/async": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", + "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -572,6 +720,16 @@ "node": "*" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", @@ -586,6 +744,37 @@ "concat-map": "0.0.1" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "engines": { + "node": "*" + } + }, "node_modules/buffers": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", @@ -827,6 +1016,14 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -871,6 +1068,11 @@ "node": ">= 6" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1062,6 +1264,31 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, + "node_modules/lodash.difference": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", + "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==" + }, + "node_modules/lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==" + }, + "node_modules/lodash.union": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", + "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==" + }, "node_modules/lower-case": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", @@ -1275,6 +1502,46 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1456,6 +1723,21 @@ "node": ">=8" } }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", diff --git a/packages/artifact/package.json b/packages/artifact/package.json index 222f1095ae..166438f4ab 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -49,6 +49,7 @@ "@octokit/plugin-retry": "^3.0.9", "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", + "archiver": "^5.3.1", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", @@ -57,6 +58,7 @@ "zip-stream": "^6.0.1" }, "devDependencies": { + "@types/archiver": "^5.3.2", "@types/unzip-stream": "^0.3.4", "typedoc": "^0.25.4", "typedoc-plugin-markdown": "^3.17.1", From ca87c5e1244e419cbedfa9130a51e0b8144046d9 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 15:53:03 +0000 Subject: [PATCH 13/72] testing debug statements --- packages/artifact/src/internal/upload/zip.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 4cb1da7c51..9d1ad656ed 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -26,9 +26,9 @@ export async function createZipUploadStream( uploadSpecification: UploadZipSpecification[], compressionLevel: number = DEFAULT_COMPRESSION_LEVEL ): Promise { - core.debug( - `Creating Artifact archive with compressionLevel: ${compressionLevel}` - ) + // core.debug( + // `Creating Artifact archive with compressionLevel: ${compressionLevel}` + // ) const zlibOptions = { zlib: {level: compressionLevel, bufferSize: getUploadChunkSize()} } @@ -85,12 +85,12 @@ export async function createZipUploadStream( const bufferSize = getUploadChunkSize() const zipUploadStream = new ZipUploadStream(bufferSize) - core.debug( - `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` - ) - core.debug( - `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` - ) + // core.debug( + // `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` + // ) + // core.debug( + // `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` + // ) zip.pipe(zipUploadStream) zip.finalize() From 7441cc7b8b99833373b035ca48cfffbfdec3fe65 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 15:55:09 +0000 Subject: [PATCH 14/72] Removing pipe --- packages/artifact/src/internal/upload/zip.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 9d1ad656ed..023c4b9aa8 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -92,7 +92,7 @@ export async function createZipUploadStream( // `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` // ) - zip.pipe(zipUploadStream) + // zip.pipe(zipUploadStream) zip.finalize() return zipUploadStream } From 06482c6da1f092615c4dc52782ac5321894701e2 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 16:06:54 +0000 Subject: [PATCH 15/72] adding promise to upload --- packages/artifact/src/internal/upload/zip.ts | 47 +++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 023c4b9aa8..06c596f86e 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -64,33 +64,36 @@ export async function createZipUploadStream( }) for (const file of uploadSpecification) { - if (file.sourcePath !== null) { - // Add a normal file to the zip - zip.entry( - createReadStream(file.sourcePath), - {name: file.destinationPath}, - function (err, entry) { + await new Promise((resolve, reject) => { + if (file.sourcePath !== null) { + // Add a normal file to the zip + zip.entry( + createReadStream(file.sourcePath), + {name: file.destinationPath}, + function (err, entry) { + core.debug(`Entry is: ${entry}`) + if (err) reject(err) + else resolve(entry) + } + ) + } else { + zip.entry(null, {name: file.destinationPath}, function (err, entry) { core.debug(`Entry is: ${entry}`) - if (err) throw err - } - ) - } else { - zip.entry(null, {name: file.destinationPath}, function (err, entry) { - core.debug(`Entry is: ${entry}`) - if (err) throw err - }) - } + if (err) reject(err) + resolve(entry) + }) + } + }) } const bufferSize = getUploadChunkSize() const zipUploadStream = new ZipUploadStream(bufferSize) - - // core.debug( - // `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` - // ) - // core.debug( - // `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` - // ) + core.debug( + `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` + ) + core.debug( + `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` + ) // zip.pipe(zipUploadStream) zip.finalize() From d97edf7f616b8ed631e3ee8662de0272144f587c Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 16:11:50 +0000 Subject: [PATCH 16/72] cleaning up warning callack --- packages/artifact/src/internal/upload/zip.ts | 48 +++++++------------- 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 06c596f86e..41e4b1debd 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -26,9 +26,9 @@ export async function createZipUploadStream( uploadSpecification: UploadZipSpecification[], compressionLevel: number = DEFAULT_COMPRESSION_LEVEL ): Promise { - // core.debug( - // `Creating Artifact archive with compressionLevel: ${compressionLevel}` - // ) + core.debug( + `Creating Artifact archive with compressionLevel: ${compressionLevel}` + ) const zlibOptions = { zlib: {level: compressionLevel, bufferSize: getUploadChunkSize()} } @@ -42,19 +42,7 @@ export async function createZipUploadStream( 'An error has occurred during zip creation for the artifact' ) }) - zip.on('warning', err => { - if (err.code === 'ENOENT') { - core.warning( - 'ENOENT warning during artifact zip creation. No such file or directory' - ) - core.info(err) - } else { - core.warning( - `A non-blocking warning has occurred during artifact zip creation: ${err.code}` - ) - core.info(err) - } - }) + zip.on('warning', zipWarningCallback) zip.on('finish', () => { core.debug('Zip stream for upload has finished.') @@ -106,21 +94,19 @@ export async function createZipUploadStream( // throw new Error('An error has occurred during zip creation for the artifact') // } - -// // eslint-disable-next-line @typescript-eslint/no-explicit-any -// const zipWarningCallback = (error: any): void => { -// if (error.code === 'ENOENT') { -// core.warning( -// 'ENOENT warning during artifact zip creation. No such file or directory' -// ) -// core.info(error) -// } else { -// core.warning( -// `A non-blocking warning has occurred during artifact zip creation: ${error.code}` -// ) -// core.info(error) -// } -// } +const zipWarningCallback = (err: any): void => { + if (err.code === 'ENOENT') { + core.warning( + 'ENOENT warning during artifact zip creation. No such file or directory' + ) + core.info(err) + } else { + core.warning( + `A non-blocking warning has occurred during artifact zip creation: ${err.code}` + ) + core.info(err) + } +} // const zipFinishCallback = (): void => { // core.debug('Zip stream for upload has finished.') From 088b9761b49871976326d73c8c13e6f8abe41bd8 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 16:15:29 +0000 Subject: [PATCH 17/72] cleaning up callback methods --- packages/artifact/src/internal/upload/zip.ts | 43 ++++++++------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 41e4b1debd..3aff9e361b 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -34,22 +34,11 @@ export async function createZipUploadStream( } const zip = new ZipStream.default(zlibOptions) // register callbacks for various events during the zip lifecycle - zip.on('error', err => { - core.error('An error has occurred while creating the zip file for upload') - core.info(err) - - throw new Error( - 'An error has occurred during zip creation for the artifact' - ) - }) + zip.on('error', zipErrorCallback) zip.on('warning', zipWarningCallback) - zip.on('finish', () => { - core.debug('Zip stream for upload has finished.') - }) - zip.on('end', () => { - core.debug('Zip stream for upload has ended.') - }) + zip.on('finish', zipFinishCallback) + zip.on('end', zipEndCallback) for (const file of uploadSpecification) { await new Promise((resolve, reject) => { @@ -82,18 +71,18 @@ export async function createZipUploadStream( core.debug( `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` ) - - // zip.pipe(zipUploadStream) zip.finalize() return zipUploadStream } -// const zipErrorCallback = (error: any): void => { -// core.error('An error has occurred while creating the zip file for upload') -// core.info(error) +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const zipErrorCallback = (error: any): void => { + core.error('An error has occurred while creating the zip file for upload') + core.info(error) -// throw new Error('An error has occurred during zip creation for the artifact') -// } + throw new Error('An error has occurred during zip creation for the artifact') +} +// eslint-disable-next-line @typescript-eslint/no-explicit-any const zipWarningCallback = (err: any): void => { if (err.code === 'ENOENT') { core.warning( @@ -108,10 +97,10 @@ const zipWarningCallback = (err: any): void => { } } -// const zipFinishCallback = (): void => { -// core.debug('Zip stream for upload has finished.') -// } +const zipFinishCallback = (): void => { + core.debug('Zip stream for upload has finished.') +} -// const zipEndCallback = (): void => { -// core.debug('Zip stream for upload has ended.') -// } +const zipEndCallback = (): void => { + core.debug('Zip stream for upload has ended.') +} From 92996632972afaa630dcadd73af02b4db0af9f20 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 17:13:32 +0000 Subject: [PATCH 18/72] adding promise all --- packages/artifact/src/internal/upload/zip.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 3aff9e361b..df7230fa45 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -40,8 +40,9 @@ export async function createZipUploadStream( zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) - for (const file of uploadSpecification) { - await new Promise((resolve, reject) => { + // for (const file of uploadSpecification) { + const fileUploadPromesses = uploadSpecification.map(async file => { + return new Promise((resolve, reject) => { if (file.sourcePath !== null) { // Add a normal file to the zip zip.entry( @@ -61,7 +62,7 @@ export async function createZipUploadStream( }) } }) - } + }) const bufferSize = getUploadChunkSize() const zipUploadStream = new ZipUploadStream(bufferSize) @@ -71,8 +72,12 @@ export async function createZipUploadStream( core.debug( `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` ) - zip.finalize() - return zipUploadStream + await Promise.all(fileUploadPromesses).then(() => { + zip.finalize() + + return zipUploadStream + }) + return Promise.reject(zipUploadStream) } // eslint-disable-next-line @typescript-eslint/no-explicit-any From b1f55c6942fa76e8899f1d27f1aa8c409d9095eb Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 17:15:08 +0000 Subject: [PATCH 19/72] adding finalize --- packages/artifact/src/internal/upload/zip.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index df7230fa45..364f8f7d69 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -72,12 +72,10 @@ export async function createZipUploadStream( core.debug( `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` ) - await Promise.all(fileUploadPromesses).then(() => { - zip.finalize() + await Promise.all(fileUploadPromesses) + zip.finalize() - return zipUploadStream - }) - return Promise.reject(zipUploadStream) + return zipUploadStream } // eslint-disable-next-line @typescript-eslint/no-explicit-any From 18751738a81fc6bc53cd3de0cd2fa111dacd5337 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 17:25:01 +0000 Subject: [PATCH 20/72] update loop for upload --- packages/artifact/package-lock.json | 1 + packages/artifact/package.json | 3 +- packages/artifact/src/internal/upload/zip.ts | 42 +++++++++----------- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index 9710156d7b..0b29125e0d 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -19,6 +19,7 @@ "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", "archiver": "^5.3.1", + "async": "^3.2.5", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", diff --git a/packages/artifact/package.json b/packages/artifact/package.json index 166438f4ab..7796e926cd 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -50,6 +50,7 @@ "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", "archiver": "^5.3.1", + "async": "^3.2.5", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", @@ -64,4 +65,4 @@ "typedoc-plugin-markdown": "^3.17.1", "typescript": "^5.2.2" } -} \ No newline at end of file +} diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 364f8f7d69..7d6340e11d 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -1,4 +1,5 @@ import * as stream from 'stream' +import * as async from 'async' import * as ZipStream from 'zip-stream' import * as core from '@actions/core' import {createReadStream} from 'fs' @@ -39,29 +40,22 @@ export async function createZipUploadStream( zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) - - // for (const file of uploadSpecification) { - const fileUploadPromesses = uploadSpecification.map(async file => { - return new Promise((resolve, reject) => { - if (file.sourcePath !== null) { - // Add a normal file to the zip - zip.entry( - createReadStream(file.sourcePath), - {name: file.destinationPath}, - function (err, entry) { - core.debug(`Entry is: ${entry}`) - if (err) reject(err) - else resolve(entry) - } - ) - } else { - zip.entry(null, {name: file.destinationPath}, function (err, entry) { + async.forEachOf(uploadSpecification, async file => { + if (file.sourcePath !== null) { + zip.entry( + createReadStream(file.sourcePath), + {name: file.destinationPath}, + function (err, entry) { core.debug(`Entry is: ${entry}`) - if (err) reject(err) - resolve(entry) - }) - } - }) + if (err) throw err + } + ) + } else { + zip.entry(null, {name: file.destinationPath}, function (err, entry) { + core.debug(`Entry is: ${entry}`) + if (err) throw err + }) + } }) const bufferSize = getUploadChunkSize() @@ -72,9 +66,9 @@ export async function createZipUploadStream( core.debug( `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` ) - await Promise.all(fileUploadPromesses) - zip.finalize() + // zip.pipe(zipUploadStream) + zip.finalize() return zipUploadStream } From bef1fc5f673942991a01bc18644cd919a6312171 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 17:30:06 +0000 Subject: [PATCH 21/72] adding check for running --- packages/artifact/src/internal/upload/zip.ts | 39 ++++++++++++-------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 7d6340e11d..7516e62a7d 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -7,6 +7,7 @@ import {UploadZipSpecification} from './upload-zip-specification' import {getUploadChunkSize} from '../shared/config' export const DEFAULT_COMPRESSION_LEVEL = 6 +export var isRunning = false // Custom stream transformer so we can set the highWaterMark property // See https://github.com/nodejs/node/issues/8855 @@ -27,6 +28,10 @@ export async function createZipUploadStream( uploadSpecification: UploadZipSpecification[], compressionLevel: number = DEFAULT_COMPRESSION_LEVEL ): Promise { + if (isRunning) { + throw new Error('The function is already running') + } + isRunning = true core.debug( `Creating Artifact archive with compressionLevel: ${compressionLevel}` ) @@ -40,23 +45,27 @@ export async function createZipUploadStream( zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) - async.forEachOf(uploadSpecification, async file => { - if (file.sourcePath !== null) { - zip.entry( - createReadStream(file.sourcePath), - {name: file.destinationPath}, - function (err, entry) { + try { + await async.forEachOf(uploadSpecification, async file => { + if (file.sourcePath !== null) { + zip.entry( + createReadStream(file.sourcePath), + {name: file.destinationPath}, + function (err, entry) { + core.debug(`Entry is: ${entry}`) + if (err) throw err + } + ) + } else { + zip.entry(null, {name: file.destinationPath}, function (err, entry) { core.debug(`Entry is: ${entry}`) if (err) throw err - } - ) - } else { - zip.entry(null, {name: file.destinationPath}, function (err, entry) { - core.debug(`Entry is: ${entry}`) - if (err) throw err - }) - } - }) + }) + } + }) + } finally { + isRunning = false + } const bufferSize = getUploadChunkSize() const zipUploadStream = new ZipUploadStream(bufferSize) From fd88cbe6df1480ac25618c68cd5c47ca1fe4cb71 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 17:32:01 +0000 Subject: [PATCH 22/72] adding check for running --- packages/artifact/src/internal/upload/zip.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 7516e62a7d..89abf59aba 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -7,7 +7,7 @@ import {UploadZipSpecification} from './upload-zip-specification' import {getUploadChunkSize} from '../shared/config' export const DEFAULT_COMPRESSION_LEVEL = 6 -export var isRunning = false +let isRunning = false // Custom stream transformer so we can set the highWaterMark property // See https://github.com/nodejs/node/issues/8855 From 66343faec46bace4a65b7ced19602359171f77b0 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 17:43:17 +0000 Subject: [PATCH 23/72] add resolve all promises again --- packages/artifact/src/internal/upload/zip.ts | 28 +++++++------------- 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 89abf59aba..7c69fc78d5 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -7,7 +7,6 @@ import {UploadZipSpecification} from './upload-zip-specification' import {getUploadChunkSize} from '../shared/config' export const DEFAULT_COMPRESSION_LEVEL = 6 -let isRunning = false // Custom stream transformer so we can set the highWaterMark property // See https://github.com/nodejs/node/issues/8855 @@ -28,10 +27,6 @@ export async function createZipUploadStream( uploadSpecification: UploadZipSpecification[], compressionLevel: number = DEFAULT_COMPRESSION_LEVEL ): Promise { - if (isRunning) { - throw new Error('The function is already running') - } - isRunning = true core.debug( `Creating Artifact archive with compressionLevel: ${compressionLevel}` ) @@ -45,27 +40,25 @@ export async function createZipUploadStream( zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) - try { - await async.forEachOf(uploadSpecification, async file => { + const uploadFilePromises = uploadSpecification.map(async file => { + return new Promise((resolve, reject) => { if (file.sourcePath !== null) { zip.entry( createReadStream(file.sourcePath), {name: file.destinationPath}, - function (err, entry) { - core.debug(`Entry is: ${entry}`) - if (err) throw err + (err, entry) => { + if (err) reject(err) + resolve(entry) } ) } else { - zip.entry(null, {name: file.destinationPath}, function (err, entry) { - core.debug(`Entry is: ${entry}`) - if (err) throw err + zip.entry(null, {name: file.destinationPath}, (err, entry) => { + if (err) reject(err) + resolve(entry) }) } }) - } finally { - isRunning = false - } + }) const bufferSize = getUploadChunkSize() const zipUploadStream = new ZipUploadStream(bufferSize) @@ -75,7 +68,7 @@ export async function createZipUploadStream( core.debug( `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` ) - + await Promise.all(uploadFilePromises) // zip.pipe(zipUploadStream) zip.finalize() return zipUploadStream @@ -88,7 +81,6 @@ const zipErrorCallback = (error: any): void => { throw new Error('An error has occurred during zip creation for the artifact') } -// eslint-disable-next-line @typescript-eslint/no-explicit-any const zipWarningCallback = (err: any): void => { if (err.code === 'ENOENT') { core.warning( From c7de68f215739283dfe53c20e607871e86c5d822 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 17:48:56 +0000 Subject: [PATCH 24/72] updating resolves --- packages/artifact/src/internal/upload/zip.ts | 27 +++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 7c69fc78d5..d022822ad4 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -1,5 +1,4 @@ import * as stream from 'stream' -import * as async from 'async' import * as ZipStream from 'zip-stream' import * as core from '@actions/core' import {createReadStream} from 'fs' @@ -60,18 +59,21 @@ export async function createZipUploadStream( }) }) - const bufferSize = getUploadChunkSize() - const zipUploadStream = new ZipUploadStream(bufferSize) - core.debug( - `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` - ) - core.debug( - `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` - ) - await Promise.all(uploadFilePromises) + await Promise.all(uploadFilePromises).then(result => { + core.debug(`Zip result is ${result}`) + zip.finalize() + const bufferSize = getUploadChunkSize() + const zipUploadStream = new ZipUploadStream(bufferSize) + core.debug( + `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` + ) + core.debug( + `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` + ) + return zipUploadStream + }) + throw new Error('An error has occurred during zip creation for the artifact') // zip.pipe(zipUploadStream) - zip.finalize() - return zipUploadStream } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -81,6 +83,7 @@ const zipErrorCallback = (error: any): void => { throw new Error('An error has occurred during zip creation for the artifact') } +// eslint-disable-next-line @typescript-eslint/no-explicit-any const zipWarningCallback = (err: any): void => { if (err.code === 'ENOENT') { core.warning( From fe0c0de7dbd55a505798bf3523557bba42e47325 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 17:53:08 +0000 Subject: [PATCH 25/72] adding lock --- packages/artifact/src/internal/upload/zip.ts | 42 ++++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index d022822ad4..3c5b1ed299 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -39,41 +39,39 @@ export async function createZipUploadStream( zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) - const uploadFilePromises = uploadSpecification.map(async file => { - return new Promise((resolve, reject) => { + for (const file of uploadSpecification) { + await new Promise((resolve, reject) => { if (file.sourcePath !== null) { + // Add a normal file to the zip zip.entry( createReadStream(file.sourcePath), {name: file.destinationPath}, - (err, entry) => { + function (err, entry) { + core.debug(`Entry is: ${entry}`) if (err) reject(err) - resolve(entry) + else resolve(entry) } ) } else { - zip.entry(null, {name: file.destinationPath}, (err, entry) => { + zip.entry(null, {name: file.destinationPath}, function (err, entry) { + core.debug(`Entry is: ${entry}`) if (err) reject(err) - resolve(entry) + else resolve(entry) }) } }) - }) + } - await Promise.all(uploadFilePromises).then(result => { - core.debug(`Zip result is ${result}`) - zip.finalize() - const bufferSize = getUploadChunkSize() - const zipUploadStream = new ZipUploadStream(bufferSize) - core.debug( - `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` - ) - core.debug( - `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` - ) - return zipUploadStream - }) - throw new Error('An error has occurred during zip creation for the artifact') - // zip.pipe(zipUploadStream) + zip.finalize() + const bufferSize = getUploadChunkSize() + const zipUploadStream = new ZipUploadStream(bufferSize) + core.debug( + `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` + ) + core.debug( + `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` + ) + return zipUploadStream } // eslint-disable-next-line @typescript-eslint/no-explicit-any From 97e4fcfcd560cda9c35267a8861e7f1dba511f47 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 18:07:20 +0000 Subject: [PATCH 26/72] wait on write stream --- .../src/internal/upload/blob-upload.ts | 36 +++++++++++++++---- packages/artifact/src/internal/upload/zip.ts | 1 + 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index 87bb723768..888077c0d1 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -52,14 +52,22 @@ export async function uploadZipToBlobStorage( zipUploadStream.pipe(hashStream).setEncoding('hex') // This stream is used to compute a hash of the zip content that gets used. Integrity check core.info('Beginning upload of artifact content to blob storage') - try { - await blockBlobClient.uploadStream( - uploadStream, - bufferSize, - maxConcurrency, - options - ) + await new Promise((resolve, reject) => { + uploadStream.on('readable', async () => { + try { + await blockBlobClient.uploadStream( + uploadStream, + bufferSize, + maxConcurrency, + options + ) + resolve('success') + } catch (error) { + reject(error) + } + }) + }) } catch (error) { if (NetworkError.isNetworkErrorCode(error?.code)) { throw new NetworkError(error?.code) @@ -67,6 +75,20 @@ export async function uploadZipToBlobStorage( throw error } + // try { + // await blockBlobClient.uploadStream( + // uploadStream, + // bufferSize, + // maxConcurrency, + // options + // ) + // } catch (error) { + // if (NetworkError.isNetworkErrorCode(error?.code)) { + // throw new NetworkError(error?.code) + // } + + // throw error + // } core.info('Finished uploading artifact content to blob storage!') diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 3c5b1ed299..e8a7117a6d 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -53,6 +53,7 @@ export async function createZipUploadStream( } ) } else { + // add directory to zip zip.entry(null, {name: file.destinationPath}, function (err, entry) { core.debug(`Entry is: ${entry}`) if (err) reject(err) From f33a3f4748eb5cf38c639495f05f6b80479449e0 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 18:14:40 +0000 Subject: [PATCH 27/72] Remove finalize --- .../src/internal/upload/blob-upload.ts | 36 ++++--------------- packages/artifact/src/internal/upload/zip.ts | 2 +- 2 files changed, 8 insertions(+), 30 deletions(-) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index 888077c0d1..87bb723768 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -52,22 +52,14 @@ export async function uploadZipToBlobStorage( zipUploadStream.pipe(hashStream).setEncoding('hex') // This stream is used to compute a hash of the zip content that gets used. Integrity check core.info('Beginning upload of artifact content to blob storage') + try { - await new Promise((resolve, reject) => { - uploadStream.on('readable', async () => { - try { - await blockBlobClient.uploadStream( - uploadStream, - bufferSize, - maxConcurrency, - options - ) - resolve('success') - } catch (error) { - reject(error) - } - }) - }) + await blockBlobClient.uploadStream( + uploadStream, + bufferSize, + maxConcurrency, + options + ) } catch (error) { if (NetworkError.isNetworkErrorCode(error?.code)) { throw new NetworkError(error?.code) @@ -75,20 +67,6 @@ export async function uploadZipToBlobStorage( throw error } - // try { - // await blockBlobClient.uploadStream( - // uploadStream, - // bufferSize, - // maxConcurrency, - // options - // ) - // } catch (error) { - // if (NetworkError.isNetworkErrorCode(error?.code)) { - // throw new NetworkError(error?.code) - // } - - // throw error - // } core.info('Finished uploading artifact content to blob storage!') diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index e8a7117a6d..23c521b463 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -63,7 +63,7 @@ export async function createZipUploadStream( }) } - zip.finalize() + // zip.finalize() const bufferSize = getUploadChunkSize() const zipUploadStream = new ZipUploadStream(bufferSize) core.debug( From 180b75bf01ff8ca7b9eb9e6c8a57ca124d7c0345 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 18:29:20 +0000 Subject: [PATCH 28/72] wrap promises in chain --- .../src/internal/upload/upload-artifact.ts | 107 ++++++++++++------ 1 file changed, 70 insertions(+), 37 deletions(-) diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index 2e209348ca..ce0b569847 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -68,45 +68,81 @@ export async function uploadArtifact( ) } - const zipUploadStream = await createZipUploadStream( - zipSpecification, - options?.compressionLevel - ) + return createZipUploadStream(zipSpecification, options?.compressionLevel) + .then(async zipUploadStream => { + return uploadZipToBlobStorage( + createArtifactResp.signedUploadUrl, + zipUploadStream + ) + }) + .then(async uploadResult => { + const finalizeArtifactReq: FinalizeArtifactRequest = { + workflowRunBackendId: backendIds.workflowRunBackendId, + workflowJobRunBackendId: backendIds.workflowJobRunBackendId, + name, + size: uploadResult.uploadSize ? uploadResult.uploadSize.toString() : '0' + } + if (uploadResult.sha256Hash) { + finalizeArtifactReq.hash = StringValue.create({ + value: `sha256:${uploadResult.sha256Hash}` + }) + } + core.info(`Finalizing artifact upload`) + const finalizeArtifactResp = + await artifactClient.FinalizeArtifact(finalizeArtifactReq) - // Upload zip to blob storage - const uploadResult = await uploadZipToBlobStorage( - createArtifactResp.signedUploadUrl, - zipUploadStream - ) + return {finalizeArtifactResp, uploadResult} + }) + .then(({finalizeArtifactResp, uploadResult}) => { + if (!finalizeArtifactResp.ok) { + throw new InvalidResponseError( + 'FinalizeArtifact: response from backend was not ok' + ) + } + const artifactId = BigInt(finalizeArtifactResp.artifactId) + core.info( + `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}` + ) + return { + size: uploadResult.uploadSize, + id: Number(artifactId) + } + }) - // finalize the artifact - const finalizeArtifactReq: FinalizeArtifactRequest = { - workflowRunBackendId: backendIds.workflowRunBackendId, - workflowJobRunBackendId: backendIds.workflowJobRunBackendId, - name, - size: uploadResult.uploadSize ? uploadResult.uploadSize.toString() : '0' - } + // // Upload zip to blob storage + // const uploadResult = await uploadZipToBlobStorage( + // createArtifactResp.signedUploadUrl, + // zipUploadStream + // ) - if (uploadResult.sha256Hash) { - finalizeArtifactReq.hash = StringValue.create({ - value: `sha256:${uploadResult.sha256Hash}` - }) - } + // finalize the artifact + // const finalizeArtifactReq: FinalizeArtifactRequest = { + // workflowRunBackendId: backendIds.workflowRunBackendId, + // workflowJobRunBackendId: backendIds.workflowJobRunBackendId, + // name, + // size: uploadResult.uploadSize ? uploadResult.uploadSize.toString() : '0' + // } - core.info(`Finalizing artifact upload`) + // if (uploadResult.sha256Hash) { + // finalizeArtifactReq.hash = StringValue.create({ + // value: `sha256:${uploadResult.sha256Hash}` + // }) + // } + // + // core.info(`Finalizing artifact upload`) - const finalizeArtifactResp = - await artifactClient.FinalizeArtifact(finalizeArtifactReq) - if (!finalizeArtifactResp.ok) { - throw new InvalidResponseError( - 'FinalizeArtifact: response from backend was not ok' - ) - } + // const finalizeArtifactResp = + // await artifactClient.FinalizeArtifact(finalizeArtifactReq) + // if (!finalizeArtifactResp.ok) { + // throw new InvalidResponseError( + // 'FinalizeArtifact: response from backend was not ok' + // ) + // } - const artifactId = BigInt(finalizeArtifactResp.artifactId) - core.info( - `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}` - ) + // const artifactId = BigInt(finalizeArtifactResp.artifactId) + // core.info( + // `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}` + // ) // if (core.isDebug()) { // setTimeout(function () { // core.debug('Processes keeping upload stream running:') @@ -115,8 +151,5 @@ export async function uploadArtifact( // } // - return { - size: uploadResult.uploadSize, - id: Number(artifactId) - } + // ) } From 077846ed3461c6a9526c7671c044188265052f3d Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 18:34:07 +0000 Subject: [PATCH 29/72] adding readable check --- packages/artifact/src/internal/upload/blob-upload.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index 87bb723768..f98cab265d 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -52,7 +52,8 @@ export async function uploadZipToBlobStorage( zipUploadStream.pipe(hashStream).setEncoding('hex') // This stream is used to compute a hash of the zip content that gets used. Integrity check core.info('Beginning upload of artifact content to blob storage') - + const isItReadable = zipUploadStream.readable + core.info(`Is the zipUploadStream readable? ${isItReadable}`) try { await blockBlobClient.uploadStream( uploadStream, From 73f526b642f4d5cca7293840d51ca2a9113c65ec Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 18:36:40 +0000 Subject: [PATCH 30/72] adding closed check --- packages/artifact/src/internal/upload/blob-upload.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index f98cab265d..d653457155 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -53,7 +53,9 @@ export async function uploadZipToBlobStorage( core.info('Beginning upload of artifact content to blob storage') const isItReadable = zipUploadStream.readable - core.info(`Is the zipUploadStream readable? ${isItReadable}`) + core.info(`Is the zipUploadStream readable? ${isItReadable}`) // it is readable, that's good + const isItClosed = zipUploadStream.closed + core.info(`Is the zipUploadStream closed? ${isItClosed}`) // it is not closed, that's good try { await blockBlobClient.uploadStream( uploadStream, From 26b62e0fd579b2a217ece33fcc2f0b1d11eb67c2 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 18:38:52 +0000 Subject: [PATCH 31/72] adding more logging --- packages/artifact/src/internal/upload/blob-upload.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index d653457155..e285b93e25 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -63,6 +63,7 @@ export async function uploadZipToBlobStorage( maxConcurrency, options ) + core.info('Even more beginning upload of artifact content to blob storage') } catch (error) { if (NetworkError.isNetworkErrorCode(error?.code)) { throw new NetworkError(error?.code) From e4c0440c3a286da3e770f5e0dddaccc1be0322d0 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 18:40:37 +0000 Subject: [PATCH 32/72] adding more logging --- packages/artifact/src/internal/upload/blob-upload.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index e285b93e25..9c11728966 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -57,13 +57,18 @@ export async function uploadZipToBlobStorage( const isItClosed = zipUploadStream.closed core.info(`Is the zipUploadStream closed? ${isItClosed}`) // it is not closed, that's good try { + core.info( + '1 Even more beginning upload of artifact content to blob storage' + ) await blockBlobClient.uploadStream( uploadStream, bufferSize, maxConcurrency, options ) - core.info('Even more beginning upload of artifact content to blob storage') + core.info( + '2 Even more beginning upload of artifact content to blob storage' + ) } catch (error) { if (NetworkError.isNetworkErrorCode(error?.code)) { throw new NetworkError(error?.code) From a768aa30c72c3cb044625ce5c328e53efa7adb20 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 18:43:58 +0000 Subject: [PATCH 33/72] adding more logging --- packages/artifact/src/internal/upload/blob-upload.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index 9c11728966..7e94c195d4 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -56,6 +56,7 @@ export async function uploadZipToBlobStorage( core.info(`Is the zipUploadStream readable? ${isItReadable}`) // it is readable, that's good const isItClosed = zipUploadStream.closed core.info(`Is the zipUploadStream closed? ${isItClosed}`) // it is not closed, that's good + core.info(`Is the buffer size appropriate? ${bufferSize}`) // it is not closed, that's good try { core.info( '1 Even more beginning upload of artifact content to blob storage' From d597cf234f70be9d22b935a3d5fdf2667596cedf Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 18:46:31 +0000 Subject: [PATCH 34/72] adding upload stream check --- packages/artifact/src/internal/upload/blob-upload.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index 7e94c195d4..22fe32e75e 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -57,6 +57,7 @@ export async function uploadZipToBlobStorage( const isItClosed = zipUploadStream.closed core.info(`Is the zipUploadStream closed? ${isItClosed}`) // it is not closed, that's good core.info(`Is the buffer size appropriate? ${bufferSize}`) // it is not closed, that's good + core.info(`is the upload stream open? ${uploadStream.closed}`) try { core.info( '1 Even more beginning upload of artifact content to blob storage' From 6ac9cbfda9c285ba32cf12e9b600def93c0790c2 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 18:51:00 +0000 Subject: [PATCH 35/72] check if upload and zipload streams are writable --- packages/artifact/src/internal/upload/blob-upload.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index 22fe32e75e..d7938439f8 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -52,12 +52,11 @@ export async function uploadZipToBlobStorage( zipUploadStream.pipe(hashStream).setEncoding('hex') // This stream is used to compute a hash of the zip content that gets used. Integrity check core.info('Beginning upload of artifact content to blob storage') - const isItReadable = zipUploadStream.readable - core.info(`Is the zipUploadStream readable? ${isItReadable}`) // it is readable, that's good - const isItClosed = zipUploadStream.closed - core.info(`Is the zipUploadStream closed? ${isItClosed}`) // it is not closed, that's good + core.info(`Is the zipUploadStream readable? ${zipUploadStream.readable}`) // it is readable, that's good + core.info(`Is the zipUploadStream closed? ${zipUploadStream.closed}`) // it is not closed, that's good core.info(`Is the buffer size appropriate? ${bufferSize}`) // it is not closed, that's good - core.info(`is the upload stream open? ${uploadStream.closed}`) + core.info(`is the upload stream closed? ${uploadStream.closed}`) + core.info(`is the upload stream readable? ${uploadStream.readable}`) try { core.info( '1 Even more beginning upload of artifact content to blob storage' From d2d69999e3ed8b443265b0d735ce98e37e88c469 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 18:53:15 +0000 Subject: [PATCH 36/72] adding write check --- packages/artifact/src/internal/upload/blob-upload.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index d7938439f8..42109c7ea4 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -53,10 +53,12 @@ export async function uploadZipToBlobStorage( core.info('Beginning upload of artifact content to blob storage') core.info(`Is the zipUploadStream readable? ${zipUploadStream.readable}`) // it is readable, that's good + core.info(`Is the zipUploadStream writable now? ${zipUploadStream.writable}`) // it is readable, that's good core.info(`Is the zipUploadStream closed? ${zipUploadStream.closed}`) // it is not closed, that's good core.info(`Is the buffer size appropriate? ${bufferSize}`) // it is not closed, that's good core.info(`is the upload stream closed? ${uploadStream.closed}`) core.info(`is the upload stream readable? ${uploadStream.readable}`) + core.info(`is the upload stream writable? ${uploadStream.writable}`) try { core.info( '1 Even more beginning upload of artifact content to blob storage' From 31a6086649d2598521b7a007baf8e94ea4e0db47 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 18:59:41 +0000 Subject: [PATCH 37/72] Adding concurrency and upload response logs --- .../artifact/src/internal/upload/blob-upload.ts | 12 ++++++------ packages/artifact/src/internal/upload/zip.ts | 16 ++++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index 42109c7ea4..4b003fbd23 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -59,16 +59,16 @@ export async function uploadZipToBlobStorage( core.info(`is the upload stream closed? ${uploadStream.closed}`) core.info(`is the upload stream readable? ${uploadStream.readable}`) core.info(`is the upload stream writable? ${uploadStream.writable}`) + core.info(`are we exceeding the max concurrency? ${maxConcurrency}`) try { core.info( '1 Even more beginning upload of artifact content to blob storage' ) - await blockBlobClient.uploadStream( - uploadStream, - bufferSize, - maxConcurrency, - options - ) + await blockBlobClient + .uploadStream(uploadStream, bufferSize, maxConcurrency, options) + .then(response => { + core.info(`Upload stream response: ${response}`) + }) core.info( '2 Even more beginning upload of artifact content to blob storage' ) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 23c521b463..e4966858bf 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -47,18 +47,22 @@ export async function createZipUploadStream( createReadStream(file.sourcePath), {name: file.destinationPath}, function (err, entry) { - core.debug(`Entry is: ${entry}`) + core.info(`Entry is: ${entry}`) if (err) reject(err) else resolve(entry) } ) } else { // add directory to zip - zip.entry(null, {name: file.destinationPath}, function (err, entry) { - core.debug(`Entry is: ${entry}`) - if (err) reject(err) - else resolve(entry) - }) + zip.entry( + null, + {name: `${file.destinationPath}/`}, + function (err, entry) { + core.info(`Entry is: ${entry}`) + if (err) reject(err) + else resolve(entry) + } + ) } }) } From c9e825e84193bbbc0cfbfd0eeb35a897c4fa662d Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 19:06:47 +0000 Subject: [PATCH 38/72] add blob client property check --- packages/artifact/src/internal/upload/blob-upload.ts | 2 ++ packages/artifact/src/internal/upload/upload-artifact.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index 4b003fbd23..04232594a6 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -60,6 +60,8 @@ export async function uploadZipToBlobStorage( core.info(`is the upload stream readable? ${uploadStream.readable}`) core.info(`is the upload stream writable? ${uploadStream.writable}`) core.info(`are we exceeding the max concurrency? ${maxConcurrency}`) + // is the blob client ready? + core.info(`blob client props: ${blockBlobClient.url}`) try { core.info( '1 Even more beginning upload of artifact content to blob storage' diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index ce0b569847..30b6978c60 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -70,6 +70,8 @@ export async function uploadArtifact( return createZipUploadStream(zipSpecification, options?.compressionLevel) .then(async zipUploadStream => { + core.info(`starting upload to blob here`) + core.info(`zipuploadstream looks like this: ${zipUploadStream}`) return uploadZipToBlobStorage( createArtifactResp.signedUploadUrl, zipUploadStream From 0a0e70d1cd3d4052cb98cbe49751c4faeacfbf28 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 19:09:53 +0000 Subject: [PATCH 39/72] Add another catch to see if upload stream is failing --- packages/artifact/src/internal/upload/blob-upload.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index 04232594a6..bc045c091b 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -68,6 +68,9 @@ export async function uploadZipToBlobStorage( ) await blockBlobClient .uploadStream(uploadStream, bufferSize, maxConcurrency, options) + .catch(error => { + core.info(`Upload stream error: ${error}`) + }) .then(response => { core.info(`Upload stream response: ${response}`) }) From 59593338a6b9704c4c40a8de85a00c83a35edf65 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 19:15:10 +0000 Subject: [PATCH 40/72] defaulting compression level --- .../artifact/src/internal/upload/blob-upload.ts | 16 ++++++---------- .../src/internal/upload/upload-artifact.ts | 5 ++++- packages/artifact/src/internal/upload/zip.ts | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index bc045c091b..b244861af8 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -60,20 +60,16 @@ export async function uploadZipToBlobStorage( core.info(`is the upload stream readable? ${uploadStream.readable}`) core.info(`is the upload stream writable? ${uploadStream.writable}`) core.info(`are we exceeding the max concurrency? ${maxConcurrency}`) - // is the blob client ready? - core.info(`blob client props: ${blockBlobClient.url}`) try { core.info( '1 Even more beginning upload of artifact content to blob storage' ) - await blockBlobClient - .uploadStream(uploadStream, bufferSize, maxConcurrency, options) - .catch(error => { - core.info(`Upload stream error: ${error}`) - }) - .then(response => { - core.info(`Upload stream response: ${response}`) - }) + await blockBlobClient.uploadStream( + uploadStream, + bufferSize, + maxConcurrency, + options + ) core.info( '2 Even more beginning upload of artifact content to blob storage' ) diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index 30b6978c60..1e0c9d093c 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -75,7 +75,10 @@ export async function uploadArtifact( return uploadZipToBlobStorage( createArtifactResp.signedUploadUrl, zipUploadStream - ) + ).catch(err => { + core.error(`Error uploading artifact: ${err}`) + throw err + }) }) .then(async uploadResult => { const finalizeArtifactReq: FinalizeArtifactRequest = { diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index e4966858bf..8bcc09c224 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -30,7 +30,7 @@ export async function createZipUploadStream( `Creating Artifact archive with compressionLevel: ${compressionLevel}` ) const zlibOptions = { - zlib: {level: compressionLevel, bufferSize: getUploadChunkSize()} + zlib: {level: DEFAULT_COMPRESSION_LEVEL, bufferSize: getUploadChunkSize()} } const zip = new ZipStream.default(zlibOptions) // register callbacks for various events during the zip lifecycle From 17c4cab8b265b0202b4dce39232ac5d968383210 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 19:16:10 +0000 Subject: [PATCH 41/72] adding constant zlib compression level --- packages/artifact/src/internal/upload/zip.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 8bcc09c224..97ea57e7a6 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -1,6 +1,7 @@ import * as stream from 'stream' import * as ZipStream from 'zip-stream' import * as core from '@actions/core' +import * as zlib from 'zlib' import {createReadStream} from 'fs' import {UploadZipSpecification} from './upload-zip-specification' import {getUploadChunkSize} from '../shared/config' @@ -30,7 +31,10 @@ export async function createZipUploadStream( `Creating Artifact archive with compressionLevel: ${compressionLevel}` ) const zlibOptions = { - zlib: {level: DEFAULT_COMPRESSION_LEVEL, bufferSize: getUploadChunkSize()} + zlib: { + level: zlib.constants.Z_DEFAULT_COMPRESSION, + bufferSize: getUploadChunkSize() + } } const zip = new ZipStream.default(zlibOptions) // register callbacks for various events during the zip lifecycle From b6c87ceea7e0cd24586f4bed402ba57206e8691b Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 19:22:29 +0000 Subject: [PATCH 42/72] Adding more error handlrs --- packages/artifact/src/internal/upload/blob-upload.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index b244861af8..c9f03a57b2 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -60,6 +60,14 @@ export async function uploadZipToBlobStorage( core.info(`is the upload stream readable? ${uploadStream.readable}`) core.info(`is the upload stream writable? ${uploadStream.writable}`) core.info(`are we exceeding the max concurrency? ${maxConcurrency}`) + + zipUploadStream.on('error', error => { + core.info(`Error in zipUploadStream: ${error}`) + }) + + uploadStream.on('error', error => { + core.info(`Error in uploadStream:', ${error}`) + }) try { core.info( '1 Even more beginning upload of artifact content to blob storage' From 9322468ef0ba03ec7a3b2a16ef45e7cc6ec16a1c Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 19:32:40 +0000 Subject: [PATCH 43/72] Adding pipe back --- packages/artifact/src/internal/upload/zip.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 97ea57e7a6..82c987082c 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -71,9 +71,11 @@ export async function createZipUploadStream( }) } - // zip.finalize() + zip.finalize() const bufferSize = getUploadChunkSize() const zipUploadStream = new ZipUploadStream(bufferSize) + zip.pipe(zipUploadStream) // Pipe the zip stream into zipUploadStream + core.debug( `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` ) From bc893bf1b4f3342394c32f07112605f8bd9b46a6 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 19:48:15 +0000 Subject: [PATCH 44/72] adding compression level back --- packages/artifact/src/internal/upload/zip.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 82c987082c..22045e2425 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -32,7 +32,7 @@ export async function createZipUploadStream( ) const zlibOptions = { zlib: { - level: zlib.constants.Z_DEFAULT_COMPRESSION, + level: compressionLevel, bufferSize: getUploadChunkSize() } } From 5288db39533c812d40aa43ce7c5eda34c00a5162 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 19:57:01 +0000 Subject: [PATCH 45/72] removing debugging statements --- .../src/internal/upload/blob-upload.ts | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index c9f03a57b2..1e60770d15 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -51,36 +51,13 @@ export async function uploadZipToBlobStorage( zipUploadStream.pipe(uploadStream) // This stream is used for the upload zipUploadStream.pipe(hashStream).setEncoding('hex') // This stream is used to compute a hash of the zip content that gets used. Integrity check - core.info('Beginning upload of artifact content to blob storage') - core.info(`Is the zipUploadStream readable? ${zipUploadStream.readable}`) // it is readable, that's good - core.info(`Is the zipUploadStream writable now? ${zipUploadStream.writable}`) // it is readable, that's good - core.info(`Is the zipUploadStream closed? ${zipUploadStream.closed}`) // it is not closed, that's good - core.info(`Is the buffer size appropriate? ${bufferSize}`) // it is not closed, that's good - core.info(`is the upload stream closed? ${uploadStream.closed}`) - core.info(`is the upload stream readable? ${uploadStream.readable}`) - core.info(`is the upload stream writable? ${uploadStream.writable}`) - core.info(`are we exceeding the max concurrency? ${maxConcurrency}`) - - zipUploadStream.on('error', error => { - core.info(`Error in zipUploadStream: ${error}`) - }) - - uploadStream.on('error', error => { - core.info(`Error in uploadStream:', ${error}`) - }) try { - core.info( - '1 Even more beginning upload of artifact content to blob storage' - ) await blockBlobClient.uploadStream( uploadStream, bufferSize, maxConcurrency, options ) - core.info( - '2 Even more beginning upload of artifact content to blob storage' - ) } catch (error) { if (NetworkError.isNetworkErrorCode(error?.code)) { throw new NetworkError(error?.code) From 0f23ae11848bc85dbab5167b59739465bad0a050 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 20:05:16 +0000 Subject: [PATCH 46/72] re-adding old async/await pattern --- .../src/internal/upload/upload-artifact.ts | 123 +++++------------- 1 file changed, 36 insertions(+), 87 deletions(-) diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index 1e0c9d093c..d4595001c3 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -67,94 +67,43 @@ export async function uploadArtifact( 'CreateArtifact: response from backend was not ok' ) } - - return createZipUploadStream(zipSpecification, options?.compressionLevel) - .then(async zipUploadStream => { - core.info(`starting upload to blob here`) - core.info(`zipuploadstream looks like this: ${zipUploadStream}`) - return uploadZipToBlobStorage( - createArtifactResp.signedUploadUrl, - zipUploadStream - ).catch(err => { - core.error(`Error uploading artifact: ${err}`) - throw err - }) - }) - .then(async uploadResult => { - const finalizeArtifactReq: FinalizeArtifactRequest = { - workflowRunBackendId: backendIds.workflowRunBackendId, - workflowJobRunBackendId: backendIds.workflowJobRunBackendId, - name, - size: uploadResult.uploadSize ? uploadResult.uploadSize.toString() : '0' - } - if (uploadResult.sha256Hash) { - finalizeArtifactReq.hash = StringValue.create({ - value: `sha256:${uploadResult.sha256Hash}` - }) - } - core.info(`Finalizing artifact upload`) - const finalizeArtifactResp = - await artifactClient.FinalizeArtifact(finalizeArtifactReq) - - return {finalizeArtifactResp, uploadResult} - }) - .then(({finalizeArtifactResp, uploadResult}) => { - if (!finalizeArtifactResp.ok) { - throw new InvalidResponseError( - 'FinalizeArtifact: response from backend was not ok' - ) - } - const artifactId = BigInt(finalizeArtifactResp.artifactId) - core.info( - `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}` - ) - return { - size: uploadResult.uploadSize, - id: Number(artifactId) - } - }) - - // // Upload zip to blob storage - // const uploadResult = await uploadZipToBlobStorage( - // createArtifactResp.signedUploadUrl, - // zipUploadStream - // ) - + const zipUploadStream = await createZipUploadStream( + zipSpecification, + options?.compressionLevel + ) + // Upload zip to blob storage + const uploadResult = await uploadZipToBlobStorage( + createArtifactResp.signedUploadUrl, + zipUploadStream + ) // finalize the artifact - // const finalizeArtifactReq: FinalizeArtifactRequest = { - // workflowRunBackendId: backendIds.workflowRunBackendId, - // workflowJobRunBackendId: backendIds.workflowJobRunBackendId, - // name, - // size: uploadResult.uploadSize ? uploadResult.uploadSize.toString() : '0' - // } - - // if (uploadResult.sha256Hash) { - // finalizeArtifactReq.hash = StringValue.create({ - // value: `sha256:${uploadResult.sha256Hash}` - // }) - // } - // - // core.info(`Finalizing artifact upload`) - - // const finalizeArtifactResp = - // await artifactClient.FinalizeArtifact(finalizeArtifactReq) - // if (!finalizeArtifactResp.ok) { - // throw new InvalidResponseError( - // 'FinalizeArtifact: response from backend was not ok' - // ) - // } + const finalizeArtifactReq: FinalizeArtifactRequest = { + workflowRunBackendId: backendIds.workflowRunBackendId, + workflowJobRunBackendId: backendIds.workflowJobRunBackendId, + name, + size: uploadResult.uploadSize ? uploadResult.uploadSize.toString() : '0' + } + if (uploadResult.sha256Hash) { + finalizeArtifactReq.hash = StringValue.create({ + value: `sha256:${uploadResult.sha256Hash}` + }) + } + core.info(`Finalizing artifact upload`) + const finalizeArtifactResp = + await artifactClient.FinalizeArtifact(finalizeArtifactReq) + if (!finalizeArtifactResp.ok) { + throw new InvalidResponseError( + 'FinalizeArtifact: response from backend was not ok' + ) + } - // const artifactId = BigInt(finalizeArtifactResp.artifactId) - // core.info( - // `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}` - // ) - // if (core.isDebug()) { - // setTimeout(function () { - // core.debug('Processes keeping upload stream running:') - // whyIsNodeRunning() - // }, 500) - // } - // + const artifactId = BigInt(finalizeArtifactResp.artifactId) + core.info( + `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}` + ) - // ) + return { + size: uploadResult.uploadSize, + id: Number(artifactId) + } } From d02a834fe32ae84c5820d5cd80665a3699951ec4 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 20:12:52 +0000 Subject: [PATCH 47/72] Remove other debug statements --- packages/artifact/src/internal/upload/upload-artifact.ts | 1 + packages/artifact/src/internal/upload/zip.ts | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index d4595001c3..03c72b5e6f 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -67,6 +67,7 @@ export async function uploadArtifact( 'CreateArtifact: response from backend was not ok' ) } + // Create the zipupload stream for use in blob upload const zipUploadStream = await createZipUploadStream( zipSpecification, options?.compressionLevel diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 22045e2425..cab36e6c67 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -51,7 +51,6 @@ export async function createZipUploadStream( createReadStream(file.sourcePath), {name: file.destinationPath}, function (err, entry) { - core.info(`Entry is: ${entry}`) if (err) reject(err) else resolve(entry) } @@ -62,7 +61,6 @@ export async function createZipUploadStream( null, {name: `${file.destinationPath}/`}, function (err, entry) { - core.info(`Entry is: ${entry}`) if (err) reject(err) else resolve(entry) } From 6d94ad125b48b3ed74d4df4726514776a6ea6273 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 20:38:01 +0000 Subject: [PATCH 48/72] Removing uneeded libs --- packages/artifact/src/internal/upload/zip.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index cab36e6c67..89c6bed03e 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -1,7 +1,6 @@ import * as stream from 'stream' import * as ZipStream from 'zip-stream' import * as core from '@actions/core' -import * as zlib from 'zlib' import {createReadStream} from 'fs' import {UploadZipSpecification} from './upload-zip-specification' import {getUploadChunkSize} from '../shared/config' From a2852cee6feabc1ba06b54c6d6890ab546a2d951 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 21:12:51 +0000 Subject: [PATCH 49/72] adding catch to upload zip --- packages/artifact/src/internal/upload/upload-artifact.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index 03c72b5e6f..f9817262ff 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -76,7 +76,11 @@ export async function uploadArtifact( const uploadResult = await uploadZipToBlobStorage( createArtifactResp.signedUploadUrl, zipUploadStream - ) + ).catch(err => { + throw new InvalidResponseError( + `uploadZipToBlobStorage: response blob was not ok: ${err}` + ) + }) // finalize the artifact const finalizeArtifactReq: FinalizeArtifactRequest = { workflowRunBackendId: backendIds.workflowRunBackendId, From c14e304555a8b7c8e8a12e82bf7a45386b3d6d5e Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 28 Mar 2024 21:17:54 +0000 Subject: [PATCH 50/72] adding more debugging statements --- packages/artifact/src/internal/upload/zip.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 89c6bed03e..531f29507f 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -42,9 +42,11 @@ export async function createZipUploadStream( zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) + core.debug(`Number of files to upload: ${uploadSpecification.length}`) for (const file of uploadSpecification) { await new Promise((resolve, reject) => { if (file.sourcePath !== null) { + core.debug(`Creating file entry: ${file.sourcePath}`) // Add a normal file to the zip zip.entry( createReadStream(file.sourcePath), @@ -56,6 +58,7 @@ export async function createZipUploadStream( ) } else { // add directory to zip + core.debug(`Creating directory entry: ${file.destinationPath}`) zip.entry( null, {name: `${file.destinationPath}/`}, @@ -69,6 +72,7 @@ export async function createZipUploadStream( } zip.finalize() + core.debug(`Finalizing entries`) const bufferSize = getUploadChunkSize() const zipUploadStream = new ZipUploadStream(bufferSize) zip.pipe(zipUploadStream) // Pipe the zip stream into zipUploadStream From 90ee020ccdaf39ff0ad55ebae768997e569dbb93 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 13:52:01 +0000 Subject: [PATCH 51/72] cleaning up debug statements --- packages/artifact/src/internal/upload/blob-upload.ts | 1 + packages/artifact/src/internal/upload/zip.ts | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index 1e60770d15..6f5bf7346b 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -51,6 +51,7 @@ export async function uploadZipToBlobStorage( zipUploadStream.pipe(uploadStream) // This stream is used for the upload zipUploadStream.pipe(hashStream).setEncoding('hex') // This stream is used to compute a hash of the zip content that gets used. Integrity check + core.info('Beginning upload of artifact content to blob storage') try { await blockBlobClient.uploadStream( uploadStream, diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 531f29507f..953fc79eb1 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -42,11 +42,9 @@ export async function createZipUploadStream( zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) - core.debug(`Number of files to upload: ${uploadSpecification.length}`) for (const file of uploadSpecification) { await new Promise((resolve, reject) => { if (file.sourcePath !== null) { - core.debug(`Creating file entry: ${file.sourcePath}`) // Add a normal file to the zip zip.entry( createReadStream(file.sourcePath), @@ -58,7 +56,6 @@ export async function createZipUploadStream( ) } else { // add directory to zip - core.debug(`Creating directory entry: ${file.destinationPath}`) zip.entry( null, {name: `${file.destinationPath}/`}, From 2da528819c1336a0c04bdf288924d7ff6117010b Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 13:58:57 +0000 Subject: [PATCH 52/72] cleanup --- packages/artifact/package-lock.json | 1 - packages/artifact/package.json | 3 +-- packages/artifact/src/internal/upload/blob-upload.ts | 1 + .../artifact/src/internal/upload/upload-artifact.ts | 1 + packages/artifact/src/internal/upload/zip.ts | 12 ++++++------ 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index 0b29125e0d..9710156d7b 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -19,7 +19,6 @@ "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", "archiver": "^5.3.1", - "async": "^3.2.5", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", diff --git a/packages/artifact/package.json b/packages/artifact/package.json index 7796e926cd..166438f4ab 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -50,7 +50,6 @@ "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", "archiver": "^5.3.1", - "async": "^3.2.5", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", @@ -65,4 +64,4 @@ "typedoc-plugin-markdown": "^3.17.1", "typescript": "^5.2.2" } -} +} \ No newline at end of file diff --git a/packages/artifact/src/internal/upload/blob-upload.ts b/packages/artifact/src/internal/upload/blob-upload.ts index 6f5bf7346b..87bb723768 100644 --- a/packages/artifact/src/internal/upload/blob-upload.ts +++ b/packages/artifact/src/internal/upload/blob-upload.ts @@ -52,6 +52,7 @@ export async function uploadZipToBlobStorage( zipUploadStream.pipe(hashStream).setEncoding('hex') // This stream is used to compute a hash of the zip content that gets used. Integrity check core.info('Beginning upload of artifact content to blob storage') + try { await blockBlobClient.uploadStream( uploadStream, diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index f9817262ff..b8f952014b 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -72,6 +72,7 @@ export async function uploadArtifact( zipSpecification, options?.compressionLevel ) + // Upload zip to blob storage const uploadResult = await uploadZipToBlobStorage( createArtifactResp.signedUploadUrl, diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 953fc79eb1..f62d560376 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -39,9 +39,9 @@ export async function createZipUploadStream( // register callbacks for various events during the zip lifecycle zip.on('error', zipErrorCallback) zip.on('warning', zipWarningCallback) - zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) + for (const file of uploadSpecification) { await new Promise((resolve, reject) => { if (file.sourcePath !== null) { @@ -91,17 +91,17 @@ const zipErrorCallback = (error: any): void => { throw new Error('An error has occurred during zip creation for the artifact') } // eslint-disable-next-line @typescript-eslint/no-explicit-any -const zipWarningCallback = (err: any): void => { - if (err.code === 'ENOENT') { +const zipWarningCallback = (error: any): void => { + if (error.code === 'ENOENT') { core.warning( 'ENOENT warning during artifact zip creation. No such file or directory' ) - core.info(err) + core.info(error) } else { core.warning( - `A non-blocking warning has occurred during artifact zip creation: ${err.code}` + `A non-blocking warning has occurred during artifact zip creation: ${error.code}` ) - core.info(err) + core.info(error) } } From 2f2738e4ae43c602c01d3676d556e6a5d1c89e7a Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 14:01:29 +0000 Subject: [PATCH 53/72] removing old packages --- packages/artifact/package-lock.json | 289 ---------------------------- packages/artifact/package.json | 4 +- 2 files changed, 1 insertion(+), 292 deletions(-) diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index 9710156d7b..e665eaf935 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -18,12 +18,10 @@ "@octokit/plugin-retry": "^3.0.9", "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", - "archiver": "^5.3.1", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", "unzip-stream": "^0.3.1", - "why-is-node-running": "^2.2.2", "zip-stream": "^6.0.1" }, "devDependencies": { @@ -546,134 +544,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/archiver": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.2.tgz", - "integrity": "sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw==", - "dependencies": { - "archiver-utils": "^2.1.0", - "async": "^3.2.4", - "buffer-crc32": "^0.2.1", - "readable-stream": "^3.6.0", - "readdir-glob": "^1.1.2", - "tar-stream": "^2.2.0", - "zip-stream": "^4.1.0" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/archiver-utils": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", - "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", - "dependencies": { - "glob": "^7.1.4", - "graceful-fs": "^4.2.0", - "lazystream": "^1.0.0", - "lodash.defaults": "^4.2.0", - "lodash.difference": "^4.5.0", - "lodash.flatten": "^4.4.0", - "lodash.isplainobject": "^4.0.6", - "lodash.union": "^4.6.0", - "normalize-path": "^3.0.0", - "readable-stream": "^2.0.0" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/archiver-utils/node_modules/readable-stream": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", - "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/archiver-utils/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/archiver-utils/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/archiver/node_modules/compress-commons": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.2.tgz", - "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", - "dependencies": { - "buffer-crc32": "^0.2.13", - "crc32-stream": "^4.0.2", - "normalize-path": "^3.0.0", - "readable-stream": "^3.6.0" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/archiver/node_modules/crc32-stream": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.3.tgz", - "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", - "dependencies": { - "crc-32": "^1.2.0", - "readable-stream": "^3.4.0" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/archiver/node_modules/zip-stream": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", - "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", - "dependencies": { - "archiver-utils": "^3.0.4", - "compress-commons": "^4.1.2", - "readable-stream": "^3.6.0" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/archiver/node_modules/zip-stream/node_modules/archiver-utils": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-3.0.4.tgz", - "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", - "dependencies": { - "glob": "^7.2.3", - "graceful-fs": "^4.2.0", - "lazystream": "^1.0.0", - "lodash.defaults": "^4.2.0", - "lodash.difference": "^4.5.0", - "lodash.flatten": "^4.4.0", - "lodash.isplainobject": "^4.0.6", - "lodash.union": "^4.6.0", - "normalize-path": "^3.0.0", - "readable-stream": "^3.6.0" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/async": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", - "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -720,16 +590,6 @@ "node": "*" } }, - "node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, "node_modules/bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", @@ -744,37 +604,6 @@ "concat-map": "0.0.1" } }, - "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "engines": { - "node": "*" - } - }, "node_modules/buffers": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", @@ -1016,14 +845,6 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, - "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dependencies": { - "once": "^1.4.0" - } - }, "node_modules/event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -1068,11 +889,6 @@ "node": ">= 6" } }, - "node_modules/fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" - }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1264,31 +1080,6 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, - "node_modules/lodash.defaults": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", - "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" - }, - "node_modules/lodash.difference": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", - "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==" - }, - "node_modules/lodash.flatten": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", - "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==" - }, - "node_modules/lodash.isplainobject": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", - "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==" - }, - "node_modules/lodash.union": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", - "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==" - }, "node_modules/lower-case": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", @@ -1502,46 +1293,6 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, - "node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/readdir-glob": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", - "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", - "dependencies": { - "minimatch": "^5.1.0" - } - }, - "node_modules/readdir-glob/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/readdir-glob/node_modules/minimatch": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", - "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1597,11 +1348,6 @@ "vscode-textmate": "^8.0.0" } }, - "node_modules/siginfo": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", - "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==" - }, "node_modules/signal-exit": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", @@ -1622,11 +1368,6 @@ "node": ">=0.10.0" } }, - "node_modules/stackback": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", - "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==" - }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -1723,21 +1464,6 @@ "node": ">=8" } }, - "node_modules/tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "dependencies": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", @@ -1951,21 +1677,6 @@ "node": ">= 8" } }, - "node_modules/why-is-node-running": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.2.2.tgz", - "integrity": "sha512-6tSwToZxTOcotxHeA+qGCq1mVzKR3CwcJGmVcY+QE8SHy6TnpFnh8PAvPNHYr7EcuVeG0QSMxtYCuO1ta/G/oA==", - "dependencies": { - "siginfo": "^2.0.0", - "stackback": "0.0.2" - }, - "bin": { - "why-is-node-running": "cli.js" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/wordwrap": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", diff --git a/packages/artifact/package.json b/packages/artifact/package.json index 166438f4ab..e464e2b63a 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -49,12 +49,10 @@ "@octokit/plugin-retry": "^3.0.9", "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", - "archiver": "^5.3.1", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", "unzip-stream": "^0.3.1", - "why-is-node-running": "^2.2.2", "zip-stream": "^6.0.1" }, "devDependencies": { @@ -64,4 +62,4 @@ "typedoc-plugin-markdown": "^3.17.1", "typescript": "^5.2.2" } -} \ No newline at end of file +} From 4c878a66e79ec19d1aee1c882c020ed9cdea74eb Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 14:30:14 +0000 Subject: [PATCH 54/72] adding back archiver --- packages/artifact/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/artifact/package.json b/packages/artifact/package.json index e464e2b63a..b267ff7837 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -56,10 +56,9 @@ "zip-stream": "^6.0.1" }, "devDependencies": { - "@types/archiver": "^5.3.2", "@types/unzip-stream": "^0.3.4", "typedoc": "^0.25.4", "typedoc-plugin-markdown": "^3.17.1", "typescript": "^5.2.2" } -} +} \ No newline at end of file From dea35958810f6eba0981348e8b5d0af453c9f5ac Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 14:33:36 +0000 Subject: [PATCH 55/72] re-adding archiver for download --- packages/artifact/package-lock.json | 282 ++++++++++++++++++++++++++-- packages/artifact/package.json | 3 +- 2 files changed, 265 insertions(+), 20 deletions(-) diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index e665eaf935..41b90a9baa 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -18,6 +18,7 @@ "@octokit/plugin-retry": "^3.0.9", "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", + "archiver": "^5.3.1", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", @@ -25,7 +26,6 @@ "zip-stream": "^6.0.1" }, "devDependencies": { - "@types/archiver": "^5.3.2", "@types/unzip-stream": "^0.3.4", "typedoc": "^0.25.4", "typedoc-plugin-markdown": "^3.17.1", @@ -443,15 +443,6 @@ "@protobuf-ts/runtime": "^2.9.1" } }, - "node_modules/@types/archiver": { - "version": "5.3.4", - "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-5.3.4.tgz", - "integrity": "sha512-Lj7fLBIMwYFgViVVZHEdExZC3lVYsl+QL0VmdNdIzGZH544jHveYWij6qdnBgJQDnR7pMKliN9z2cPZFEbhyPw==", - "dev": true, - "dependencies": { - "@types/readdir-glob": "*" - } - }, "node_modules/@types/node": { "version": "20.4.9", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.4.9.tgz", @@ -479,15 +470,6 @@ "node": ">= 6" } }, - "node_modules/@types/readdir-glob": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@types/readdir-glob/-/readdir-glob-1.1.5.tgz", - "integrity": "sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/tunnel": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", @@ -544,6 +526,134 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/archiver": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.1.tgz", + "integrity": "sha512-8KyabkmbYrH+9ibcTScQ1xCJC/CGcugdVIwB+53f5sZziXgwUh3iXlAlANMxcZyDEfTHMe6+Z5FofV8nopXP7w==", + "dependencies": { + "archiver-utils": "^2.1.0", + "async": "^3.2.3", + "buffer-crc32": "^0.2.1", + "readable-stream": "^3.6.0", + "readdir-glob": "^1.0.0", + "tar-stream": "^2.2.0", + "zip-stream": "^4.1.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", + "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", + "dependencies": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/archiver-utils/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/archiver-utils/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/archiver-utils/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/archiver/node_modules/compress-commons": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.2.tgz", + "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", + "dependencies": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^4.0.2", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver/node_modules/crc32-stream": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.3.tgz", + "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^3.4.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver/node_modules/zip-stream": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", + "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", + "dependencies": { + "archiver-utils": "^3.0.4", + "compress-commons": "^4.1.2", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver/node_modules/zip-stream/node_modules/archiver-utils": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-3.0.4.tgz", + "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", + "dependencies": { + "glob": "^7.2.3", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/async": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", + "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -590,6 +700,16 @@ "node": "*" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", @@ -604,6 +724,37 @@ "concat-map": "0.0.1" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "engines": { + "node": "*" + } + }, "node_modules/buffers": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", @@ -845,6 +996,14 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -889,6 +1048,11 @@ "node": ">= 6" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1080,6 +1244,31 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, + "node_modules/lodash.difference": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", + "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==" + }, + "node_modules/lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==" + }, + "node_modules/lodash.union": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", + "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==" + }, "node_modules/lower-case": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", @@ -1293,6 +1482,46 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1464,6 +1693,21 @@ "node": ">=8" } }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", diff --git a/packages/artifact/package.json b/packages/artifact/package.json index b267ff7837..292ba63d8c 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -49,6 +49,7 @@ "@octokit/plugin-retry": "^3.0.9", "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", + "archiver": "^5.3.1", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", @@ -61,4 +62,4 @@ "typedoc-plugin-markdown": "^3.17.1", "typescript": "^5.2.2" } -} \ No newline at end of file +} From 2c4f0f555e3d3adda9b45907a9ebdd1720cc32ed Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 15:07:28 +0000 Subject: [PATCH 56/72] adding error checks --- packages/artifact/src/internal/upload/zip.ts | 2 ++ packages/attest/package-lock.json | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index f62d560376..022330a516 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -50,6 +50,7 @@ export async function createZipUploadStream( createReadStream(file.sourcePath), {name: file.destinationPath}, function (err, entry) { + core.debug(`${err}`) if (err) reject(err) else resolve(entry) } @@ -60,6 +61,7 @@ export async function createZipUploadStream( null, {name: `${file.destinationPath}/`}, function (err, entry) { + core.debug(`${err}`) if (err) reject(err) else resolve(entry) } diff --git a/packages/attest/package-lock.json b/packages/attest/package-lock.json index f1c537719d..949dce3e44 100644 --- a/packages/attest/package-lock.json +++ b/packages/attest/package-lock.json @@ -1,12 +1,12 @@ { "name": "@actions/attest", - "version": "1.0.0", + "version": "1.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@actions/attest", - "version": "1.0.0", + "version": "1.1.0", "license": "MIT", "dependencies": { "@actions/core": "^1.10.1", From 6eff4e928d591b8c91f1f7ac7ef94facf75c58a7 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 15:15:28 +0000 Subject: [PATCH 57/72] adding more debug statements --- packages/artifact/src/internal/upload/zip.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 022330a516..639ef3f4bb 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -45,9 +45,13 @@ export async function createZipUploadStream( for (const file of uploadSpecification) { await new Promise((resolve, reject) => { if (file.sourcePath !== null) { + core.debug(`createReadStream with: ${file.sourcePath}`) // Add a normal file to the zip + const readsstream = createReadStream(file.sourcePath) + readsstream.on('error', reject) + zip.entry( - createReadStream(file.sourcePath), + readsstream, {name: file.destinationPath}, function (err, entry) { core.debug(`${err}`) @@ -57,6 +61,7 @@ export async function createZipUploadStream( ) } else { // add directory to zip + core.debug(`add directory with: ${file.destinationPath}`) zip.entry( null, {name: `${file.destinationPath}/`}, From 4778aebf5b73c0fe83d0db054b53d60792a93f69 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 16:54:55 +0000 Subject: [PATCH 58/72] adding asnyc handler back --- packages/artifact/package-lock.json | 1 + packages/artifact/package.json | 1 + .../src/internal/upload/upload-artifact.ts | 6 +- packages/artifact/src/internal/upload/zip.ts | 113 +++++++++++++----- 4 files changed, 92 insertions(+), 29 deletions(-) diff --git a/packages/artifact/package-lock.json b/packages/artifact/package-lock.json index 41b90a9baa..bc503a710d 100644 --- a/packages/artifact/package-lock.json +++ b/packages/artifact/package-lock.json @@ -19,6 +19,7 @@ "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", "archiver": "^5.3.1", + "async": "^3.2.5", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", diff --git a/packages/artifact/package.json b/packages/artifact/package.json index 292ba63d8c..104a5d67e8 100644 --- a/packages/artifact/package.json +++ b/packages/artifact/package.json @@ -50,6 +50,7 @@ "@octokit/request-error": "^5.0.0", "@protobuf-ts/plugin": "^2.2.3-alpha.1", "archiver": "^5.3.1", + "async": "^3.2.5", "crypto": "^1.0.1", "jwt-decode": "^3.1.2", "twirp-ts": "^2.5.0", diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index b8f952014b..e810ccc60c 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -71,7 +71,11 @@ export async function uploadArtifact( const zipUploadStream = await createZipUploadStream( zipSpecification, options?.compressionLevel - ) + ).catch(err => { + throw new InvalidResponseError( + `createZipUploadStream: response from backend was not ok: ${err}` + ) + }) // Upload zip to blob storage const uploadResult = await uploadZipToBlobStorage( diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 639ef3f4bb..974df55aff 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -1,6 +1,7 @@ import * as stream from 'stream' import * as ZipStream from 'zip-stream' import * as core from '@actions/core' +import async from 'async' import {createReadStream} from 'fs' import {UploadZipSpecification} from './upload-zip-specification' import {getUploadChunkSize} from '../shared/config' @@ -42,39 +43,95 @@ export async function createZipUploadStream( zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) - for (const file of uploadSpecification) { - await new Promise((resolve, reject) => { - if (file.sourcePath !== null) { - core.debug(`createReadStream with: ${file.sourcePath}`) - // Add a normal file to the zip - const readsstream = createReadStream(file.sourcePath) - readsstream.on('error', reject) + // for (const file of uploadSpecification) { + // await new Promise((resolve, reject) => { + // if (file.sourcePath !== null) { + // core.debug(`createReadStream with: ${file.sourcePath}`) + // // Add a normal file to the zip + // const readStream = createReadStream(file.sourcePath) + // readStream.on('data', chunk => { + // core.debug(`Received ${chunk.length} bytes of data.`) + // }) + // readStream.on('end', () => { + // core.debug('There will be no more data.') + // }) + // readStream.on('error', reject) // Catch any errors from createReadStream + + // core.debug(`readsstream errors: ${readStream.errored}`) + // const entry = zip.entry( + // readStream, + // {name: file.destinationPath}, + // function (err) { + // core.debug(`Is stream paused: ${readStream.isPaused()}`) + // if (err) { + // core.error('An error occurred:', err) + // reject(err) + // } + // core.debug(`Is stream paused: ${readStream.isPaused()}`) + // resolve('resolved artifact') + // } + // ) + // readStream.pipe(entry) + // } else { + // zip.entry(null, {name: `${file.destinationPath}/`}, function (err) { + // if (err) { + // core.error('An error occurred:', err) + // reject(err) + // } + // resolve('resolved artifact') + // }) + // } + // }) + // } + const fileUploadQueue = async.queue(function (task, callback) { + core.info(`hello ${task.name}`) + callback() + }, 1) - zip.entry( - readsstream, - {name: file.destinationPath}, - function (err, entry) { - core.debug(`${err}`) - if (err) reject(err) - else resolve(entry) + fileUploadQueue.error(function (err, task) { + core.error(`task experienced an error: ${task} ${err}`) + }) + + for (const file of uploadSpecification) { + if (file.sourcePath !== null) { + const readStream = createReadStream(file.sourcePath) + readStream.on('data', chunk => { + core.debug(`Received ${chunk.length} bytes of data.`) + }) + readStream.on('end', () => { + core.debug('There will be no more data.') + }) + readStream.on('error', function (err) { + core.debug(`${err}`) + }) // Catch any errors from createReadStream + fileUploadQueue.push( + zip.entry(readStream, {name: file.destinationPath}, function (err) { + core.debug(`Is stream paused: ${readStream.isPaused()}`) + if (err) { + core.error('An error occurred:', err) } - ) - } else { - // add directory to zip - core.debug(`add directory with: ${file.destinationPath}`) - zip.entry( - null, - {name: `${file.destinationPath}/`}, - function (err, entry) { - core.debug(`${err}`) - if (err) reject(err) - else resolve(entry) + core.debug(`Is stream paused: ${readStream.isPaused()}`) + }) + ) + } else { + fileUploadQueue.push( + zip.entry(null, {name: `${file.destinationPath}/`}, function (err) { + if (err) { + core.error('An error occurred:', err) } - ) - } - }) + }) + ) + } } + core.debug(`Starting the finalizing of all entries`) + + for (const item of fileUploadQueue) { + core.debug(`Starting the finalizing ${item}`) + } + fileUploadQueue.drain(() => { + core.debug('all items have been processed') + }) zip.finalize() core.debug(`Finalizing entries`) const bufferSize = getUploadChunkSize() From 23039a4345496b70f215e573a013ef686f5eb388 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 16:57:50 +0000 Subject: [PATCH 59/72] adding asnyc handler back --- packages/artifact/src/internal/upload/zip.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 974df55aff..dae88d9adf 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -84,7 +84,7 @@ export async function createZipUploadStream( // }) // } const fileUploadQueue = async.queue(function (task, callback) { - core.info(`hello ${task.name}`) + core.debug(`adding file to upload queue ${task}`) callback() }, 1) @@ -126,9 +126,6 @@ export async function createZipUploadStream( core.debug(`Starting the finalizing of all entries`) - for (const item of fileUploadQueue) { - core.debug(`Starting the finalizing ${item}`) - } fileUploadQueue.drain(() => { core.debug('all items have been processed') }) From a8fa53b6090ef9eaf0853135859a8b3d1728f1dc Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 19:57:11 +0000 Subject: [PATCH 60/72] test async eachof --- packages/artifact/src/internal/upload/zip.ts | 107 +++++++++++-------- 1 file changed, 65 insertions(+), 42 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index dae88d9adf..4ac9b0e22a 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -22,7 +22,26 @@ export class ZipUploadStream extends stream.Transform { cb(null, chunk) } } - +export async function zipFileUpload(file, zip, callback): Promise { + const readStream = createReadStream(file.sourcePath) + if (file.sourcePath !== null) { + zip.entry(readStream, {name: file.destinationPath}, function (err) { + core.debug(`Is stream paused: ${readStream.isPaused()}`) + if (err) { + core.error('An error occurred:', err) + return callback(err) + } + core.debug(`Is stream paused: ${readStream.isPaused()}`) + }) + } else { + zip.entry(null, {name: `${file.destinationPath}/`}, function (err) { + if (err) { + core.error('An error occurred:', err) + } + }) + } + callback() +} export async function createZipUploadStream( uploadSpecification: UploadZipSpecification[], compressionLevel: number = DEFAULT_COMPRESSION_LEVEL @@ -83,51 +102,55 @@ export async function createZipUploadStream( // } // }) // } - const fileUploadQueue = async.queue(function (task, callback) { - core.debug(`adding file to upload queue ${task}`) - callback() - }, 1) + // see https://caolan.github.io/async/v3/docs.html#queue for options + // const fileUploadQueue = async.queue() // concurrency for uploads automatically set to 1 - fileUploadQueue.error(function (err, task) { - core.error(`task experienced an error: ${task} ${err}`) - }) + // fileUploadQueue.error(function (err, task) { + // core.error(`task experienced an error: ${task} ${err}`) + // }) - for (const file of uploadSpecification) { - if (file.sourcePath !== null) { - const readStream = createReadStream(file.sourcePath) - readStream.on('data', chunk => { - core.debug(`Received ${chunk.length} bytes of data.`) - }) - readStream.on('end', () => { - core.debug('There will be no more data.') - }) - readStream.on('error', function (err) { - core.debug(`${err}`) - }) // Catch any errors from createReadStream - fileUploadQueue.push( - zip.entry(readStream, {name: file.destinationPath}, function (err) { - core.debug(`Is stream paused: ${readStream.isPaused()}`) - if (err) { - core.error('An error occurred:', err) - } - core.debug(`Is stream paused: ${readStream.isPaused()}`) - }) - ) - } else { - fileUploadQueue.push( - zip.entry(null, {name: `${file.destinationPath}/`}, function (err) { - if (err) { - core.error('An error occurred:', err) - } - }) - ) - } - } + // for (const file of uploadSpecification) { + // if (file.sourcePath !== null) { + // const readStream = createReadStream(file.sourcePath) + // readStream.on('data', chunk => { + // core.debug(`Received ${chunk.length} bytes of data.`) + // }) + // readStream.on('end', () => { + // core.debug('There will be no more data.') + // }) + // readStream.on('error', function (err) { + // core.debug(`${err}`) + // }) // Catch any errors from createReadStream + // fileUploadQueue.push( + // zip.entry(readStream, {name: file.destinationPath}, function (err) { + // core.debug(`Is stream paused: ${readStream.isPaused()}`) + // if (err) { + // core.error('An error occurred:', err) + // } + // core.debug(`Is stream paused: ${readStream.isPaused()}`) + // }) + // ) + // } else { + // fileUploadQueue.push( + // zip.entry(null, {name: `${file.destinationPath}/`}, function (err) { + // if (err) { + // core.error('An error occurred:', err) + // } + // }) + // ) + // } + // } - core.debug(`Starting the finalizing of all entries`) + // core.debug(`Starting the finalizing of all entries`) - fileUploadQueue.drain(() => { - core.debug('all items have been processed') + // fileUploadQueue.drain(() => { + // core.debug('all items have been processed') + // }) + + async.forEachOf(uploadSpecification, zipFileUpload, function (err) { + if (err) { + core.error('An error occurred:', err) + } }) zip.finalize() core.debug(`Finalizing entries`) From a2a8a724c22ff905cbbbfac243be62ced7170629 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 20:01:19 +0000 Subject: [PATCH 61/72] test queue --- packages/artifact/src/internal/upload/zip.ts | 104 +++++++------------ 1 file changed, 40 insertions(+), 64 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 4ac9b0e22a..2241d2e6ce 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -22,26 +22,7 @@ export class ZipUploadStream extends stream.Transform { cb(null, chunk) } } -export async function zipFileUpload(file, zip, callback): Promise { - const readStream = createReadStream(file.sourcePath) - if (file.sourcePath !== null) { - zip.entry(readStream, {name: file.destinationPath}, function (err) { - core.debug(`Is stream paused: ${readStream.isPaused()}`) - if (err) { - core.error('An error occurred:', err) - return callback(err) - } - core.debug(`Is stream paused: ${readStream.isPaused()}`) - }) - } else { - zip.entry(null, {name: `${file.destinationPath}/`}, function (err) { - if (err) { - core.error('An error occurred:', err) - } - }) - } - callback() -} + export async function createZipUploadStream( uploadSpecification: UploadZipSpecification[], compressionLevel: number = DEFAULT_COMPRESSION_LEVEL @@ -103,55 +84,50 @@ export async function createZipUploadStream( // }) // } // see https://caolan.github.io/async/v3/docs.html#queue for options - // const fileUploadQueue = async.queue() // concurrency for uploads automatically set to 1 + const fileUploadQueue = async.queue() // concurrency for uploads automatically set to 1 - // fileUploadQueue.error(function (err, task) { - // core.error(`task experienced an error: ${task} ${err}`) - // }) - - // for (const file of uploadSpecification) { - // if (file.sourcePath !== null) { - // const readStream = createReadStream(file.sourcePath) - // readStream.on('data', chunk => { - // core.debug(`Received ${chunk.length} bytes of data.`) - // }) - // readStream.on('end', () => { - // core.debug('There will be no more data.') - // }) - // readStream.on('error', function (err) { - // core.debug(`${err}`) - // }) // Catch any errors from createReadStream - // fileUploadQueue.push( - // zip.entry(readStream, {name: file.destinationPath}, function (err) { - // core.debug(`Is stream paused: ${readStream.isPaused()}`) - // if (err) { - // core.error('An error occurred:', err) - // } - // core.debug(`Is stream paused: ${readStream.isPaused()}`) - // }) - // ) - // } else { - // fileUploadQueue.push( - // zip.entry(null, {name: `${file.destinationPath}/`}, function (err) { - // if (err) { - // core.error('An error occurred:', err) - // } - // }) - // ) - // } - // } + fileUploadQueue.error(function (err, task) { + core.error(`task experienced an error: ${task} ${err}`) + }) - // core.debug(`Starting the finalizing of all entries`) + for (const file of uploadSpecification) { + if (file.sourcePath !== null) { + const readStream = createReadStream(file.sourcePath) + readStream.on('data', chunk => { + core.debug(`Received ${chunk.length} bytes of data.`) + }) + readStream.on('end', () => { + core.debug('There will be no more data.') + }) + readStream.on('error', function (err) { + core.debug(`${err}`) + }) // Catch any errors from createReadStream + fileUploadQueue.push( + zip.entry(readStream, {name: file.destinationPath}, function (err) { + core.debug(`Is stream paused: ${readStream.isPaused()}`) + if (err) { + core.error('An error occurred:', err) + } + core.debug(`Is stream paused: ${readStream.isPaused()}`) + }) + ) + } else { + fileUploadQueue.push( + zip.entry(null, {name: `${file.destinationPath}/`}, function (err) { + if (err) { + core.error('An error occurred:', err) + } + }) + ) + } + } - // fileUploadQueue.drain(() => { - // core.debug('all items have been processed') - // }) + core.debug(`Starting the finalizing of all entries`) - async.forEachOf(uploadSpecification, zipFileUpload, function (err) { - if (err) { - core.error('An error occurred:', err) - } + fileUploadQueue.drain(() => { + core.debug('all items have been processed') }) + zip.finalize() core.debug(`Finalizing entries`) const bufferSize = getUploadChunkSize() From a926fff3857a6f236c8505448216652a160d6405 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 20:04:49 +0000 Subject: [PATCH 62/72] adding queue back --- packages/artifact/src/internal/upload/zip.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 2241d2e6ce..c5fcb881a0 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -84,7 +84,10 @@ export async function createZipUploadStream( // }) // } // see https://caolan.github.io/async/v3/docs.html#queue for options - const fileUploadQueue = async.queue() // concurrency for uploads automatically set to 1 + const fileUploadQueue = async.queue(function (task, callback) { + core.debug(`adding file to upload queue ${task}`) + callback() + }) // concurrency for uploads automatically set to 1 fileUploadQueue.error(function (err, task) { core.error(`task experienced an error: ${task} ${err}`) @@ -127,7 +130,6 @@ export async function createZipUploadStream( fileUploadQueue.drain(() => { core.debug('all items have been processed') }) - zip.finalize() core.debug(`Finalizing entries`) const bufferSize = getUploadChunkSize() From 84d3cef541863a574cef91efb4969a1744ea9718 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 20:14:07 +0000 Subject: [PATCH 63/72] Removing extra logs --- packages/artifact/src/internal/upload/zip.ts | 48 +------------------- 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index c5fcb881a0..68e24636eb 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -42,47 +42,6 @@ export async function createZipUploadStream( zip.on('warning', zipWarningCallback) zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) - - // for (const file of uploadSpecification) { - // await new Promise((resolve, reject) => { - // if (file.sourcePath !== null) { - // core.debug(`createReadStream with: ${file.sourcePath}`) - // // Add a normal file to the zip - // const readStream = createReadStream(file.sourcePath) - // readStream.on('data', chunk => { - // core.debug(`Received ${chunk.length} bytes of data.`) - // }) - // readStream.on('end', () => { - // core.debug('There will be no more data.') - // }) - // readStream.on('error', reject) // Catch any errors from createReadStream - - // core.debug(`readsstream errors: ${readStream.errored}`) - // const entry = zip.entry( - // readStream, - // {name: file.destinationPath}, - // function (err) { - // core.debug(`Is stream paused: ${readStream.isPaused()}`) - // if (err) { - // core.error('An error occurred:', err) - // reject(err) - // } - // core.debug(`Is stream paused: ${readStream.isPaused()}`) - // resolve('resolved artifact') - // } - // ) - // readStream.pipe(entry) - // } else { - // zip.entry(null, {name: `${file.destinationPath}/`}, function (err) { - // if (err) { - // core.error('An error occurred:', err) - // reject(err) - // } - // resolve('resolved artifact') - // }) - // } - // }) - // } // see https://caolan.github.io/async/v3/docs.html#queue for options const fileUploadQueue = async.queue(function (task, callback) { core.debug(`adding file to upload queue ${task}`) @@ -96,22 +55,17 @@ export async function createZipUploadStream( for (const file of uploadSpecification) { if (file.sourcePath !== null) { const readStream = createReadStream(file.sourcePath) - readStream.on('data', chunk => { - core.debug(`Received ${chunk.length} bytes of data.`) - }) readStream.on('end', () => { - core.debug('There will be no more data.') + core.debug('The upload read stream is ending') }) readStream.on('error', function (err) { core.debug(`${err}`) }) // Catch any errors from createReadStream fileUploadQueue.push( zip.entry(readStream, {name: file.destinationPath}, function (err) { - core.debug(`Is stream paused: ${readStream.isPaused()}`) if (err) { core.error('An error occurred:', err) } - core.debug(`Is stream paused: ${readStream.isPaused()}`) }) ) } else { From 96ef8d5d33c7290744608a5ec754df3012384e5b Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 1 Apr 2024 20:21:01 +0000 Subject: [PATCH 64/72] adding more error handling --- packages/artifact/src/internal/upload/zip.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 68e24636eb..e5f5b6af8e 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -43,9 +43,14 @@ export async function createZipUploadStream( zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) // see https://caolan.github.io/async/v3/docs.html#queue for options - const fileUploadQueue = async.queue(function (task, callback) { - core.debug(`adding file to upload queue ${task}`) - callback() + const fileUploadQueue = async.queue(function (fileItem, callback) { + try { + core.debug(`adding ${fileItem} to the file queue`) + callback() + } catch (err) { + core.error(`task experienced an error: ${fileItem} ${err}`) + callback(err) + } }) // concurrency for uploads automatically set to 1 fileUploadQueue.error(function (err, task) { From c5f3463c0be6740f7d512b54b77cfcef8c7608d6 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Tue, 2 Apr 2024 05:25:10 +0000 Subject: [PATCH 65/72] adding more descriptive error --- packages/artifact/src/internal/upload/zip.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index e5f5b6af8e..9c016650b2 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -69,7 +69,7 @@ export async function createZipUploadStream( fileUploadQueue.push( zip.entry(readStream, {name: file.destinationPath}, function (err) { if (err) { - core.error('An error occurred:', err) + core.error('A file entry error occurred:', err) } }) ) @@ -77,7 +77,7 @@ export async function createZipUploadStream( fileUploadQueue.push( zip.entry(null, {name: `${file.destinationPath}/`}, function (err) { if (err) { - core.error('An error occurred:', err) + core.error('A directory entry error occurred:', err) } }) ) From 38f54e9d738bbaf3f7f76b9a20b76a21b0a31010 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Tue, 2 Apr 2024 05:39:02 +0000 Subject: [PATCH 66/72] Adding debug statement for upload --- packages/artifact/src/internal/upload/zip.ts | 24 ++++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 9c016650b2..4dbcda2ee4 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -66,20 +66,30 @@ export async function createZipUploadStream( readStream.on('error', function (err) { core.debug(`${err}`) }) // Catch any errors from createReadStream - fileUploadQueue.push( - zip.entry(readStream, {name: file.destinationPath}, function (err) { + const fileEntry = zip.entry( + readStream, + {name: file.destinationPath}, + function (err, entry) { if (err) { core.error('A file entry error occurred:', err) } - }) + core.debug(`File entry was succesfull: ${entry}`) + } ) + + fileUploadQueue.push(fileEntry) } else { fileUploadQueue.push( - zip.entry(null, {name: `${file.destinationPath}/`}, function (err) { - if (err) { - core.error('A directory entry error occurred:', err) + zip.entry( + null, + {name: `${file.destinationPath}/`}, + function (err, entry) { + if (err) { + core.error('A directory entry error occurred:', err) + } + core.debug(`File entry was succesfull: ${entry}`) } - }) + ) ) } } From ce603525f8c6e60710f9427b8add967a90744c58 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 8 Apr 2024 15:27:00 +0000 Subject: [PATCH 67/72] adding asysnc zip entry logic handling --- packages/artifact/src/internal/upload/zip.ts | 97 ++++++++------------ 1 file changed, 37 insertions(+), 60 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 4dbcda2ee4..579757c3d7 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -22,7 +22,9 @@ export class ZipUploadStream extends stream.Transform { cb(null, chunk) } } - +interface NodeJSError extends Error { + code?: string +} export async function createZipUploadStream( uploadSpecification: UploadZipSpecification[], compressionLevel: number = DEFAULT_COMPRESSION_LEVEL @@ -37,73 +39,49 @@ export async function createZipUploadStream( } } const zip = new ZipStream.default(zlibOptions) + const bufferSize = getUploadChunkSize() + const zipUploadStream = new ZipUploadStream(bufferSize) + // register callbacks for various events during the zip lifecycle zip.on('error', zipErrorCallback) zip.on('warning', zipWarningCallback) zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) - // see https://caolan.github.io/async/v3/docs.html#queue for options - const fileUploadQueue = async.queue(function (fileItem, callback) { - try { - core.debug(`adding ${fileItem} to the file queue`) - callback() - } catch (err) { - core.error(`task experienced an error: ${fileItem} ${err}`) - callback(err) - } - }) // concurrency for uploads automatically set to 1 - - fileUploadQueue.error(function (err, task) { - core.error(`task experienced an error: ${task} ${err}`) - }) - - for (const file of uploadSpecification) { + const addFileToZip = ( + file: UploadZipSpecification, + callback: (error?: Error) => void + ): void => { if (file.sourcePath !== null) { - const readStream = createReadStream(file.sourcePath) - readStream.on('end', () => { - core.debug('The upload read stream is ending') - }) - readStream.on('error', function (err) { - core.debug(`${err}`) - }) // Catch any errors from createReadStream - const fileEntry = zip.entry( - readStream, + zip.entry( + createReadStream(file.sourcePath), {name: file.destinationPath}, - function (err, entry) { - if (err) { - core.error('A file entry error occurred:', err) + (error: NodeJSError) => { + if (error) { + callback(error) + return } - core.debug(`File entry was succesfull: ${entry}`) + callback() } ) - - fileUploadQueue.push(fileEntry) } else { - fileUploadQueue.push( - zip.entry( - null, - {name: `${file.destinationPath}/`}, - function (err, entry) { - if (err) { - core.error('A directory entry error occurred:', err) - } - core.debug(`File entry was succesfull: ${entry}`) - } - ) - ) + zip.entry('', {name: file.destinationPath}, (error: NodeJSError) => { + if (error) { + callback(error) + return + } + callback() + }) } } - core.debug(`Starting the finalizing of all entries`) - - fileUploadQueue.drain(() => { - core.debug('all items have been processed') + async.eachSeries(uploadSpecification, addFileToZip, (error: NodeJSError) => { + if (error) { + core.error('Failed to add a file to the zip:') + core.info(error.toString()) + return + } + zip.finalize() // Finalize the archive once all files have been added }) - zip.finalize() - core.debug(`Finalizing entries`) - const bufferSize = getUploadChunkSize() - const zipUploadStream = new ZipUploadStream(bufferSize) - zip.pipe(zipUploadStream) // Pipe the zip stream into zipUploadStream core.debug( `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` @@ -111,28 +89,27 @@ export async function createZipUploadStream( core.debug( `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` ) + return zipUploadStream } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const zipErrorCallback = (error: any): void => { +const zipErrorCallback = (error: Error): void => { core.error('An error has occurred while creating the zip file for upload') - core.info(error) + core.info(error.message) throw new Error('An error has occurred during zip creation for the artifact') } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const zipWarningCallback = (error: any): void => { +const zipWarningCallback = (error: NodeJSError): void => { if (error.code === 'ENOENT') { core.warning( 'ENOENT warning during artifact zip creation. No such file or directory' ) - core.info(error) + core.info(error.toString()) // Convert error object to string } else { core.warning( `A non-blocking warning has occurred during artifact zip creation: ${error.code}` ) - core.info(error) + core.info(error.toString()) // Convert error object to string } } From 4f54b861cfb4792adca0680f3906d27a0cd66526 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 8 Apr 2024 15:40:32 +0000 Subject: [PATCH 68/72] updating errors --- .../src/internal/upload/upload-artifact.ts | 205 +++++++++--------- packages/artifact/src/internal/upload/zip.ts | 33 +-- 2 files changed, 123 insertions(+), 115 deletions(-) diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index e810ccc60c..6e174d8ee0 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -1,119 +1,124 @@ +import * as stream from 'stream' +import * as ZipStream from 'zip-stream' import * as core from '@actions/core' -import { - UploadArtifactOptions, - UploadArtifactResponse -} from '../shared/interfaces' -import {getExpiration} from './retention' -import {validateArtifactName} from './path-and-artifact-name-validation' -import {internalArtifactTwirpClient} from '../shared/artifact-twirp-client' -import { - UploadZipSpecification, - getUploadZipSpecification, - validateRootDirectory -} from './upload-zip-specification' -import {getBackendIdsFromToken} from '../shared/util' -import {uploadZipToBlobStorage} from './blob-upload' -import {createZipUploadStream} from './zip' -import { - CreateArtifactRequest, - FinalizeArtifactRequest, - StringValue -} from '../../generated' -import {FilesNotFoundError, InvalidResponseError} from '../shared/errors' +import async from 'async' +import {createReadStream} from 'fs' +import {UploadZipSpecification} from './upload-zip-specification' +import {getUploadChunkSize} from '../shared/config' -export async function uploadArtifact( - name: string, - files: string[], - rootDirectory: string, - options?: UploadArtifactOptions | undefined -): Promise { - validateArtifactName(name) - validateRootDirectory(rootDirectory) +export const DEFAULT_COMPRESSION_LEVEL = 6 - const zipSpecification: UploadZipSpecification[] = getUploadZipSpecification( - files, - rootDirectory - ) - if (zipSpecification.length === 0) { - throw new FilesNotFoundError( - zipSpecification.flatMap(s => (s.sourcePath ? [s.sourcePath] : [])) - ) +// Custom stream transformer so we can set the highWaterMark property +// See https://github.com/nodejs/node/issues/8855 +export class ZipUploadStream extends stream.Transform { + constructor(bufferSize: number) { + super({ + highWaterMark: bufferSize + }) } - // get the IDs needed for the artifact creation - const backendIds = getBackendIdsFromToken() + // eslint-disable-next-line @typescript-eslint/no-explicit-any + _transform(chunk: any, enc: any, cb: any): void { + cb(null, chunk) + } +} - // create the artifact client - const artifactClient = internalArtifactTwirpClient() +export async function createZipUploadStream( + uploadSpecification: UploadZipSpecification[], + compressionLevel: number = DEFAULT_COMPRESSION_LEVEL +): Promise { + core.debug( + `Creating Artifact archive with compressionLevel: ${compressionLevel}` + ) - // create the artifact - const createArtifactReq: CreateArtifactRequest = { - workflowRunBackendId: backendIds.workflowRunBackendId, - workflowJobRunBackendId: backendIds.workflowJobRunBackendId, - name, - version: 4 + const zlibOptions = { + zlib: { + level: compressionLevel, + bufferSize: getUploadChunkSize() + } } + const zip = new ZipStream.default(zlibOptions) - // if there is a retention period, add it to the request - const expiresAt = getExpiration(options?.retentionDays) - if (expiresAt) { - createArtifactReq.expiresAt = expiresAt + const bufferSize = getUploadChunkSize() + const zipUploadStream = new ZipUploadStream(bufferSize) + zip.pipe(zipUploadStream) + // register callbacks for various events during the zip lifecycle + zip.on('error', zipErrorCallback) + zip.on('warning', zipWarningCallback) + zip.on('finish', zipFinishCallback) + zip.on('end', zipEndCallback) + const addFileToZip = ( + file: UploadZipSpecification, + callback: (error?: Error) => void + ) => { + if (file.sourcePath !== null) { + zip.entry( + createReadStream(file.sourcePath), + {name: file.destinationPath}, + (error: any) => { + if (error) { + callback(error) + return + } + callback() + } + ) + } else { + zip.entry('', {name: file.destinationPath}, (error: any) => { + if (error) { + callback(error) + return + } + callback() + }) + } } - const createArtifactResp = - await artifactClient.CreateArtifact(createArtifactReq) - if (!createArtifactResp.ok) { - throw new InvalidResponseError( - 'CreateArtifact: response from backend was not ok' - ) - } - // Create the zipupload stream for use in blob upload - const zipUploadStream = await createZipUploadStream( - zipSpecification, - options?.compressionLevel - ).catch(err => { - throw new InvalidResponseError( - `createZipUploadStream: response from backend was not ok: ${err}` - ) + async.eachSeries(uploadSpecification, addFileToZip, (error: any) => { + if (error) { + core.error('Failed to add a file to the zip:') + core.info(error) + return + } + zip.finalize() // Finalize the archive once all files have been added }) - // Upload zip to blob storage - const uploadResult = await uploadZipToBlobStorage( - createArtifactResp.signedUploadUrl, - zipUploadStream - ).catch(err => { - throw new InvalidResponseError( - `uploadZipToBlobStorage: response blob was not ok: ${err}` + core.debug( + `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` + ) + core.debug( + `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` + ) + + return zipUploadStream +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const zipErrorCallback = (error: any): void => { + core.error('An error has occurred while creating the zip file for upload') + core.info(error) + + throw new Error('An error has occurred during zip creation for the artifact') +} +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const zipWarningCallback = (error: any): void => { + if (error.code === 'ENOENT') { + core.warning( + 'ENOENT warning during artifact zip creation. No such file or directory' ) - }) - // finalize the artifact - const finalizeArtifactReq: FinalizeArtifactRequest = { - workflowRunBackendId: backendIds.workflowRunBackendId, - workflowJobRunBackendId: backendIds.workflowJobRunBackendId, - name, - size: uploadResult.uploadSize ? uploadResult.uploadSize.toString() : '0' - } - if (uploadResult.sha256Hash) { - finalizeArtifactReq.hash = StringValue.create({ - value: `sha256:${uploadResult.sha256Hash}` - }) - } - core.info(`Finalizing artifact upload`) - const finalizeArtifactResp = - await artifactClient.FinalizeArtifact(finalizeArtifactReq) - if (!finalizeArtifactResp.ok) { - throw new InvalidResponseError( - 'FinalizeArtifact: response from backend was not ok' + core.info(error) + } else { + core.warning( + `A non-blocking warning has occurred during artifact zip creation: ${error.code}` ) + core.info(error) } +} - const artifactId = BigInt(finalizeArtifactResp.artifactId) - core.info( - `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}` - ) +const zipFinishCallback = (): void => { + core.debug('Zip stream for upload has finished.') +} - return { - size: uploadResult.uploadSize, - id: Number(artifactId) - } +const zipEndCallback = (): void => { + core.debug('Zip stream for upload has ended.') } diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 579757c3d7..3f71769499 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -22,9 +22,7 @@ export class ZipUploadStream extends stream.Transform { cb(null, chunk) } } -interface NodeJSError extends Error { - code?: string -} + export async function createZipUploadStream( uploadSpecification: UploadZipSpecification[], compressionLevel: number = DEFAULT_COMPRESSION_LEVEL @@ -32,6 +30,7 @@ export async function createZipUploadStream( core.debug( `Creating Artifact archive with compressionLevel: ${compressionLevel}` ) + const zlibOptions = { zlib: { level: compressionLevel, @@ -39,14 +38,16 @@ export async function createZipUploadStream( } } const zip = new ZipStream.default(zlibOptions) + const bufferSize = getUploadChunkSize() const zipUploadStream = new ZipUploadStream(bufferSize) - + zip.pipe(zipUploadStream) // register callbacks for various events during the zip lifecycle zip.on('error', zipErrorCallback) zip.on('warning', zipWarningCallback) zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) + const addFileToZip = ( file: UploadZipSpecification, callback: (error?: Error) => void @@ -55,18 +56,18 @@ export async function createZipUploadStream( zip.entry( createReadStream(file.sourcePath), {name: file.destinationPath}, - (error: NodeJSError) => { + (error: unknown) => { if (error) { - callback(error) + callback(error as Error) // Cast the error object to the Error type return } callback() } ) } else { - zip.entry('', {name: file.destinationPath}, (error: NodeJSError) => { + zip.entry('', {name: file.destinationPath}, (error: unknown) => { if (error) { - callback(error) + callback(error as Error) return } callback() @@ -74,10 +75,10 @@ export async function createZipUploadStream( } } - async.eachSeries(uploadSpecification, addFileToZip, (error: NodeJSError) => { + async.eachSeries(uploadSpecification, addFileToZip, (error: unknown) => { if (error) { core.error('Failed to add a file to the zip:') - core.info(error.toString()) + core.info(error.toString()) // Convert error to string return } zip.finalize() // Finalize the archive once all files have been added @@ -93,23 +94,25 @@ export async function createZipUploadStream( return zipUploadStream } -const zipErrorCallback = (error: Error): void => { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const zipErrorCallback = (error: any): void => { core.error('An error has occurred while creating the zip file for upload') - core.info(error.message) + core.info(error) throw new Error('An error has occurred during zip creation for the artifact') } -const zipWarningCallback = (error: NodeJSError): void => { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const zipWarningCallback = (error: any): void => { if (error.code === 'ENOENT') { core.warning( 'ENOENT warning during artifact zip creation. No such file or directory' ) - core.info(error.toString()) // Convert error object to string + core.info(error) } else { core.warning( `A non-blocking warning has occurred during artifact zip creation: ${error.code}` ) - core.info(error.toString()) // Convert error object to string + core.info(error) } } From 67c383759c81ccff5a529046352d022e8ae675c2 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 8 Apr 2024 15:43:51 +0000 Subject: [PATCH 69/72] updating errors --- .../src/internal/upload/upload-artifact.ts | 205 +++++++++--------- 1 file changed, 100 insertions(+), 105 deletions(-) diff --git a/packages/artifact/src/internal/upload/upload-artifact.ts b/packages/artifact/src/internal/upload/upload-artifact.ts index 6e174d8ee0..e810ccc60c 100644 --- a/packages/artifact/src/internal/upload/upload-artifact.ts +++ b/packages/artifact/src/internal/upload/upload-artifact.ts @@ -1,124 +1,119 @@ -import * as stream from 'stream' -import * as ZipStream from 'zip-stream' import * as core from '@actions/core' -import async from 'async' -import {createReadStream} from 'fs' -import {UploadZipSpecification} from './upload-zip-specification' -import {getUploadChunkSize} from '../shared/config' +import { + UploadArtifactOptions, + UploadArtifactResponse +} from '../shared/interfaces' +import {getExpiration} from './retention' +import {validateArtifactName} from './path-and-artifact-name-validation' +import {internalArtifactTwirpClient} from '../shared/artifact-twirp-client' +import { + UploadZipSpecification, + getUploadZipSpecification, + validateRootDirectory +} from './upload-zip-specification' +import {getBackendIdsFromToken} from '../shared/util' +import {uploadZipToBlobStorage} from './blob-upload' +import {createZipUploadStream} from './zip' +import { + CreateArtifactRequest, + FinalizeArtifactRequest, + StringValue +} from '../../generated' +import {FilesNotFoundError, InvalidResponseError} from '../shared/errors' -export const DEFAULT_COMPRESSION_LEVEL = 6 +export async function uploadArtifact( + name: string, + files: string[], + rootDirectory: string, + options?: UploadArtifactOptions | undefined +): Promise { + validateArtifactName(name) + validateRootDirectory(rootDirectory) -// Custom stream transformer so we can set the highWaterMark property -// See https://github.com/nodejs/node/issues/8855 -export class ZipUploadStream extends stream.Transform { - constructor(bufferSize: number) { - super({ - highWaterMark: bufferSize - }) + const zipSpecification: UploadZipSpecification[] = getUploadZipSpecification( + files, + rootDirectory + ) + if (zipSpecification.length === 0) { + throw new FilesNotFoundError( + zipSpecification.flatMap(s => (s.sourcePath ? [s.sourcePath] : [])) + ) } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - _transform(chunk: any, enc: any, cb: any): void { - cb(null, chunk) - } -} + // get the IDs needed for the artifact creation + const backendIds = getBackendIdsFromToken() -export async function createZipUploadStream( - uploadSpecification: UploadZipSpecification[], - compressionLevel: number = DEFAULT_COMPRESSION_LEVEL -): Promise { - core.debug( - `Creating Artifact archive with compressionLevel: ${compressionLevel}` - ) + // create the artifact client + const artifactClient = internalArtifactTwirpClient() - const zlibOptions = { - zlib: { - level: compressionLevel, - bufferSize: getUploadChunkSize() - } + // create the artifact + const createArtifactReq: CreateArtifactRequest = { + workflowRunBackendId: backendIds.workflowRunBackendId, + workflowJobRunBackendId: backendIds.workflowJobRunBackendId, + name, + version: 4 } - const zip = new ZipStream.default(zlibOptions) - const bufferSize = getUploadChunkSize() - const zipUploadStream = new ZipUploadStream(bufferSize) - zip.pipe(zipUploadStream) - // register callbacks for various events during the zip lifecycle - zip.on('error', zipErrorCallback) - zip.on('warning', zipWarningCallback) - zip.on('finish', zipFinishCallback) - zip.on('end', zipEndCallback) - const addFileToZip = ( - file: UploadZipSpecification, - callback: (error?: Error) => void - ) => { - if (file.sourcePath !== null) { - zip.entry( - createReadStream(file.sourcePath), - {name: file.destinationPath}, - (error: any) => { - if (error) { - callback(error) - return - } - callback() - } - ) - } else { - zip.entry('', {name: file.destinationPath}, (error: any) => { - if (error) { - callback(error) - return - } - callback() - }) - } + // if there is a retention period, add it to the request + const expiresAt = getExpiration(options?.retentionDays) + if (expiresAt) { + createArtifactReq.expiresAt = expiresAt } - async.eachSeries(uploadSpecification, addFileToZip, (error: any) => { - if (error) { - core.error('Failed to add a file to the zip:') - core.info(error) - return - } - zip.finalize() // Finalize the archive once all files have been added + const createArtifactResp = + await artifactClient.CreateArtifact(createArtifactReq) + if (!createArtifactResp.ok) { + throw new InvalidResponseError( + 'CreateArtifact: response from backend was not ok' + ) + } + // Create the zipupload stream for use in blob upload + const zipUploadStream = await createZipUploadStream( + zipSpecification, + options?.compressionLevel + ).catch(err => { + throw new InvalidResponseError( + `createZipUploadStream: response from backend was not ok: ${err}` + ) }) - core.debug( - `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` - ) - core.debug( - `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` - ) - - return zipUploadStream -} - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const zipErrorCallback = (error: any): void => { - core.error('An error has occurred while creating the zip file for upload') - core.info(error) - - throw new Error('An error has occurred during zip creation for the artifact') -} -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const zipWarningCallback = (error: any): void => { - if (error.code === 'ENOENT') { - core.warning( - 'ENOENT warning during artifact zip creation. No such file or directory' + // Upload zip to blob storage + const uploadResult = await uploadZipToBlobStorage( + createArtifactResp.signedUploadUrl, + zipUploadStream + ).catch(err => { + throw new InvalidResponseError( + `uploadZipToBlobStorage: response blob was not ok: ${err}` ) - core.info(error) - } else { - core.warning( - `A non-blocking warning has occurred during artifact zip creation: ${error.code}` + }) + // finalize the artifact + const finalizeArtifactReq: FinalizeArtifactRequest = { + workflowRunBackendId: backendIds.workflowRunBackendId, + workflowJobRunBackendId: backendIds.workflowJobRunBackendId, + name, + size: uploadResult.uploadSize ? uploadResult.uploadSize.toString() : '0' + } + if (uploadResult.sha256Hash) { + finalizeArtifactReq.hash = StringValue.create({ + value: `sha256:${uploadResult.sha256Hash}` + }) + } + core.info(`Finalizing artifact upload`) + const finalizeArtifactResp = + await artifactClient.FinalizeArtifact(finalizeArtifactReq) + if (!finalizeArtifactResp.ok) { + throw new InvalidResponseError( + 'FinalizeArtifact: response from backend was not ok' ) - core.info(error) } -} -const zipFinishCallback = (): void => { - core.debug('Zip stream for upload has finished.') -} + const artifactId = BigInt(finalizeArtifactResp.artifactId) + core.info( + `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}` + ) -const zipEndCallback = (): void => { - core.debug('Zip stream for upload has ended.') + return { + size: uploadResult.uploadSize, + id: Number(artifactId) + } } From 6e1d7543c4603a0803b99457aee11bbd8331d801 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 8 Apr 2024 17:21:24 +0000 Subject: [PATCH 70/72] updating tests --- .../__tests__/upload-artifact.test.ts | 112 +++++++++++++++++- packages/artifact/src/internal/upload/zip.ts | 35 +++--- 2 files changed, 128 insertions(+), 19 deletions(-) diff --git a/packages/artifact/__tests__/upload-artifact.test.ts b/packages/artifact/__tests__/upload-artifact.test.ts index b0dca5c8f6..b03b4aabc8 100644 --- a/packages/artifact/__tests__/upload-artifact.test.ts +++ b/packages/artifact/__tests__/upload-artifact.test.ts @@ -7,11 +7,21 @@ import {Timestamp, ArtifactServiceClientJSON} from '../src/generated' import * as blobUpload from '../src/internal/upload/blob-upload' import {uploadArtifact} from '../src/internal/upload/upload-artifact' import {noopLogs} from './common' -import {FilesNotFoundError} from '../src/internal/shared/errors' +import { + FilesNotFoundError, + InvalidResponseError +} from '../src/internal/shared/errors' +class NodeJSError extends Error { + code: string + constructor(message?: string, code?: string) { + super(message) // Pass the message to the Error constructor + this.code = code || '' + } +} describe('upload-artifact', () => { beforeEach(() => { - noopLogs() + // noopLogs() }) afterEach(() => { @@ -351,4 +361,102 @@ describe('upload-artifact', () => { expect(uploadResp).rejects.toThrow() }) + + describe('should respond with non-successful callback on different zipstream lifecycle methods', () => { + beforeEach(() => { + noopLogs() + }) + + afterEach(() => { + jest.restoreAllMocks() + }) + + it('should handle ENOENT error', async () => { + const mockDate = new Date('2020-01-01') + jest + .spyOn(uploadZipSpecification, 'validateRootDirectory') + .mockReturnValue() + jest + .spyOn(uploadZipSpecification, 'getUploadZipSpecification') + .mockReturnValue([ + { + sourcePath: '/home/user/files/plz-upload/file1.txt', + destinationPath: 'file1.txt' + }, + { + sourcePath: '/home/user/files/plz-upload/file2.txt', + destinationPath: 'file2.txt' + } + ]) + + const mockZipStream = { + entry: jest.fn((source, data, callback) => { + const err = (new NodeJSError( + "ENOENT: no such file or directory, open '/home/user/files/plz-upload/file1.txt'" + ).code = 'ENOENT') + callback(null, err) + }), + pipe: jest.fn(), + on: jest.fn(), + finalize: jest.fn() + } + + jest.mock('zip-stream', () => { + return { + default: jest.fn().mockImplementation(() => mockZipStream) + } + }) + + jest + .spyOn(zip, 'createZipUploadStream') + .mockReturnValue( + Promise.reject( + new NodeJSError( + "ENOENT: no such file or directory, open '/home/user/files/plz-upload/file1.txt'" + ) + ) + ) + jest.spyOn(util, 'getBackendIdsFromToken').mockReturnValue({ + workflowRunBackendId: '1234', + workflowJobRunBackendId: '5678' + }) + jest + .spyOn(retention, 'getExpiration') + .mockReturnValue(Timestamp.fromDate(mockDate)) + jest + .spyOn(ArtifactServiceClientJSON.prototype, 'CreateArtifact') + .mockReturnValue( + Promise.resolve({ + ok: true, + signedUploadUrl: 'https://signed-upload-url.com' + }) + ) + jest.spyOn(blobUpload, 'uploadZipToBlobStorage').mockReturnValue( + Promise.resolve({ + uploadSize: 1234, + sha256Hash: 'test-sha256-hash' + }) + ) + jest + .spyOn(ArtifactServiceClientJSON.prototype, 'FinalizeArtifact') + .mockReturnValue(Promise.resolve({ok: true, artifactId: '1'})) + + // ArtifactHttpClient mocks + jest.spyOn(config, 'getRuntimeToken').mockReturnValue('test-token') + jest + .spyOn(config, 'getResultsServiceUrl') + .mockReturnValue('https://test-url.com') + + const uploadResp = uploadArtifact( + 'test-artifact', + [ + '/home/user/files/plz-upload/file1.txt', + '/home/user/files/plz-upload/file2.txt', + '/home/user/files/plz-upload/dir/file3.txt' + ], + '/home/user/files/plz-upload' + ) + expect(uploadResp).rejects.toThrowError(InvalidResponseError) + }) + }) }) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index 3f71769499..bdae11b68e 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -43,8 +43,8 @@ export async function createZipUploadStream( const zipUploadStream = new ZipUploadStream(bufferSize) zip.pipe(zipUploadStream) // register callbacks for various events during the zip lifecycle - zip.on('error', zipErrorCallback) zip.on('warning', zipWarningCallback) + zip.on('error', zipErrorCallback) zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) @@ -75,23 +75,24 @@ export async function createZipUploadStream( } } - async.eachSeries(uploadSpecification, addFileToZip, (error: unknown) => { - if (error) { - core.error('Failed to add a file to the zip:') - core.info(error.toString()) // Convert error to string - return - } - zip.finalize() // Finalize the archive once all files have been added + return new Promise((resolve, reject) => { + async.eachSeries(uploadSpecification, addFileToZip, (error: unknown) => { + if (error) { + core.error('Failed to add a file to the zip:') + core.info(error.toString()) // Convert error to string + reject(error) + return + } + zip.finalize() // Finalize the archive once all files have been added + core.debug( + `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` + ) + core.debug( + `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` + ) + resolve(zipUploadStream) + }) }) - - core.debug( - `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` - ) - core.debug( - `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` - ) - - return zipUploadStream } // eslint-disable-next-line @typescript-eslint/no-explicit-any From da5e926162e14c8139a382d7122207982f9c9ac2 Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 8 Apr 2024 17:39:15 +0000 Subject: [PATCH 71/72] removing promise wrap --- packages/artifact/src/internal/upload/zip.ts | 36 +++++++++----------- packages/attest/package-lock.json | 2 +- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/packages/artifact/src/internal/upload/zip.ts b/packages/artifact/src/internal/upload/zip.ts index bdae11b68e..9008d5d950 100644 --- a/packages/artifact/src/internal/upload/zip.ts +++ b/packages/artifact/src/internal/upload/zip.ts @@ -43,11 +43,10 @@ export async function createZipUploadStream( const zipUploadStream = new ZipUploadStream(bufferSize) zip.pipe(zipUploadStream) // register callbacks for various events during the zip lifecycle - zip.on('warning', zipWarningCallback) zip.on('error', zipErrorCallback) + zip.on('warning', zipWarningCallback) zip.on('finish', zipFinishCallback) zip.on('end', zipEndCallback) - const addFileToZip = ( file: UploadZipSpecification, callback: (error?: Error) => void @@ -75,24 +74,23 @@ export async function createZipUploadStream( } } - return new Promise((resolve, reject) => { - async.eachSeries(uploadSpecification, addFileToZip, (error: unknown) => { - if (error) { - core.error('Failed to add a file to the zip:') - core.info(error.toString()) // Convert error to string - reject(error) - return - } - zip.finalize() // Finalize the archive once all files have been added - core.debug( - `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` - ) - core.debug( - `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` - ) - resolve(zipUploadStream) - }) + async.eachSeries(uploadSpecification, addFileToZip, (error: unknown) => { + if (error) { + core.error('Failed to add a file to the zip:') + core.info(error.toString()) // Convert error to string + return + } + zip.finalize() // Finalize the archive once all files have been added }) + + core.debug( + `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}` + ) + core.debug( + `Zip read high watermark value ${zipUploadStream.readableHighWaterMark}` + ) + + return zipUploadStream } // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/packages/attest/package-lock.json b/packages/attest/package-lock.json index 362633fb50..569cc7ed32 100644 --- a/packages/attest/package-lock.json +++ b/packages/attest/package-lock.json @@ -3561,4 +3561,4 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" } } -} \ No newline at end of file +} From bcaba45bb9b0453a5ce97226812126c142daff8a Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Mon, 8 Apr 2024 18:24:09 +0000 Subject: [PATCH 72/72] reverting http update --- packages/http-client/package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client/package-lock.json b/packages/http-client/package-lock.json index 52038ad373..ea3d79510b 100644 --- a/packages/http-client/package-lock.json +++ b/packages/http-client/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "@actions/http-client", - "version": "2.2.1", + "version": "2.2.0", "license": "MIT", "dependencies": { "tunnel": "^0.0.6",