[docker] Add script to generate commands to fix docker images (ray-pr…

…oject#34362)

We sometimes have to manually fix docker images, e.g. by updating a public key, or by updating a dependency.

This PR adds a script that generates the commands needed to fix docker images manually. I've added the command to the build-docker-images.py script, so we can use the original tagging function (see ray-project#33804).

Signed-off-by: Kai Fricke <[email protected]>

ci/build/build-docker-images.py

@@ -21,11 +21,15 @@
 PYTHON_WHL_VERSION = "cp3"
 ADDITIONAL_PLATFORMS = ["aarch64"]
 
+DOCKER_HUB_REPO = "rayproject"
+
 DOCKER_HUB_DESCRIPTION = {
  "base-deps": (
- "Internal Image, refer to " "https://hub.docker.com/r/rayproject/ray"
+ f"Internal Image, refer to https://hub.docker.com/r/{DOCKER_HUB_REPO}/ray"
+ ),
+ "ray-deps": (
+ f"Internal Image, refer to https://hub.docker.com/r/{DOCKER_HUB_REPO}/ray"
  ),
- "ray-deps": ("Internal Image, refer to " "https://hub.docker.com/r/rayproject/ray"),
  "ray": "Official Docker Images for Ray, the distributed computing API.",
  "ray-ml": "Developer ready Docker Image for Ray.",
  "ray-worker-container": "Internal Image for CI test",
@@ -233,7 +237,7 @@ def _build_docker_image(
  # can be found.
  build_args["FIND_LINKS_PATH"] = ".whl"
 
- tagged_name = f"rayproject/{image_name}:nightly-{py_version}-{device_tag}"
+ tagged_name = f"{DOCKER_HUB_REPO}/{image_name}:nightly-{py_version}-{device_tag}"
 
  tagged_name = _with_suffix(tagged_name, suffix=suffix)
 
@@ -355,7 +359,7 @@ def build_or_pull_base_images(
  suffix: Optional[str] = None,
 ) -> bool:
  """Returns images to tag and build."""
- repositories = ["rayproject/base-deps", "rayproject/ray-deps"]
+ repositories = [f"{DOCKER_HUB_REPO}/base-deps", f"{DOCKER_HUB_REPO}/ray-deps"]
  tags = [
  f"nightly-{py_version}-{image_type}"
  for py_version, image_type in itertools.product(py_versions, image_types)
@@ -672,7 +676,7 @@ def push_readmes(merge_build: bool):
  "PUSHRM_DEBUG": 1,
  "PUSHRM_SHORT": tag_line,
  }
- cmd_string = f"rayproject/{image}"
+ cmd_string = f"{DOCKER_HUB_REPO}/{image}"
 
  print(
  DOCKER_CLIENT.containers.run(
@@ -879,5 +883,72 @@ def main(
  # push_readmes(build_type is MERGE)
 
 
+def fix_docker_images(
+ image: str = "ray-ml",
+ version: str = "nightly",
+ repo: str = DOCKER_HUB_REPO,
+):
+ """Print commands to manually update docker images post-release.
+
+ This function prints commands that can be run to add new layers to
+ fix docker images post-release, e.g. when dependencies have to be fixed
+ or public keys expired.
+
+ The commands can be copied/pasted and executed in a shell.
+
+ Example:
+ FIX_IMAGE=ray-ml FIX_VERSION=2.3.0 python build-docker-images.py
+
+ """
+ tags = create_image_tags(
+ image_name=image,
+ py_versions=list(PY_MATRIX.keys()),
+ image_types=list(BASE_IMAGES.keys()),
+ specific_tag=None, # Set to `latest` for latest image fixes
+ version=version,
+ suffix=None,
+ )
+ print(dict(tags))
+
+ # Pull images we want to rebuild
+ for base_tag in tags:
+ base_image = f"{repo}/{image}:{base_tag}"
+
+ print(f"docker pull {base_image}")
+
+ # Re-tag these base images as e.g. pinned/ray-ml:tag
+ # This is so we can re-run the build command safely.
+ pinned_base_image = {}
+ for base_tag in tags:
+ base_image = f"{repo}/{image}:{base_tag}"
+ pinned_image = f"pinned/{image}:{base_tag}"
+
+ pinned_base_image[base_image] = pinned_image
+
+ print(f"docker tag {base_image} {pinned_image}")
+
+ # Create commands to build the new layer for the base images.
+ for base_tag in tags:
+ base_image = f"{repo}/{image}:{base_tag}"
+ pinned_image = pinned_base_image[base_image]
+
+ print(f"docker build --build-arg BASE_IMAGE={pinned_image} -t {base_image} .")
+ for subtag in tags[base_tag]:
+ if subtag == base_tag:
+ continue
+
+ # This will overwrite the rayproject/ray-ml:tag image
+ # - but we still have the pinned/ image if we want to re-run!
+ target_image = f"{repo}/{image}:{subtag}"
+ print(f"docker tag {base_image} {target_image}")
+
+ # Lastly, push new layers
+ print(f"docker push --all-tags {repo}/{image}")
+
+
 if __name__ == "__main__":
- main()
+ fix_image = os.environ.get("FIX_IMAGE")
+ if not fix_image:
+ main()
+ else:
+ fix_docker_images(fix_image, os.environ.get("FIX_VERSION"))