BUG make sure rerenders work and set permissions correctly

.pre-commit-config.yaml

@@ -44,7 +44,7 @@ repos:
  name: isort (python)
 
  - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.3.5
+ rev: v0.4.1
  hooks:
  - id: ruff
  args: [ --fix ]

conda_forge_tick/os_utils.py

@@ -1,8 +1,13 @@
 import contextlib
 import copy
+import functools
+import logging
 import os
 import shutil
 import subprocess
+from concurrent.futures import ProcessPoolExecutor
+
+logger = logging.getLogger(__name__)
 
 
 # https://stackoverflow.com/questions/6194499/pushd-through-os-system
@@ -156,3 +161,85 @@ def chmod_plus_rwX(file_or_dir, recursive=False, skip_on_error=False):
  _chmod_plus_rw(os.path.join(root, f), skip_on_error=skip_on_error)
  else:
  _chmod_plus_rw(file_or_dir, skip_on_error=skip_on_error)
+
+
+def get_user_execute_permissions(path):
+ """Get the user execute permissions of directory `path` and all of its contents.
+
+ Parameters
+ ----------
+ path : str
+ The path to the directory.
+
+ Returns
+ -------
+ dict
+ A dictionary mapping file paths to True if the user has execute permission or False otherwise.
+ """
+ fnames = _all_fnames(path)
+ perms = {}
+ for fname in sorted(fnames):
+ if ".git" in fname.split(os.path.sep):
+ continue
+
+ perm = os.stat(fname).st_mode
+ has_user_exe = os.stat(fname).st_mode & 0o100
+ key = os.path.relpath(fname, path)
+ logger.debug(f"got permissions of {key} as {perm:#o}")
+ perms[key] = has_user_exe
+ return perms
+
+
+def reset_permissions_with_user_execute(path, perms):
+ """Set the userpermissions of a directory `path` and all of its contents.
+
+ Parameters
+ ----------
+ path : str
+ The path to the directory.
+ perms : dict
+ A dictionary mapping file paths to True if the user has execute permission or False otherwise.
+ """
+ fnames = sorted(_all_fnames(path))
+ for fname in fnames:
+ if ".git" in fname.split(os.path.sep):
+ continue
+
+ path_fname = os.path.join(path, fname)
+ if os.path.exists(path_fname):
+ key = os.path.relpath(fname, path)
+ has_exec = perms.get(key, False)
+
+ if os.path.isdir(path_fname) or has_exec:
+ new_perm = get_dir_default_permissions()
+ else:
+ new_perm = get_file_default_permissions()
+
+ logger.debug(
+ f"setting permissions of {key} to {new_perm:#o} from {os.stat(path_fname).st_mode:#o}"
+ )
+ os.chmod(path_fname, new_perm)
+
+
+def _current_umask():
+ tmp = os.umask(0o666)
+ os.umask(tmp)
+ return tmp
+
+
+@functools.lru_cache(maxsize=1)
+def get_umask():
+ """Get the current umask."""
+ # done in a separate process for safety
+ with ProcessPoolExecutor(max_workers=1) as pool:
+ return pool.submit(_current_umask).result()
+
+
+def get_dir_default_permissions():
+ """Get the default permissions for directories."""
+ return 0o777 ^ get_umask()
+
+
+def get_file_default_permissions():
+ """Get the default permissions for files."""
+ return 0o666 ^ get_umask()

conda_forge_tick/rerender_feedstock.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import os
 import shutil
@@ -7,7 +8,13 @@
 import time
 from threading import Thread
 
-from conda_forge_tick.os_utils import chmod_plus_rwX, pushd, sync_dirs
+from conda_forge_tick.os_utils import (
+ chmod_plus_rwX,
+ get_user_execute_permissions,
+ pushd,
+ reset_permissions_with_user_execute,
+ sync_dirs,
+)
 from conda_forge_tick.utils import run_container_task
 
 logger = logging.getLogger(__name__)
@@ -78,6 +85,13 @@ def rerender_feedstock_containerized(feedstock_dir, timeout=900):
  feedstock_dir, tmp_feedstock_dir, ignore_dot_git=True, update_git=False
  )
 
+ perms = get_user_execute_permissions(feedstock_dir)
+ with open(
+ os.path.join(tmpdir, f"permissions-{os.path.basename(feedstock_dir)}.json"),
+ "w",
+ ) as f:
+ json.dump(perms, f)
+
  chmod_plus_rwX(tmpdir, recursive=True)
 
  logger.debug(f"host feedstock dir {feedstock_dir}: {os.listdir(feedstock_dir)}")
@@ -99,6 +113,7 @@ def rerender_feedstock_containerized(feedstock_dir, timeout=900):
  ignore_dot_git=True,
  update_git=True,
  )
+ reset_permissions_with_user_execute(feedstock_dir, data["permissions"])
 
  # When tempfile removes tempdir, it tries to reset permissions on subdirs.
  # This causes a permission error since the subdirs were made by the user

conda_forge_tick/utils.py

@@ -187,6 +187,9 @@ def run_container_task(
  else:
  mnt_args = []
 
+ log_level_str = str(logging.getLevelName(logger.getEffectiveLevel())).lower()
+ logger.debug("computed effective logging level: %s", log_level_str)
+
  cmd = [
  *get_default_container_run_args(tmpfs_size_mb=tmpfs_size_mb),
  *mnt_args,
@@ -197,7 +200,7 @@ def run_container_task(
  name,
  *args,
  "--log-level",
- str(logging.getLevelName(logger.getEffectiveLevel())).lower(),
+ log_level_str,
  ]
  res = subprocess.run(
  cmd,

docker/run_bot_task.py

@@ -13,8 +13,12 @@
 """
 
 import copy
+import glob
+import json
 import logging
 import os
+import shutil
+import subprocess
 import sys
 import tempfile
 import traceback
@@ -156,11 +160,34 @@ def _provide_source_code():
  return dict()
 
 
-def _rerender_feedstock(*, timeout):
- import glob
- import subprocess
+def _execute_git_cmds_and_report(*, cmds, cwd, msg):
+ logger = logging.getLogger("conda_forge_tick.container")
 
- from conda_forge_tick.os_utils import chmod_plus_rwX, sync_dirs
+ try:
+ _output = ""
+ for cmd in cmds:
+ gitret = subprocess.run(
+ cmd,
+ cwd=cwd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ )
+ logger.debug("git command %r output: %s", cmd, gitret.stdout)
+ _output += gitret.stdout
+ gitret.check_returncode()
+ except Exception as e:
+ logger.error(f"{msg}\noutput: {_output}", exc_info=e)
+ raise e
+
+
+def _rerender_feedstock(*, timeout):
+ from conda_forge_tick.os_utils import (
+ chmod_plus_rwX,
+ get_user_execute_permissions,
+ reset_permissions_with_user_execute,
+ sync_dirs,
+ )
  from conda_forge_tick.rerender_feedstock import rerender_feedstock_local
 
  logger = logging.getLogger("conda_forge_tick.container")
@@ -172,51 +199,71 @@ def _rerender_feedstock(*, timeout):
  logger.debug(
  f"input container feedstock dir {input_fs_dir}: {os.listdir(input_fs_dir)}"
  )
+ input_permissions = os.path.join(
+ "/cf_tick_dir", f"permissions-{os.path.basename(input_fs_dir)}.json"
+ )
+ with open(input_permissions) as f:
+ input_permissions = json.load(f)
 
  fs_dir = os.path.join(tmpdir, os.path.basename(input_fs_dir))
  sync_dirs(input_fs_dir, fs_dir, ignore_dot_git=True, update_git=False)
- if os.path.exists(os.path.join(fs_dir, ".gitignore")):
- os.remove(os.path.join(fs_dir, ".gitignore"))
  logger.debug(f"copied container feedstock dir {fs_dir}: {os.listdir(fs_dir)}")
 
- try:
- _output = ""
- cmds = [
- ["git", "init", "-b", "main", "."],
- ["git", "add", "."],
- ["git", "commit", "-am", "initial commit"],
- ]
- for cmd in cmds:
- gitret = subprocess.run(
- cmd,
- cwd=fs_dir,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- text=True,
- )
- _output += gitret.stdout
- gitret.check_returncode()
- except Exception as e:
- logger.error(
- f"git repo init failed for rerender\noutput: {_output}", exc_info=e
+ reset_permissions_with_user_execute(fs_dir, input_permissions)
+
+ has_gitignore = os.path.exists(os.path.join(fs_dir, ".gitignore"))
+ if has_gitignore:
+ shutil.move(
+ os.path.join(fs_dir, ".gitignore"),
+ os.path.join(fs_dir, ".gitignore.bak"),
  )
- raise e
+
+ cmds = [
+ ["git", "init", "-b", "main", "."],
+ ["git", "add", "."],
+ ["git", "commit", "-am", "initial commit"],
+ ]
+ if has_gitignore:
+ cmds += [
+ ["git", "mv", ".gitignore.bak", ".gitignore"],
+ ["git", "commit", "-am", "put back gitignore"],
+ ]
+ _execute_git_cmds_and_report(
+ cmds=cmds,
+ cwd=fs_dir,
+ msg="git init failed for rerender",
+ )
 
  if timeout is not None:
  kwargs = {"timeout": timeout}
  else:
  kwargs = {}
  msg = rerender_feedstock_local(fs_dir, **kwargs)
 
- chmod_plus_rwX(fs_dir, recursive=True, skip_on_error=False)
+ if logger.getEffectiveLevel() <= logging.DEBUG:
+ cmds = [
+ ["git", "status"],
+ ["git", "diff", "--name-only"],
+ ["git", "diff", "--name-only", "--staged"],
+ ["git", "--no-pager", "diff"],
+ ["git", "--no-pager", "diff", "--staged"],
+ ]
+ _execute_git_cmds_and_report(
+ cmds=cmds,
+ cwd=fs_dir,
+ msg="git status failed for rerender",
+ )
 
  # if something changed, copy back the new feedstock
  if msg is not None:
+ output_permissions = get_user_execute_permissions(fs_dir)
  sync_dirs(fs_dir, input_fs_dir, ignore_dot_git=True, update_git=False)
+ else:
+ output_permissions = input_permissions
 
  chmod_plus_rwX(input_fs_dir, recursive=True, skip_on_error=True)
 
- return {"commit_message": msg}
+ return {"commit_message": msg, "permissions": output_permissions}
 
 
 def _get_latest_version(*, attrs, sources):