bpo-34977: Use venv redirector instead of original python.exe on Wind…

…ows (pythonGH-11029)

Doc/library/venv.rst

@@ -130,7 +130,6 @@ creation according to their needs, the :class:`EnvBuilder` class.
  .. versionadded:: 3.6
  Added the ``prompt`` parameter
 
-
  Creators of third-party virtual environment tools will be free to use the
  provided ``EnvBuilder`` class as a base class.
 
@@ -177,23 +176,27 @@ creation according to their needs, the :class:`EnvBuilder` class.
 
  .. method:: setup_python(context)
 
- Creates a copy of the Python executable (and, under Windows, DLLs) in
- the environment. On a POSIX system, if a specific executable
- ``python3.x`` was used, symlinks to ``python`` and ``python3`` will be
- created pointing to that executable, unless files with those names
- already exist.
+ Creates a copy of the Python executable in the environment on POSIX
+ systems. If a specific executable ``python3.x`` was used, symlinks to
+ ``python`` and ``python3`` will be created pointing to that executable,
+ unless files with those names already exist.
 
  .. method:: setup_scripts(context)
 
  Installs activation scripts appropriate to the platform into the virtual
- environment.
+ environment. On Windows, also installs the ``python[w].exe`` scripts.
 
  .. method:: post_setup(context)
 
  A placeholder method which can be overridden in third party
  implementations to pre-install packages in the virtual environment or
  perform other post-creation steps.
 
+ .. versionchanged:: 3.7.2
+ Windows now uses redirector scripts for ``python[w].exe`` instead of
+ copying the actual binaries, and so :meth:`setup_python` does nothing
+ unless running from a build in the source tree.
+
  In addition, :class:`EnvBuilder` provides this utility method that can be
  called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to
  assist in installing custom scripts into the virtual environment.

Doc/whatsnew/3.7.rst

@@ -2512,3 +2512,13 @@ In 3.7.1 the :mod:`tokenize` module now implicitly emits a ``NEWLINE`` token
 when provided with input that does not have a trailing new line. This behavior
 now matches what the C tokenizer does internally.
 (Contributed by Ammar Askar in :issue:`33899`.)
+
+Notable changes in Python 3.7.2
+===============================
+
+In 3.7.2, :mod:`venv` on Windows no longer copies the original binaries, but
+creates redirector scripts named ``python.exe`` and ``pythonw.exe`` instead.
+This resolves a long standing issue where all virtual environments would have
+to be upgraded or recreated with each Python update. However, note that this
+release will still require recreation of virtual environments in order to get
+the new scripts.

Lib/test/test_venv.py

@@ -243,6 +243,7 @@ def test_isolation(self):
  self.assertIn('include-system-site-packages = %s\n' % s, data)
 
  @unittest.skipUnless(can_symlink(), 'Needs symlinks')
+ @unittest.skipIf(os.name == 'nt', 'Symlinks are never used on Windows')
  def test_symlinking(self):
  """
  Test symlinking works as expected

Lib/venv/__init__.py

@@ -64,10 +64,11 @@ def create(self, env_dir):
  self.system_site_packages = False
  self.create_configuration(context)
  self.setup_python(context)
+ if not self.upgrade:
+ self.setup_scripts(context)
  if self.with_pip:
  self._setup_pip(context)
  if not self.upgrade:
- self.setup_scripts(context)
  self.post_setup(context)
  if true_system_site_packages:
  # We had set it to False before, now
@@ -158,14 +159,6 @@ def create_configuration(self, context):
  f.write('include-system-site-packages = %s\n' % incl)
  f.write('version = %d.%d.%d\n' % sys.version_info[:3])
 
- if os.name == 'nt':
- def include_binary(self, f):
- if f.endswith(('.pyd', '.dll')):
- result = True
- else:
- result = f.startswith('python') and f.endswith('.exe')
- return result
-
  def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
  """
  Try symlinking a file, and if that fails, fall back to copying.
@@ -195,9 +188,9 @@ def setup_python(self, context):
  binpath = context.bin_path
  path = context.env_exe
  copier = self.symlink_or_copy
- copier(context.executable, path)
  dirname = context.python_dir
  if os.name != 'nt':
+ copier(context.executable, path)
  if not os.path.islink(path):
  os.chmod(path, 0o755)
  for suffix in ('python', 'python3'):
@@ -209,26 +202,22 @@ def setup_python(self, context):
  if not os.path.islink(path):
  os.chmod(path, 0o755)
  else:
- # See bpo-34011. When using a proper install, we should only need to
- # copy the top-level of DLLs.
- include = self.include_binary
- files = [f for f in os.listdir(dirname) if include(f)]
- for f in files:
- src = os.path.join(dirname, f)
- dst = os.path.join(binpath, f)
- if dst != context.env_exe: # already done, above
- copier(src, dst)
-
- # When creating from a build directory, we continue to copy all files.
+ # For normal cases, the venvlauncher will be copied from
+ # our scripts folder. For builds, we need to copy it
+ # manually.
  if sysconfig.is_python_build(True):
- subdir = 'DLLs'
- dirname = os.path.join(dirname, subdir)
- if os.path.isdir(dirname):
- files = [f for f in os.listdir(dirname) if include(f)]
- for f in files:
- src = os.path.join(dirname, f)
- dst = os.path.join(binpath, f)
- copier(src, dst)
+ suffix = '.exe'
+ if context.python_exe.lower().endswith('_d.exe'):
+ suffix = '_d.exe'
+
+ src = os.path.join(dirname, "venvlauncher" + suffix)
+ dst = os.path.join(binpath, context.python_exe)
+ copier(src, dst)
+
+ src = os.path.join(dirname, "venvwlauncher" + suffix)
+ dst = os.path.join(binpath, "pythonw" + suffix)
+ copier(src, dst)
+
  # copy init.tcl over
  for root, dirs, files in os.walk(context.python_dir):
  if 'init.tcl' in files:
@@ -326,7 +315,7 @@ def install_scripts(self, context, path):
  dstfile = os.path.join(dstdir, f)
  with open(srcfile, 'rb') as f:
  data = f.read()
- if not srcfile.endswith('.exe'):
+ if not srcfile.endswith(('.exe', '.pdb')):
  try:
  data = data.decode('utf-8')
  data = self.replace_variables(data, context)

Misc/NEWS.d/next/Windows/2018-12-07-10-00-38.bpo-34977.agQJbD.rst

@@ -0,0 +1,2 @@
+venv on Windows will now use a python.exe redirector rather than copying the
+actual binaries from the base environment.

PC/getpathp.c

@@ -536,10 +536,16 @@ static _PyInitError
 get_program_full_path(const _PyCoreConfig *core_config,
  PyCalculatePath *calculate, _PyPathConfig *config)
 {
+ const wchar_t *pyvenv_launcher;
  wchar_t program_full_path[MAXPATHLEN+1];
  memset(program_full_path, 0, sizeof(program_full_path));
 
- if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) {
+ /* The launcher may need to force the executable path to a
+ * different environment, so override it here. */
+ pyvenv_launcher = _wgetenv(L"__PYVENV_LAUNCHER__");
+ if (pyvenv_launcher && pyvenv_launcher[0]) {
+ wcscpy_s(program_full_path, MAXPATHLEN+1, pyvenv_launcher);
+ } else if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) {
  /* GetModuleFileName should never fail when passed NULL */
  return _Py_INIT_ERR("Cannot determine program path");
  }