From e98f6a039c393e3dbbb5491444c69a8084011000 Mon Sep 17 00:00:00 2001 From: Cuong Tran Date: Tue, 13 Nov 2018 14:27:43 +0000 Subject: [PATCH 001/164] added capability to take JSON as log's config --- docs/source/settings.rst | 21 ++++++++++++++++----- gunicorn/config.py | 28 ++++++++++++++++++++++------ gunicorn/glogging.py | 18 +++++++++++++++++- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index b9bd56de3..00f93cabe 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -264,6 +264,16 @@ The log config file to use. Gunicorn uses the standard Python logging module's Configuration file format. +.. _logconfig-json: + +logiconfig_json +~~~~~~~~~ + +* ``--log-config-json FILE`` +* ``None`` + +The log config file written in JSON. + .. _logconfig-dict: logconfig_dict @@ -274,8 +284,9 @@ logconfig_dict The log config dictionary to use, using the standard Python logging module's dictionary configuration format. This option -takes precedence over the :ref:`logconfig` option, which uses the -older file configuration format. +takes precedence over the :ref:`logconfig` and :ref:`logConfigJson` options, which uses the +older file configuration format and JSON respectively. + Format: https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig @@ -1173,11 +1184,11 @@ libraries may be installed using setuptools' ``extra_require`` feature. A string referring to one of the following bundled classes: * ``sync`` -* ``eventlet`` - Requires eventlet >= 0.9.7 (or install it via +* ``eventlet`` - Requires eventlet >= 0.9.7 (or install it via ``pip install gunicorn[eventlet]``) -* ``gevent`` - Requires gevent >= 0.13 (or install it via +* ``gevent`` - Requires gevent >= 0.13 (or install it via ``pip install gunicorn[gevent]``) -* ``tornado`` - Requires tornado >= 0.2 (or install it via +* ``tornado`` - Requires tornado >= 0.2 (or install it via ``pip install gunicorn[tornado]``) * ``gthread`` - Python 2 requires the futures package to be installed (or install it via ``pip install gunicorn[gthread]``) diff --git a/gunicorn/config.py b/gunicorn/config.py index e14161b6f..152720097 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -627,11 +627,11 @@ class WorkerClass(Setting): A string referring to one of the following bundled classes: * ``sync`` - * ``eventlet`` - Requires eventlet >= 0.9.7 (or install it via + * ``eventlet`` - Requires eventlet >= 0.9.7 (or install it via ``pip install gunicorn[eventlet]``) - * ``gevent`` - Requires gevent >= 0.13 (or install it via + * ``gevent`` - Requires gevent >= 0.13 (or install it via ``pip install gunicorn[gevent]``) - * ``tornado`` - Requires tornado >= 0.2 (or install it via + * ``tornado`` - Requires tornado >= 0.2 (or install it via ``pip install gunicorn[tornado]``) * ``gthread`` - Python 2 requires the futures package to be installed (or install it via ``pip install gunicorn[gthread]``) @@ -668,7 +668,7 @@ class WorkerThreads(Setting): If it is not defined, the default is ``1``. This setting only affects the Gthread worker type. - + .. note:: If you try to use the ``sync`` worker type and set the ``threads`` setting to more than 1, the ``gthread`` worker type will be used @@ -1368,8 +1368,8 @@ class LogConfigDict(Setting): desc = """\ The log config dictionary to use, using the standard Python logging module's dictionary configuration format. This option - takes precedence over the :ref:`logconfig` option, which uses the - older file configuration format. + takes precedence over the :ref:`logconfig` and :ref:`logConfigJson` options, which uses the + older file configuration format and JSON respectively. Format: https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig @@ -1377,6 +1377,22 @@ class LogConfigDict(Setting): """ +class LogConfigJson(Setting): + name = "logconfig_json" + section = "Logging" + cli = ["--log-config-json"] + meta = "FILE" + validator = validate_string + default = None + desc = """\ + The log config JSON reads config from a JSON file + + Format: https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig + + .. versionadded:: 19.9 + """ + + class SyslogTo(Setting): name = "syslog_addr" section = "Logging" diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index 3f2669774..966abe3da 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -5,6 +5,7 @@ import base64 import binascii +import json import time import logging logging.Logger.manager.emittedNoHandlerWarning = 1 @@ -238,6 +239,21 @@ def setup(self, cfg): TypeError ) as exc: raise RuntimeError(str(exc)) + elif cfg.logconfig_json: + config = CONFIG_DEFAULTS.copy() + if os.path.exists(cfg.logconfig_json): + try: + config_json = json.load(open(cfg.logconfig_json)) + config.update(config_json) + dictConfig(config) + except( + json.JSONDecodeError, + AttributeError, + ImportError, + ValueError, + TypeError + ) as exc: + raise RuntimeError(str(exc)) elif cfg.logconfig: if os.path.exists(cfg.logconfig): defaults = CONFIG_DEFAULTS.copy() @@ -330,7 +346,7 @@ def access(self, resp, req, environ, request_time): """ if not (self.cfg.accesslog or self.cfg.logconfig or - self.cfg.logconfig_dict or + self.cfg.logconfig_dict or self.cfg.logconfig_json or (self.cfg.syslog and not self.cfg.disable_redirect_access_to_syslog)): return From d20a26a347915836fd18c69ddffd234eee7dd2e4 Mon Sep 17 00:00:00 2001 From: Cuong Tran Date: Tue, 20 Nov 2018 00:08:28 +0000 Subject: [PATCH 002/164] updated styling --- gunicorn/config.py | 19 ++++++++++--------- gunicorn/glogging.py | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 152720097..eac77a758 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -627,11 +627,11 @@ class WorkerClass(Setting): A string referring to one of the following bundled classes: * ``sync`` - * ``eventlet`` - Requires eventlet >= 0.9.7 (or install it via + * ``eventlet`` - Requires eventlet >= 0.9.7 (or install it via ``pip install gunicorn[eventlet]``) - * ``gevent`` - Requires gevent >= 0.13 (or install it via + * ``gevent`` - Requires gevent >= 0.13 (or install it via ``pip install gunicorn[gevent]``) - * ``tornado`` - Requires tornado >= 0.2 (or install it via + * ``tornado`` - Requires tornado >= 0.2 (or install it via ``pip install gunicorn[tornado]``) * ``gthread`` - Python 2 requires the futures package to be installed (or install it via ``pip install gunicorn[gthread]``) @@ -668,7 +668,7 @@ class WorkerThreads(Setting): If it is not defined, the default is ``1``. This setting only affects the Gthread worker type. - + .. note:: If you try to use the ``sync`` worker type and set the ``threads`` setting to more than 1, the ``gthread`` worker type will be used @@ -1368,8 +1368,9 @@ class LogConfigDict(Setting): desc = """\ The log config dictionary to use, using the standard Python logging module's dictionary configuration format. This option - takes precedence over the :ref:`logconfig` and :ref:`logConfigJson` options, which uses the - older file configuration format and JSON respectively. + takes precedence over the :ref:`logconfig` and :ref:`logConfigJson` options, + which uses the older file configuration format and JSON + respectively. Format: https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig @@ -1385,11 +1386,11 @@ class LogConfigJson(Setting): validator = validate_string default = None desc = """\ - The log config JSON reads config from a JSON file + The log config to read config from a JSON file - Format: https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig + Format: https://docs.python.org/3/library/logging.config.html#logging.config.jsonConfig - .. versionadded:: 19.9 + .. versionadded:: 20.0 """ diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index 966abe3da..e32fdf099 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -246,7 +246,7 @@ def setup(self, cfg): config_json = json.load(open(cfg.logconfig_json)) config.update(config_json) dictConfig(config) - except( + except ( json.JSONDecodeError, AttributeError, ImportError, From 54d35d7358934bf60e11dd661f7045002c546e5f Mon Sep 17 00:00:00 2001 From: l <> Date: Wed, 23 Jan 2019 22:09:08 +0800 Subject: [PATCH 003/164] Fix #1965: About gunicorn [CRITICAL] Worker Timeout --- gunicorn/arbiter.py | 2 +- gunicorn/workers/workertmp.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 7eaa2c177..f7f86f005 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -494,7 +494,7 @@ def murder_workers(self): workers = list(self.WORKERS.items()) for (pid, worker) in workers: try: - if time.time() - worker.tmp.last_update() <= self.timeout: + if time.monotonic() - worker.tmp.last_update() <= self.timeout: continue except (OSError, ValueError): continue diff --git a/gunicorn/workers/workertmp.py b/gunicorn/workers/workertmp.py index 22aaef34c..66defe946 100644 --- a/gunicorn/workers/workertmp.py +++ b/gunicorn/workers/workertmp.py @@ -4,6 +4,7 @@ # See the NOTICE for more information. import os +import time import platform import tempfile @@ -35,14 +36,12 @@ def __init__(self, cfg): os.close(fd) raise - self.spinner = 0 - def notify(self): - self.spinner = (self.spinner + 1) % 2 - os.fchmod(self._tmp.fileno(), self.spinner) + new_time = time.monotonic() + os.utime(self._tmp.name, (new_time, new_time)) def last_update(self): - return os.fstat(self._tmp.fileno()).st_ctime + return os.fstat(self._tmp.fileno()).st_mtime def fileno(self): return self._tmp.fileno() From 2a0433e7ed7185266f1430d774d15474562c0403 Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Sun, 17 Feb 2019 14:37:37 +0800 Subject: [PATCH 004/164] Update gunicorn/workers/workertmp.py Co-Authored-By: skytoup <875766917@qq.com> --- gunicorn/workers/workertmp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/workers/workertmp.py b/gunicorn/workers/workertmp.py index 66defe946..604c759c0 100644 --- a/gunicorn/workers/workertmp.py +++ b/gunicorn/workers/workertmp.py @@ -38,7 +38,7 @@ def __init__(self, cfg): def notify(self): new_time = time.monotonic() - os.utime(self._tmp.name, (new_time, new_time)) + os.utime(self._tmp.fileno(), (new_time, new_time)) def last_update(self): return os.fstat(self._tmp.fileno()).st_mtime From 5803f835f6fbf8c166af99cba1617cb0d3863210 Mon Sep 17 00:00:00 2001 From: Wojciech Malinowski Date: Thu, 6 Jun 2019 15:13:23 +0200 Subject: [PATCH 005/164] Added a possibility of logging the metrics to a Unix domain socket instead of UDP --- docs/source/settings.rst | 11 +++++++++++ gunicorn/config.py | 16 +++++++++++++++- gunicorn/instrument/statsd.py | 14 +++++++++++--- tests/test_config.py | 9 ++++++++- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 16d8961ae..e759ded79 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -363,6 +363,17 @@ statsd_host .. versionadded:: 19.1 +.. _statsd-socket: + +statsd_socket +~~~~~~~~~~~~~ + +* ``--statsd-socket STATSD_SOCKET`` +* ``None`` + +Unix domain socket of the statsd server to log to. +Supersedes ``statsd_host`` if provided. + .. _dogstatsd-tags: dogstatsd_tags diff --git a/gunicorn/config.py b/gunicorn/config.py index e8e0f926a..3f48b6dd2 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -144,7 +144,9 @@ def logger_class(self): # if default logger is in use, and statsd is on, automagically switch # to the statsd logger if uri == LoggerClass.default: - if 'statsd_host' in self.settings and self.settings['statsd_host'].value is not None: + statsd_address = 'statsd_socket' in self.settings and self.settings['statsd_socket'].value or \ + 'statsd_host' in self.settings and self.settings['statsd_host'].value + if statsd_address is not None: uri = "gunicorn.instrument.statsd.Statsd" logger_class = util.load_class( @@ -1478,6 +1480,18 @@ class StatsdHost(Setting): .. versionadded:: 19.1 """ +class StatsdSocket(Setting): + name = "statsd_socket" + section = "Logging" + cli = ["--statsd-socket"] + meta = "STATSD_SOCKET" + default = None + validator = validate_string + desc = """\ + Unix domain socket of the statsd server to log to. + Supersedes ``statsd_host`` if provided. + """ + # Datadog Statsd (dogstatsd) tags. https://docs.datadoghq.com/developers/dogstatsd/ class DogstatsdTags(Setting): name = "dogstatsd_tags" diff --git a/gunicorn/instrument/statsd.py b/gunicorn/instrument/statsd.py index 9a537205b..ee6d53086 100644 --- a/gunicorn/instrument/statsd.py +++ b/gunicorn/instrument/statsd.py @@ -27,10 +27,18 @@ def __init__(self, cfg): """ Logger.__init__(self, cfg) self.prefix = sub(r"^(.+[^.]+)\.*$", "\\g<1>.", cfg.statsd_prefix) - try: + + if cfg.statsd_socket: + address_family = socket.AF_UNIX + address = cfg.statsd_socket + elif cfg.statsd_host: + address_family = socket.AF_INET host, port = cfg.statsd_host - self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - self.sock.connect((host, int(port))) + address = (host, int(port)) + + try: + self.sock = socket.socket(address_family, socket.SOCK_DGRAM) + self.sock.connect(address) except Exception: self.sock = None diff --git a/tests/test_config.py b/tests/test_config.py index 0587c63cf..db108b9b0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -306,13 +306,20 @@ def nworkers_changed_3(server, new_value, old_value): assert c.nworkers_changed(1, 2, 3) == 3 -def test_statsd_changes_logger(): +def test_statsd_host_changes_logger(): c = config.Config() assert c.logger_class == glogging.Logger c.set('statsd_host', 'localhost:12345') assert c.logger_class == statsd.Statsd +def test_statsd_socket_changes_logger(): + c = config.Config() + assert c.logger_class == glogging.Logger + c.set('statsd_socket', '/var/run/sock') + assert c.logger_class == statsd.Statsd + + class MyLogger(glogging.Logger): # dummy custom logger class for testing pass From d1f6a7788c8416533d448cfe490957efabdb13a0 Mon Sep 17 00:00:00 2001 From: Wojciech Malinowski Date: Thu, 6 Jun 2019 15:22:14 +0200 Subject: [PATCH 006/164] Code style --- gunicorn/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 3f48b6dd2..95b615d40 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -144,8 +144,10 @@ def logger_class(self): # if default logger is in use, and statsd is on, automagically switch # to the statsd logger if uri == LoggerClass.default: - statsd_address = 'statsd_socket' in self.settings and self.settings['statsd_socket'].value or \ - 'statsd_host' in self.settings and self.settings['statsd_host'].value + statsd_address = 'statsd_socket' in self.settings and \ + self.settings['statsd_socket'].value or \ + 'statsd_host' in self.settings and \ + self.settings['statsd_host'].value if statsd_address is not None: uri = "gunicorn.instrument.statsd.Statsd" From 74cf2ce084a38031ea592903176e5374ecd3f38d Mon Sep 17 00:00:00 2001 From: REN Xiaolei Date: Thu, 14 Nov 2019 11:49:58 +0800 Subject: [PATCH 007/164] Update faq.rst and fix technical mistake on ulimit --- docs/source/faq.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/source/faq.rst b/docs/source/faq.rst index bd8f1d187..6531b99bf 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -128,10 +128,9 @@ How can I increase the maximum number of file descriptors? One of the first settings that usually needs to be bumped is the maximum number of open file descriptors for a given process. For the confused out there, remember that Unices treat sockets as files. - -:: - - $ sudo ulimit -n 2048 +Considering non-privileged users are not able to relax the limit, you should +firstly switch to root user, increase the limit, then run gunicorn. Using ``sudo +ulimit`` would not take effect. How can I increase the maximum socket backlog? ---------------------------------------------- From 9545e01d17e62fb1e959b6b18c7abe74ce99b9f1 Mon Sep 17 00:00:00 2001 From: REN Xiaolei Date: Mon, 18 Nov 2019 20:53:32 +0800 Subject: [PATCH 008/164] Update faq.rst --- docs/source/faq.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 6531b99bf..b728084c2 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -128,10 +128,14 @@ How can I increase the maximum number of file descriptors? One of the first settings that usually needs to be bumped is the maximum number of open file descriptors for a given process. For the confused out there, remember that Unices treat sockets as files. + +.. warning:: Considering non-privileged users are not able to relax the limit, you should firstly switch to root user, increase the limit, then run gunicorn. Using ``sudo ulimit`` would not take effect. +Try systemd's service unit file, or an initscript which runs as root. + How can I increase the maximum socket backlog? ---------------------------------------------- From 5858f81566f812d5f12c107969d70162dc50eae7 Mon Sep 17 00:00:00 2001 From: REN Xiaolei Date: Mon, 18 Nov 2019 20:58:39 +0800 Subject: [PATCH 009/164] Update faq.rst --- docs/source/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/faq.rst b/docs/source/faq.rst index b728084c2..55303122a 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -129,7 +129,7 @@ One of the first settings that usually needs to be bumped is the maximum number of open file descriptors for a given process. For the confused out there, remember that Unices treat sockets as files. -.. warning:: +.. warning:: ``sudo ulimit`` may not work Considering non-privileged users are not able to relax the limit, you should firstly switch to root user, increase the limit, then run gunicorn. Using ``sudo ulimit`` would not take effect. From c5a254ad9d3b46950a86d6b868cd6249c827b354 Mon Sep 17 00:00:00 2001 From: REN Xiaolei Date: Tue, 19 Nov 2019 16:32:43 +0800 Subject: [PATCH 010/164] Update faq.rst --- docs/source/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 55303122a..e98e4eb6a 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -130,7 +130,7 @@ of open file descriptors for a given process. For the confused out there, remember that Unices treat sockets as files. .. warning:: ``sudo ulimit`` may not work -Considering non-privileged users are not able to relax the limit, you should + Considering non-privileged users are not able to relax the limit, you should firstly switch to root user, increase the limit, then run gunicorn. Using ``sudo ulimit`` would not take effect. From 00b58979385c98a60c516b1b1963d2a91a7238d8 Mon Sep 17 00:00:00 2001 From: REN Xiaolei Date: Tue, 19 Nov 2019 16:34:40 +0800 Subject: [PATCH 011/164] Update faq.rst --- docs/source/faq.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/faq.rst b/docs/source/faq.rst index e98e4eb6a..a982f808e 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -130,7 +130,8 @@ of open file descriptors for a given process. For the confused out there, remember that Unices treat sockets as files. .. warning:: ``sudo ulimit`` may not work - Considering non-privileged users are not able to relax the limit, you should + +Considering non-privileged users are not able to relax the limit, you should firstly switch to root user, increase the limit, then run gunicorn. Using ``sudo ulimit`` would not take effect. From ee1e5c1928c71fa366dfac99c65f630e03aed8b7 Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Mon, 20 Apr 2020 19:31:53 -0700 Subject: [PATCH 012/164] Log abnormal worker exit codes --- gunicorn/arbiter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 532426d21..1bab2c612 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -520,6 +520,8 @@ def reap_workers(self): # that it could not boot, we'll shut it down to avoid # infinite start/stop cycles. exitcode = status >> 8 + if exitcode != 0: + self.log.error('Worker (pid:%s) exited with code %s', wpid, exitcode) if exitcode == self.WORKER_BOOT_ERROR: reason = "Worker failed to boot." raise HaltServer(reason, self.WORKER_BOOT_ERROR) From 79fc9a3b00e43dd152894d38ba81ae0776461127 Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Mon, 20 Apr 2020 19:32:12 -0700 Subject: [PATCH 013/164] Log abnormal arbiter shutdown at error level --- gunicorn/arbiter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 1bab2c612..f827572d9 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -340,9 +340,12 @@ def wakeup(self): def halt(self, reason=None, exit_status=0): """ halt arbiter """ self.stop() - self.log.info("Shutting down: %s", self.master_name) + + log_func = self.log.info if exit_status == 0 else self.log.error + log_func("Shutting down: %s", self.master_name) if reason is not None: - self.log.info("Reason: %s", reason) + log_func("Reason: %s", reason) + if self.pidfile is not None: self.pidfile.unlink() self.cfg.on_exit(self) From 15abac7e819174e6add9b3d979053f8666be297d Mon Sep 17 00:00:00 2001 From: larribas Date: Sun, 19 Jul 2020 19:45:50 +0200 Subject: [PATCH 014/164] Allow specifying a UDS socket address through --statsd-host --- docs/source/settings.rst | 16 +++++--------- gunicorn/config.py | 41 ++++++++++++++--------------------- gunicorn/instrument/statsd.py | 11 +++------- tests/test_config.py | 22 ++++++++++++------- tests/test_statsd.py | 12 ++++++++++ 5 files changed, 50 insertions(+), 52 deletions(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index e759ded79..a3cb12dd1 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -359,20 +359,14 @@ statsd_host * ``--statsd-host STATSD_ADDR`` * ``None`` -``host:port`` of the statsd server to log to. +The address of the StatsD server to log to. -.. versionadded:: 19.1 - -.. _statsd-socket: - -statsd_socket -~~~~~~~~~~~~~ +Address is a string of the form: -* ``--statsd-socket STATSD_SOCKET`` -* ``None`` +* ``unix://PATH`` : for a unix domain socket. +* ``HOST:PORT`` : for a network address -Unix domain socket of the statsd server to log to. -Supersedes ``statsd_host`` if provided. +.. versionadded:: 19.1 .. _dogstatsd-tags: diff --git a/gunicorn/config.py b/gunicorn/config.py index 95b615d40..99b4ba319 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -144,11 +144,7 @@ def logger_class(self): # if default logger is in use, and statsd is on, automagically switch # to the statsd logger if uri == LoggerClass.default: - statsd_address = 'statsd_socket' in self.settings and \ - self.settings['statsd_socket'].value or \ - 'statsd_host' in self.settings and \ - self.settings['statsd_host'].value - if statsd_address is not None: + if 'statsd_host' in self.settings and self.settings['statsd_host'].value is not None: uri = "gunicorn.instrument.statsd.Statsd" logger_class = util.load_class( @@ -499,15 +495,17 @@ def validate_chdir(val): return path -def validate_hostport(val): +def validate_address(val): val = validate_string(val) if val is None: return None - elements = val.split(":") - if len(elements) == 2: - return (elements[0], int(elements[1])) - else: - raise TypeError("Value must consist of: hostname:port") + + try: + address = util.parse_address(val, default_port='8125') + except RuntimeError: + raise TypeError("Value must be one of ('host:port', 'unix://PATH')") + + return address def validate_reload_engine(val): @@ -1475,23 +1473,16 @@ class StatsdHost(Setting): cli = ["--statsd-host"] meta = "STATSD_ADDR" default = None - validator = validate_hostport + validator = validate_address desc = """\ - ``host:port`` of the statsd server to log to. + The address of the StatsD server to log to. - .. versionadded:: 19.1 - """ + Address is a string of the form: -class StatsdSocket(Setting): - name = "statsd_socket" - section = "Logging" - cli = ["--statsd-socket"] - meta = "STATSD_SOCKET" - default = None - validator = validate_string - desc = """\ - Unix domain socket of the statsd server to log to. - Supersedes ``statsd_host`` if provided. + * ``unix://PATH`` : for a unix domain socket. + * ``HOST:PORT`` : for a network address + + .. versionadded:: 19.1 """ # Datadog Statsd (dogstatsd) tags. https://docs.datadoghq.com/developers/dogstatsd/ diff --git a/gunicorn/instrument/statsd.py b/gunicorn/instrument/statsd.py index ee6d53086..143d0bbc1 100644 --- a/gunicorn/instrument/statsd.py +++ b/gunicorn/instrument/statsd.py @@ -23,22 +23,17 @@ class Statsd(Logger): """statsD-based instrumentation, that passes as a logger """ def __init__(self, cfg): - """host, port: statsD server - """ Logger.__init__(self, cfg) self.prefix = sub(r"^(.+[^.]+)\.*$", "\\g<1>.", cfg.statsd_prefix) - if cfg.statsd_socket: + if isinstance(cfg.statsd_host, str): address_family = socket.AF_UNIX - address = cfg.statsd_socket - elif cfg.statsd_host: + else: address_family = socket.AF_INET - host, port = cfg.statsd_host - address = (host, int(port)) try: self.sock = socket.socket(address_family, socket.SOCK_DGRAM) - self.sock.connect(address) + self.sock.connect(cfg.statsd_host) except Exception: self.sock = None diff --git a/tests/test_config.py b/tests/test_config.py index db108b9b0..db33cb3f5 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -306,17 +306,23 @@ def nworkers_changed_3(server, new_value, old_value): assert c.nworkers_changed(1, 2, 3) == 3 -def test_statsd_host_changes_logger(): +def test_statsd_host(): c = config.Config() - assert c.logger_class == glogging.Logger - c.set('statsd_host', 'localhost:12345') - assert c.logger_class == statsd.Statsd - - -def test_statsd_socket_changes_logger(): + assert c.statsd_host is None + c.set("statsd_host", "localhost") + assert c.statsd_host == ("localhost", 8125) + c.set("statsd_host", "statsd:7777") + assert c.statsd_host == ("statsd", 7777) + c.set("statsd_host", "unix:/path/to.sock") + assert c.statsd_host == "/path/to.sock" + pytest.raises(TypeError, c.set, "statsd_host", 666) + pytest.raises(TypeError, c.set, "statsd_host", "host:string") + + +def test_statsd_changes_logger(): c = config.Config() assert c.logger_class == glogging.Logger - c.set('statsd_socket', '/var/run/sock') + c.set('statsd_host', 'localhost:12345') assert c.logger_class == statsd.Statsd diff --git a/tests/test_statsd.py b/tests/test_statsd.py index 06c1d9646..6f7bf426b 100644 --- a/tests/test_statsd.py +++ b/tests/test_statsd.py @@ -59,6 +59,18 @@ def test_statsd_fail(): logger.exception("No impact on logging") +def test_statsd_host_initialization(): + c = Config() + c.set('statsd_host', 'unix:test.sock') + logger = Statsd(c) + logger.info("Can be initialized and used with a UDS socket") + + # Can be initialized and used with a UDP address + c.set('statsd_host', 'host:8080') + logger = Statsd(c) + logger.info("Can be initialized and used with a UDP socket") + + def test_dogstatsd_tags(): c = Config() tags = 'yucatan,libertine:rhubarb' From 2a16fcd3ceed5a3fdd30a7b6fab0b065180576d6 Mon Sep 17 00:00:00 2001 From: larribas Date: Sun, 19 Jul 2020 20:40:08 +0200 Subject: [PATCH 015/164] Test and defend against the specific case where the statsd hostname is 'unix' --- gunicorn/config.py | 12 ++++++++++-- tests/test_config.py | 13 ++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 99b4ba319..5549a372a 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -495,11 +495,19 @@ def validate_chdir(val): return path -def validate_address(val): +def validate_statsd_address(val): val = validate_string(val) if val is None: return None + # As of major release 20, util.parse_address would recognize unix:PORT + # as a UDS address, breaking backwards compatibility. We defend against + # that regression here (this is also unit-tested). + # Feel free to remove in the next major release. + unix_hostname_regression = re.match(r'^unix:(\d+)$', val) + if unix_hostname_regression: + return ('unix', int(unix_hostname_regression.group(1))) + try: address = util.parse_address(val, default_port='8125') except RuntimeError: @@ -1473,7 +1481,7 @@ class StatsdHost(Setting): cli = ["--statsd-host"] meta = "STATSD_ADDR" default = None - validator = validate_address + validator = validate_statsd_address desc = """\ The address of the StatsD server to log to. diff --git a/tests/test_config.py b/tests/test_config.py index db33cb3f5..e3a3212bd 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -313,12 +313,23 @@ def test_statsd_host(): assert c.statsd_host == ("localhost", 8125) c.set("statsd_host", "statsd:7777") assert c.statsd_host == ("statsd", 7777) - c.set("statsd_host", "unix:/path/to.sock") + c.set("statsd_host", "unix:///path/to.sock") assert c.statsd_host == "/path/to.sock" pytest.raises(TypeError, c.set, "statsd_host", 666) pytest.raises(TypeError, c.set, "statsd_host", "host:string") +def test_statsd_host_with_unix_as_hostname(): + # This is a regression test for major release 20. After this release + # we should consider modifying the behavior of util.parse_address to + # simplify gunicorn's code + c = config.Config() + c.set("statsd_host", "unix:7777") + assert c.statsd_host == ("unix", 7777) + c.set("statsd_host", "unix://some.socket") + assert c.statsd_host == "some.socket" + + def test_statsd_changes_logger(): c = config.Config() assert c.logger_class == glogging.Logger From d81c6cefcb1b0a359c56675906b39ba641aaf016 Mon Sep 17 00:00:00 2001 From: Ken Okada Date: Thu, 20 Aug 2020 09:00:03 +0900 Subject: [PATCH 016/164] Warn in the case of bad systemd configuration --- gunicorn/app/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gunicorn/app/base.py b/gunicorn/app/base.py index df8c666f2..c74ea3e8a 100644 --- a/gunicorn/app/base.py +++ b/gunicorn/app/base.py @@ -218,6 +218,10 @@ def run(self): debug.spew() if self.cfg.daemon: + if os.environ.get('NOTIFY_SOCKET'): + msg = "Warning: you shouldn't specify `daemon = True` when launching by systemd with `Type = notify`" + print(msg, file=sys.stderr, flush=True) + util.daemonize(self.cfg.enable_stdio_inheritance) # set python paths From f2d8b6d100bc9d1041f8db42b7ba5481ea9cf58a Mon Sep 17 00:00:00 2001 From: Ken Okada Date: Mon, 24 Aug 2020 21:15:31 +0900 Subject: [PATCH 017/164] Split long line --- gunicorn/app/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gunicorn/app/base.py b/gunicorn/app/base.py index c74ea3e8a..dbd05bc7f 100644 --- a/gunicorn/app/base.py +++ b/gunicorn/app/base.py @@ -219,7 +219,8 @@ def run(self): if self.cfg.daemon: if os.environ.get('NOTIFY_SOCKET'): - msg = "Warning: you shouldn't specify `daemon = True` when launching by systemd with `Type = notify`" + msg = "Warning: you shouldn't specify `daemon = True`" \ + " when launching by systemd with `Type = notify`" print(msg, file=sys.stderr, flush=True) util.daemonize(self.cfg.enable_stdio_inheritance) From 28df9926d7c650248d267eeafaa8b2c2cdd9af38 Mon Sep 17 00:00:00 2001 From: Chris Mildebrandt Date: Fri, 11 Sep 2020 22:41:27 -0700 Subject: [PATCH 018/164] Add additional logs when worker exits abnormally --- gunicorn/arbiter.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 532426d21..caa3fbad8 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -527,6 +527,27 @@ def reap_workers(self): reason = "App failed to load." raise HaltServer(reason, self.APP_LOAD_ERROR) + if exitcode > 0: + # If the exit code of the worker is greater than 0, + # let the user know. + self.log.error("Worker (pid:%s) exited with code %s.", + wpid, exitcode) + elif status > 0: + # If the exit code of the worker is 0 and the status + # is greater than 0, then it was most likely killed + # via a signal. + try: + sig_name = signal.Signals(status).name + except ValueError: + sig_name = "code %s".format(status) + msg = "Worker (pid:{}) was sent {}!".format( + wpid, sig_name) + + # Additional hint for SIGKILL + if status == signal.SIGKILL: + msg += " Perhaps out of memory?" + self.log.error(msg) + worker = self.WORKERS.pop(wpid, None) if not worker: continue From ca36d410fd86c82cb2c998a1dab5f7f967405fdd Mon Sep 17 00:00:00 2001 From: Chris Mildebrandt Date: Fri, 11 Sep 2020 23:05:38 -0700 Subject: [PATCH 019/164] Fix format call --- gunicorn/arbiter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index caa3fbad8..b8b92fec9 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -539,7 +539,7 @@ def reap_workers(self): try: sig_name = signal.Signals(status).name except ValueError: - sig_name = "code %s".format(status) + sig_name = "code {}".format(status) msg = "Worker (pid:{}) was sent {}!".format( wpid, sig_name) From da3b89b76550699a7475c31f3fbc56bbee8398db Mon Sep 17 00:00:00 2001 From: Joshua Kugler Date: Thu, 29 Oct 2020 17:26:59 -0800 Subject: [PATCH 020/164] The signature of __init__ on the "fall through" InotifyReloader was missing the extra_files paramater, so specifying the inotify reload engine on the command line when one did not have inotify installed would, instead of a nice message about needed inotify installed would result in the following traceback: ``` [2020-10-30 00:55:43 +0000] [7] [ERROR] Exception in worker process Traceback (most recent call last): File "/usr/lib/python3/dist-packages/gunicorn/arbiter.py", line 583, in spawn_worker worker.init_process() File "/usr/lib/python3/dist-packages/gunicorn/workers/base.py", line 132, in init_process self.reloader = reloader_cls(extra_files=self.cfg.reload_extra_files, TypeError: __init__() got an unexpected keyword argument 'extra_files' ``` I didn't see an easy way to writing a test for this, but would be happy to take pointers. --- gunicorn/reloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/reloader.py b/gunicorn/reloader.py index c1964785a..5db57bab8 100644 --- a/gunicorn/reloader.py +++ b/gunicorn/reloader.py @@ -118,7 +118,7 @@ def run(self): else: class InotifyReloader(object): - def __init__(self, callback=None): + def __init__(self, extra_files=None, callback=None): raise ImportError('You must have the inotify module installed to ' 'use the inotify reloader') From d8a125606519086a3636628ee2fd5d984deac6c3 Mon Sep 17 00:00:00 2001 From: Alex Hill Date: Thu, 10 Dec 2020 16:10:50 +0800 Subject: [PATCH 021/164] Document that gthread also uses worker_connections The `ThreadWorker` uses `worker_connections` it in its run loop to limit how many connections are accepted. --- docs/source/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 81b3c7749..070627a48 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -1312,7 +1312,7 @@ worker_connections The maximum number of simultaneous clients. -This setting only affects the Eventlet and Gevent worker types. +This setting only affects the ``gthread``, ``eventlet`` and ``gevent`` worker types. .. _max-requests: From 540b7100b3aee5217c1e8e023dfa04741af6c520 Mon Sep 17 00:00:00 2001 From: sergeypeloton <46500149+sergeypeloton@users.noreply.github.com> Date: Fri, 8 Jan 2021 14:36:14 -0500 Subject: [PATCH 022/164] Use warning level for 'Bad request' logs #2491 Debug level might not be practical to enable in production. --- gunicorn/workers/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index a6d84bd23..136b4afc8 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -245,7 +245,7 @@ def handle_error(self, req, client, addr, exc): status_int = 403 msg = "Invalid request from ip={ip}: {error}" - self.log.debug(msg.format(ip=addr[0], error=str(exc))) + self.log.warning(msg.format(ip=addr[0], error=str(exc))) else: if hasattr(req, "uri"): self.log.exception("Error handling request %s", req.uri) From 426fe70a541368bea9209de35df70e0cbb0ac3ed Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Fri, 12 Feb 2021 22:57:48 +0100 Subject: [PATCH 023/164] link to 2021 --- docs/source/news.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/news.rst b/docs/source/news.rst index a8e379ebe..0fb996133 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -49,6 +49,7 @@ History .. toctree:: :titlesonly: + 2021-news 2020-news 2019-news 2018-news From 6fbe6a115bb7978f53b3d33dd86f104a61fd2a4b Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Mon, 15 Feb 2021 18:21:57 -0800 Subject: [PATCH 024/164] Clarify grammar in FAQ about worker restarts Noticed by @guettli on GitHub, this sentence was likely supposed to say that workers may "be killed" and "start up", where "to start up" is a verb phrase while "startup" is a single word noun. Clarify by changing this to read "to stop and start". --- docs/source/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 71fb2b848..74cd50042 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -216,7 +216,7 @@ If you use a reverse proxy like NGINX you might see 502 returned to a client. In the gunicorn logs you might simply see ``[35] [INFO] Booting worker with pid: 35`` -It's completely normal for workers to be killed and startup, for example due to +It's completely normal for workers to be stop and start, for example due to max-requests setting. Ordinarily gunicorn will capture any signals and log something. This particular failure case is usually due to a SIGKILL being received, as it's From 1cfb59caaa3b4f12a387a087e25e3366ad2b2ad4 Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Mon, 15 Feb 2021 20:00:13 -0800 Subject: [PATCH 025/164] Fix changelog lint --- docs/source/2021-news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/2021-news.rst b/docs/source/2021-news.rst index b9f27b0c2..7875f0a77 100644 --- a/docs/source/2021-news.rst +++ b/docs/source/2021-news.rst @@ -11,7 +11,7 @@ Changelog - 2021 - document WEB_CONCURRENCY is set by, at least, Heroku - capture peername from accept: Avoid calls to getpeername by capturing the peer name returned by -accept + accept - log a warning when a worker was terminated due to a signal - fix tornado usage with latest versions of Django - add support for python -m gunicorn From 01f5d4dd0cf4250e92263a5e1e0749546ac24049 Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Mon, 15 Feb 2021 20:00:25 -0800 Subject: [PATCH 026/164] Remove 2020 unreleased changelog --- docs/source/2020-news.rst | 47 --------------------------------------- 1 file changed, 47 deletions(-) diff --git a/docs/source/2020-news.rst b/docs/source/2020-news.rst index c83a67549..1d91ef7e5 100644 --- a/docs/source/2020-news.rst +++ b/docs/source/2020-news.rst @@ -5,50 +5,3 @@ Changelog - 2020 .. note:: Please see :doc:`news` for the latest changes - -Unreleased -========== - -- document WEB_CONCURRENCY is set by, at least, Heroku -- capture peername from accept: Avoid calls to getpeername by capturing the peer name returned by -accept -- log a warning when a worker was terminated due to a signal -- fix tornado usage with latest versions of Django -- add support for python -m gunicorn -- fix systemd socket activation example -- allows to set wsgi application in configg file using `wsgi_app` -- document `--timeout = 0` -- always close a connection when the number of requests exceeds the max requests -- Disable keepalive during graceful shutdown -- kill tasks in the gthread workers during upgrade -- fix latency in gevent worker when accepting new requests -- fix file watcher: handle errors when new worker reboot and ensure the list of files is kept -- document the default name and path of the configuration file -- document how variable impact configuration -- document the `$PORT` environment variable -- added milliseconds option to request_time in access_log -- added PIP requirements to be used for example -- remove version from the Server header -- fix sendfile: use `socket.sendfile` instead of `os.sendfile` -- reloader: use absolute path to prevent empty to prevent0 `InotifyError` when a file - is added to the working directory -- Add --print-config option to print the resolved settings at startup. -- remove the `--log-dict-config` CLI flag because it never had a working format - (the `logconfig_dict` setting in configuration files continues to work) - - -** Breaking changes ** - -- minimum version is Python 3.5 -- remove version from the Server header - -** Documentation ** - - - -** Others ** - -- miscellaneous changes in the code base to be a better citizen with Python 3 -- remove dead code -- fix documentation generation - From 3b8ce3df1f36ef29a0b8e7a68d3d46d7c1cab180 Mon Sep 17 00:00:00 2001 From: Daan Luttik Date: Wed, 17 Feb 2021 00:54:49 +0100 Subject: [PATCH 027/164] Added a line to the documentation to further explain the use of logconfig_dict Added a line to make the question "how to use `logconfig_dict`" way easier to answer. --- docs/source/settings.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 6e977d1a0..4c8e9212f 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -321,6 +321,8 @@ older file configuration format. Format: https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig +For more context you can look at the default configuration dictionary for logging, which can be found at ``gunicorn.glogging.CONFIG_DEFAULTS``. + .. versionadded:: 19.8 .. _syslog-addr: From fb05e333776348651d1b2d759a5857969cb2f476 Mon Sep 17 00:00:00 2001 From: Smiler Lee Date: Wed, 24 Feb 2021 17:43:17 +0800 Subject: [PATCH 028/164] Fix a naming error for gthread worker --- gunicorn/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 8fd281bea..e7388c5ea 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -106,7 +106,7 @@ def worker_class_str(self): # are we using a threaded worker? is_sync = uri.endswith('SyncWorker') or uri == 'sync' if is_sync and self.threads > 1: - return "threads" + return "gthread" return uri @property From b83448bfb42184f43adf155f86bcd2a570a1139d Mon Sep 17 00:00:00 2001 From: Michael Milton Date: Tue, 9 Mar 2021 05:58:22 +1100 Subject: [PATCH 029/164] Clarify the secure scheme behaviour, with examples (#2492) --- gunicorn/config.py | 72 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index e7388c5ea..bc24b7000 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1236,10 +1236,16 @@ class SecureSchemeHeader(Setting): desc = """\ A dictionary containing headers and values that the front-end proxy - uses to indicate HTTPS requests. These tell Gunicorn to set + uses to indicate HTTPS requests. If the source IP is permitted by + ``forwarded-allow-ips`` (below), *and* at least one request header matches + a key-value pair listed in this dictionary, then Gunicorn will set ``wsgi.url_scheme`` to ``https``, so your application can tell that the request is secure. + If the other headers listed in this dictionary are not present in the request, they will be ignored, + but if the other headers are present and do not match the provided values, then + the request will fail to parse. See the note below for more detailed examples of this behaviour. + The dictionary should map upper-case header names to exact string values. The value comparisons are case-sensitive, unlike the header names, so make sure they're exactly what your front-end proxy sends @@ -1267,6 +1273,70 @@ class ForwardedAllowIPS(Setting): By default, the value of the ``FORWARDED_ALLOW_IPS`` environment variable. If it is not defined, the default is ``"127.0.0.1"``. + + .. note:: + + The interplay between the request headers, the value of ``forwarded_allow_ips``, and the value of + ``secure_scheme_headers`` is complex. Various scenarios are documented below to further elaborate. In each case, we + have a request from the remote address 134.213.44.18, and the default value of ``secure_scheme_headers``: + + .. code:: + + secure_scheme_headers = { + 'X-FORWARDED-PROTOCOL': 'ssl', + 'X-FORWARDED-PROTO': 'https', + 'X-FORWARDED-SSL': 'on' + } + + + .. list-table:: + :header-rows: 1 + :align: center + :widths: auto + + * - ``forwarded-allow-ips`` + - Secure Request Headers + - Result + - Explanation + * - .. code:: + + ["127.0.0.1"] + - .. code:: + + X-Forwarded-Proto: https + - .. code:: + + wsgi.url_scheme = "http" + - IP address was not allowed + * - .. code:: + + "*" + - + - .. code:: + + wsgi.url_scheme = "http" + - IP address allowed, but no secure headers provided + * - .. code:: + + "*" + - .. code:: + + X-Forwarded-Proto: https + - .. code:: + + wsgi.url_scheme = "https" + - IP address allowed, one request header matched + * - .. code:: + + ["134.213.44.18"] + - .. code:: + + X-Forwarded-Ssl: on + X-Forwarded-Proto: http + - ``InvalidSchemeHeaders()`` raised + - IP address allowed, but the two secure headers disagreed on if HTTPS was used + + """ From 917ebcdd5701e0859a3c250fbf2f0e80233b3fe5 Mon Sep 17 00:00:00 2001 From: Jack Zhang Date: Thu, 25 Mar 2021 22:31:58 +0800 Subject: [PATCH 030/164] Fixes #1695 --- docs/source/deploy.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/source/deploy.rst b/docs/source/deploy.rst index 43bad9a80..ff78575d0 100644 --- a/docs/source/deploy.rst +++ b/docs/source/deploy.rst @@ -38,6 +38,22 @@ To turn off buffering, you only need to add ``proxy_buffering off;`` to your } ... +If you want to ignore aborted requests like health check of Load Balancer, some +of which close the connection without waiting for a response, you need to turn +on ignoring client abort. + +To ignore aborted requests, you only need to add +``proxy_ignore_client_abort on;`` to your ``location`` block:: + + ... + proxy_ignore_client_abort on; + ... + +.. note:: + The default value of ``proxy_ignore_client_abort`` is ``off``. Error code + 499 may appear in Nginx log and ``Ignoring EPIPE`` may appear in Gunicorn + log if loglevel is set to ``debug``. + It is recommended to pass protocol information to Gunicorn. Many web frameworks use this information to generate URLs. Without this information, the application may mistakenly generate 'http' URLs in From cf55d2cec277f220ebd605989ce78ad1bb553c46 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 27 Mar 2021 03:07:27 +0100 Subject: [PATCH 031/164] bump version --- docs/site/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/site/index.html b/docs/site/index.html index 4fc6e170b..d7312d822 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -16,7 +16,7 @@
Latest version: 20.0.4 + href="https://docs.gunicorn.org/en/stable/">20.1.0
From 927c604fb7b7463bf66bfef416581925f6bafe17 Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Mon, 26 Apr 2021 12:34:01 +0000 Subject: [PATCH 032/164] Set daemon attribute instead of using setDaemon method that was deprecated in Python 3.10 --- gunicorn/reloader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gunicorn/reloader.py b/gunicorn/reloader.py index c1964785a..b7df8591a 100644 --- a/gunicorn/reloader.py +++ b/gunicorn/reloader.py @@ -17,7 +17,7 @@ class Reloader(threading.Thread): def __init__(self, extra_files=None, interval=1, callback=None): super().__init__() - self.setDaemon(True) + self.daemon = True self._extra_files = set(extra_files or ()) self._interval = interval self._callback = callback @@ -74,7 +74,7 @@ class InotifyReloader(threading.Thread): def __init__(self, extra_files=None, callback=None): super().__init__() - self.setDaemon(True) + self.daemon = True self._callback = callback self._dirs = set() self._watcher = Inotify() From 6a8ebb4844b2f28596ffe7421eb9f7d08c8dc4d8 Mon Sep 17 00:00:00 2001 From: Sergey Shepelev Date: Thu, 6 May 2021 12:54:06 +0300 Subject: [PATCH 033/164] eventlet worker: ALREADY_HANDLED -> WSGI_LOCAL Eventlet v0.30.3+ removed wsgi.ALREADY_HANDLED in favor of `wsgi.WSGI_LOCAL.already_handled: bool` Sorry, this breaking change happened during only patch version increase 0.30.2 -> 0.30.3 https://github.com/eventlet/eventlet/issues/543 https://github.com/eventlet/eventlet/pull/544 --- gunicorn/workers/geventlet.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/gunicorn/workers/geventlet.py b/gunicorn/workers/geventlet.py index ffdb206c0..ea82f3d62 100644 --- a/gunicorn/workers/geventlet.py +++ b/gunicorn/workers/geventlet.py @@ -17,11 +17,16 @@ from eventlet import hubs, greenthread from eventlet.greenio import GreenSocket -from eventlet.wsgi import ALREADY_HANDLED as EVENTLET_ALREADY_HANDLED +import eventlet.wsgi import greenlet from gunicorn.workers.base_async import AsyncWorker +# ALREADY_HANDLED is removed in 0.30.3+ now it's `WSGI_LOCAL.already_handled: bool` +# https://github.com/eventlet/eventlet/pull/544 +EVENTLET_WSGI_LOCAL = getattr(eventlet.wsgi, "WSGI_LOCAL", None) +EVENTLET_ALREADY_HANDLED = getattr(eventlet.wsgi, "ALREADY_HANDLED", None) + def _eventlet_socket_sendfile(self, file, offset=0, count=None): # Based on the implementation in gevent which in turn is slightly @@ -125,6 +130,10 @@ def patch(self): patch_sendfile() def is_already_handled(self, respiter): + # eventlet >= 0.30.3 + if getattr(EVENTLET_WSGI_LOCAL, "already_handled", None): + raise StopIteration() + # eventlet < 0.30.3 if respiter == EVENTLET_ALREADY_HANDLED: raise StopIteration() return super().is_already_handled(respiter) From 90ef9a6a9407fcf602da769a950b5e984fa9d025 Mon Sep 17 00:00:00 2001 From: Jack Zhang Date: Thu, 10 Jun 2021 10:40:13 +0800 Subject: [PATCH 034/164] Add nginx documentation for proxy_ignore_client_abort --- docs/source/deploy.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/deploy.rst b/docs/source/deploy.rst index ff78575d0..f96766161 100644 --- a/docs/source/deploy.rst +++ b/docs/source/deploy.rst @@ -40,7 +40,7 @@ To turn off buffering, you only need to add ``proxy_buffering off;`` to your If you want to ignore aborted requests like health check of Load Balancer, some of which close the connection without waiting for a response, you need to turn -on ignoring client abort. +on `ignoring client abort`_. To ignore aborted requests, you only need to add ``proxy_ignore_client_abort on;`` to your ``location`` block:: @@ -373,3 +373,4 @@ utility:: .. _Virtualenv: https://pypi.python.org/pypi/virtualenv .. _Systemd: https://www.freedesktop.org/wiki/Software/systemd/ .. _Gaffer: https://gaffer.readthedocs.io/ +.. _`ignoring client abort`: http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_ignore_client_abort From 45687c358ccc4067100f526565213e4835b4a5f7 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 18 Aug 2021 13:09:03 +0300 Subject: [PATCH 035/164] Replace deprecated unittest alias --- .../frameworks/django/testing/testing/apps/someapp/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/frameworks/django/testing/testing/apps/someapp/tests.py b/examples/frameworks/django/testing/testing/apps/someapp/tests.py index 3b3114889..85c920bb8 100755 --- a/examples/frameworks/django/testing/testing/apps/someapp/tests.py +++ b/examples/frameworks/django/testing/testing/apps/someapp/tests.py @@ -12,7 +12,7 @@ def test_basic_addition(self): """ Tests that 1 + 1 always equals 2. """ - self.failUnlessEqual(1 + 1, 2) + self.assertEqual(1 + 1, 2) __test__ = {"doctest": """ Another way to test that 1 + 1 is equal to 2. From 0215f5dc36395186ccf23b685e8881e57374c6a3 Mon Sep 17 00:00:00 2001 From: yingjie Date: Thu, 26 Aug 2021 22:41:12 +0800 Subject: [PATCH 036/164] docs: add examples for cert-reqs --- docs/source/settings.rst | 10 ++++++++++ gunicorn/config.py | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 4c8e9212f..a8bb31d39 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -540,6 +540,16 @@ TLS_SERVER Auto-negotiate the highest protocol version like TLS, Whether client certificate is required (see stdlib ssl module's) +**Options:** + +`--cert-reqs=0` --- no client veirifcation + +`--cert-reqs=1` --- ssl.CERT_OPTIONAL + +`--cert-reqs=2` --- ssl.CERT_REQUIRED + + + .. _ca-certs: ``ca_certs`` diff --git a/gunicorn/config.py b/gunicorn/config.py index bc24b7000..c27775fc2 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -2088,6 +2088,13 @@ class CertReqs(Setting): default = ssl.CERT_NONE desc = """\ Whether client certificate is required (see stdlib ssl module's) + ============== =========================== + `--cert-reqs=0` --- no client veirifcation + + `--cert-reqs=1` --- ssl.CERT_OPTIONAL + + `--cert-reqs=2` --- ssl.CERT_REQUIRED + ============== =========================== """ From df82cac7d98fee717473b004482a31b702581176 Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Wed, 25 Aug 2021 13:54:50 +0800 Subject: [PATCH 037/164] fix: gunicorn run tornado app failed --- gunicorn/workers/gtornado.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gunicorn/workers/gtornado.py b/gunicorn/workers/gtornado.py index 9dd3d7bc4..d3bd6d6ba 100644 --- a/gunicorn/workers/gtornado.py +++ b/gunicorn/workers/gtornado.py @@ -110,7 +110,8 @@ def run(self): if not isinstance(app, tornado.web.Application) or \ isinstance(app, tornado.wsgi.WSGIApplication): app = WSGIContainer(app) - elif not isinstance(app, WSGIContainer): + elif not isinstance(app, WSGIContainer) and \ + not isinstance(app, tornado.web.Application): app = WSGIContainer(app) # Monkey-patching HTTPConnection.finish to count the From 5a581c0b144c1f356c39d513d60015e878655ea4 Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Fri, 10 Sep 2021 19:59:15 +0300 Subject: [PATCH 038/164] Update SSLContext handling * Change deprecated ssl.wrap_socket() to SSLContext.wrap_context(). * Add new server hook to allow user to create custom SSLContext. * Updated the documentation. Signed-off-by: Tero Saarni --- docs/source/settings.rst | 21 ++++++++++++++++++++- examples/example_config.py | 30 ++++++++++++++++++++++++++++++ gunicorn/config.py | 19 +++++++++++++++++++ gunicorn/sock.py | 20 +++++++++++++++++++- gunicorn/workers/geventlet.py | 5 ++--- gunicorn/workers/ggevent.py | 3 ++- gunicorn/workers/gthread.py | 4 ++-- gunicorn/workers/gtornado.py | 10 +++------- gunicorn/workers/sync.py | 5 ++--- 9 files changed, 99 insertions(+), 18 deletions(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 4c8e9212f..bf2ad7378 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -916,6 +916,26 @@ Called just before exiting Gunicorn. The callable needs to accept a single instance variable for the Arbiter. +.. _ssl-context: + +``ssl_context`` +~~~~~~~~~~~~~~~ + +**Default:** + +.. code-block:: python + + def ssl_context(config): + return None + +Called when SSLContext is needed. + +Allows fully customized SSL context to be used in place of the default +context. + +The callable needs to accept a single instance variable for the Config. +The callable needs to return SSLContext object. + Server Mechanics ---------------- @@ -1506,4 +1526,3 @@ set this to a higher value. .. note:: ``sync`` worker does not support persistent connections and will ignore this option. - diff --git a/examples/example_config.py b/examples/example_config.py index f8f3c1dfc..83c3ffbcb 100644 --- a/examples/example_config.py +++ b/examples/example_config.py @@ -214,3 +214,33 @@ def worker_int(worker): def worker_abort(worker): worker.log.info("worker received SIGABRT signal") + +def ssl_context(conf): + import ssl + + def set_defaults(context): + context.verify_mode = conf.cert_reqs + context.minimum_version = ssl.TLSVersion.TLSv1_3 + if conf.ciphers: + context.set_ciphers(conf.ciphers) + if conf.ca_certs: + context.load_verify_locations(cafile=conf.ca_certs) + + # Return different server certificate depending which hostname the client + # uses. Requires Python 3.7 or later. + def sni_callback(socket, server_hostname, context): + if server_hostname == "foo.127.0.0.1.nip.io": + new_context = ssl.SSLContext() + new_context.load_cert_chain(certfile="foo.pem", keyfile="foo-key.pem") + set_defaults(new_context) + socket.context = new_context + + context = ssl.SSLContext(conf.ssl_version) + context.sni_callback = sni_callback + set_defaults(context) + + # Load fallback certificate that will be returned when there is no match + # or client did not set TLS SNI (server_hostname == None) + context.load_cert_chain(certfile=conf.certfile, keyfile=conf.keyfile) + + return context diff --git a/gunicorn/config.py b/gunicorn/config.py index bc24b7000..3f397d126 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1967,6 +1967,25 @@ def on_exit(server): The callable needs to accept a single instance variable for the Arbiter. """ +class NewSSLContext(Setting): + name = "ssl_context" + section = "Server Hooks" + validator = validate_callable(1) + type = callable + + def ssl_context(config): + return None + + default = staticmethod(ssl_context) + desc = """\ + Called when SSLContext is needed. + + Allows fully customized SSL context to be used in place of the default + context. + + The callable needs to accept a single instance variable for the Config. + The callable needs to return SSLContext object. + """ class ProxyProtocol(Setting): name = "proxy_protocol" diff --git a/gunicorn/sock.py b/gunicorn/sock.py index d45867709..1481c23bb 100644 --- a/gunicorn/sock.py +++ b/gunicorn/sock.py @@ -6,6 +6,7 @@ import errno import os import socket +import ssl import stat import sys import time @@ -203,10 +204,27 @@ def create_sockets(conf, log, fds=None): return listeners - def close_sockets(listeners, unlink=True): for sock in listeners: sock_name = sock.getsockname() sock.close() if unlink and _sock_type(sock_name) is UnixSocket: os.unlink(sock_name) + +def ssl_context(conf): + context = conf.ssl_context(conf) + if context is None: + context = ssl.SSLContext(conf.ssl_version) + context.load_cert_chain(certfile=conf.certfile, keyfile=conf.keyfile) + context.verify_mode = conf.cert_reqs + if conf.ciphers: + context.set_ciphers(conf.ciphers) + if conf.ca_certs: + context.load_verify_locations(cafile=conf.ca_certs) + + return context + +def ssl_wrap_socket(sock, conf): + return ssl_context(conf).wrap_socket(sock, server_side=True, + suppress_ragged_eofs=conf.suppress_ragged_eofs, + do_handshake_on_connect=conf.do_handshake_on_connect) diff --git a/gunicorn/workers/geventlet.py b/gunicorn/workers/geventlet.py index ea82f3d62..844a1e061 100644 --- a/gunicorn/workers/geventlet.py +++ b/gunicorn/workers/geventlet.py @@ -21,6 +21,7 @@ import greenlet from gunicorn.workers.base_async import AsyncWorker +from gunicorn.sock import ssl_wrap_socket # ALREADY_HANDLED is removed in 0.30.3+ now it's `WSGI_LOCAL.already_handled: bool` # https://github.com/eventlet/eventlet/pull/544 @@ -153,9 +154,7 @@ def timeout_ctx(self): def handle(self, listener, client, addr): if self.cfg.is_ssl: - client = eventlet.wrap_ssl(client, server_side=True, - **self.cfg.ssl_options) - + client = ssl_wrap_socket(client, self.cfg) super().handle(listener, client, addr) def run(self): diff --git a/gunicorn/workers/ggevent.py b/gunicorn/workers/ggevent.py index 3941814f5..81a339a80 100644 --- a/gunicorn/workers/ggevent.py +++ b/gunicorn/workers/ggevent.py @@ -24,6 +24,7 @@ import gunicorn from gunicorn.http.wsgi import base_environ +from gunicorn.sock import ssl_context from gunicorn.workers.base_async import AsyncWorker VERSION = "gevent/%s gunicorn/%s" % (gevent.__version__, gunicorn.__version__) @@ -58,7 +59,7 @@ def run(self): ssl_args = {} if self.cfg.is_ssl: - ssl_args = dict(server_side=True, **self.cfg.ssl_options) + ssl_args = dict(ssl_context=ssl_context(self.cfg)) for s in self.sockets: s.setblocking(1) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index d53181154..73ea6fdf8 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -27,6 +27,7 @@ from . import base from .. import http from .. import util +from .. import sock from ..http import wsgi @@ -49,8 +50,7 @@ def init(self): if self.parser is None: # wrap the socket if needed if self.cfg.is_ssl: - self.sock = ssl.wrap_socket(self.sock, server_side=True, - **self.cfg.ssl_options) + self.sock = sock.ssl_wrap_socket(self.sock, self.cfg) # initialize the parser self.parser = http.RequestParser(self.cfg, self.sock, self.client) diff --git a/gunicorn/workers/gtornado.py b/gunicorn/workers/gtornado.py index 9dd3d7bc4..9da49957b 100644 --- a/gunicorn/workers/gtornado.py +++ b/gunicorn/workers/gtornado.py @@ -17,6 +17,7 @@ from tornado.wsgi import WSGIContainer from gunicorn.workers.base import Worker from gunicorn import __version__ as gversion +from gunicorn.sock import ssl_context # Tornado 5.0 updated its IOLoop, and the `io_loop` arguments to many @@ -139,16 +140,11 @@ def on_close(instance, server_conn): server_class = _HTTPServer if self.cfg.is_ssl: - _ssl_opt = copy.deepcopy(self.cfg.ssl_options) - # tornado refuses initialization if ssl_options contains following - # options - del _ssl_opt["do_handshake_on_connect"] - del _ssl_opt["suppress_ragged_eofs"] if TORNADO5: - server = server_class(app, ssl_options=_ssl_opt) + server = server_class(app, ssl_options=ssl_context(self.cfg)) else: server = server_class(app, io_loop=self.ioloop, - ssl_options=_ssl_opt) + ssl_options=ssl_context(self.cfg)) else: if TORNADO5: server = server_class(app) diff --git a/gunicorn/workers/sync.py b/gunicorn/workers/sync.py index eeb7f633f..4bdf0003e 100644 --- a/gunicorn/workers/sync.py +++ b/gunicorn/workers/sync.py @@ -14,6 +14,7 @@ import gunicorn.http as http import gunicorn.http.wsgi as wsgi +import gunicorn.sock as sock import gunicorn.util as util import gunicorn.workers.base as base @@ -128,9 +129,7 @@ def handle(self, listener, client, addr): req = None try: if self.cfg.is_ssl: - client = ssl.wrap_socket(client, server_side=True, - **self.cfg.ssl_options) - + client = sock.ssl_wrap_socket(client, self.cfg) parser = http.RequestParser(self.cfg, client, addr) req = next(parser) self.handle_request(listener, req, client, addr) From 3a6649a9ef9081c342bd5363828c7e035d20aaba Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Mon, 27 Sep 2021 16:09:50 +0530 Subject: [PATCH 039/164] Fix link --- docs/source/design.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/design.rst b/docs/source/design.rst index 066fceafe..992340495 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -151,4 +151,4 @@ code in the master process). .. _aiohttp: https://docs.aiohttp.org/en/stable/deployment.html#nginx-gunicorn .. _`example`: https://github.com/benoitc/gunicorn/blob/master/examples/frameworks/flaskapp_aiohttp_wsgi.py .. _Psycopg: http://initd.org/psycopg/ -.. _psycogreen: https://bitbucket.org/dvarrazzo/psycogreen +.. _psycogreen: https://github.com/psycopg/psycogreen/ From e3b0b1f91aae0d2ec50098595fd43b05e68b3758 Mon Sep 17 00:00:00 2001 From: Paul Miller Date: Fri, 8 Oct 2021 21:04:12 -0400 Subject: [PATCH 040/164] integral? that took me a while to figure out integral: "In mathematics, an integral assigns numbers to functions in a way that describes displacement, area, volume, and other concepts that arise by combining infinitesimal data." integer: an int type --- gunicorn/http/body.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/http/body.py b/gunicorn/http/body.py index afde36854..aa1af2cb3 100644 --- a/gunicorn/http/body.py +++ b/gunicorn/http/body.py @@ -18,7 +18,7 @@ def __init__(self, req, unreader): def read(self, size): if not isinstance(size, int): - raise TypeError("size must be an integral type") + raise TypeError("size must be an integer type") if size < 0: raise ValueError("Size must be positive.") if size == 0: From 141a8f3d773fe99b0df4746a8379c06082c90ae3 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Fri, 12 Nov 2021 13:03:20 -0500 Subject: [PATCH 041/164] docs: gthread is a sync worker --- docs/source/design.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/design.rst b/docs/source/design.rst index 066fceafe..0d5454492 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -38,6 +38,12 @@ applications are programmed. closed after response has been sent (even if you manually add ``Keep-Alive`` or ``Connection: keep-alive`` header in your application). +The worker `gthread` is a threaded worker. It accepts connections in the +main loop, accepted connections are added to the thread pool as a +connection job. On keepalive connections are put back in the loop +waiting for an event. If no event happen after the keep alive timeout, +the connection is closed. + Async Workers ------------- @@ -68,12 +74,6 @@ AsyncIO Workers These workers are compatible with Python 3. -The worker `gthread` is a threaded worker. It accepts connections in the -main loop, accepted connections are added to the thread pool as a -connection job. On keepalive connections are put back in the loop -waiting for an event. If no event happen after the keep alive timeout, -the connection is closed. - You can port also your application to use aiohttp_'s ``web.Application`` API and use the ``aiohttp.worker.GunicornWebWorker`` worker. From f639128bd7673195993e6156edc288d7a11f64fa Mon Sep 17 00:00:00 2001 From: Vytautas Liuolia Date: Fri, 12 Nov 2021 21:39:10 +0100 Subject: [PATCH 042/164] Fix sendfile behaviour for open files with non-zero offset. --- gunicorn/http/wsgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 478677f4b..f14e6794e 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -372,7 +372,7 @@ def sendfile(self, respiter): chunk_size = "%X\r\n" % nbytes self.sock.sendall(chunk_size.encode('utf-8')) - self.sock.sendfile(respiter.filelike, count=nbytes) + self.sock.sendfile(respiter.filelike, offset=offset, count=nbytes) if self.is_chunked(): self.sock.sendall(b"\r\n") From 76f8da24cbb992d168e01bda811452bcf3b8f5b3 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 17 Dec 2021 15:16:34 +0100 Subject: [PATCH 043/164] Revert "Log a warning when a worker was terminated due to a signal" This reverts commit b695b497b9b8e7351808848ab9e342219601742a. --- gunicorn/arbiter.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 24ec38744..7ca2f6b79 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -526,12 +526,6 @@ def reap_workers(self): if exitcode == self.APP_LOAD_ERROR: reason = "App failed to load." raise HaltServer(reason, self.APP_LOAD_ERROR) - if os.WIFSIGNALED(status): - self.log.warning( - "Worker with pid %s was terminated due to signal %s", - wpid, - os.WTERMSIG(status) - ) worker = self.WORKERS.pop(wpid, None) if not worker: From cf3619f831a14baa9d3bda8a3aa589b9bf23eaf4 Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Fri, 14 Jan 2022 23:34:02 +0800 Subject: [PATCH 044/164] Fix typos --- NOTICE | 2 +- docs/site/index.html | 2 +- docs/source/2012-news.rst | 2 +- docs/source/2013-news.rst | 6 +++--- docs/source/2014-news.rst | 4 ++-- docs/source/2019-news.rst | 2 +- docs/source/community.rst | 2 +- gunicorn/glogging.py | 2 +- gunicorn/workers/workertmp.py | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/NOTICE b/NOTICE index 8506656bd..942a605d3 100644 --- a/NOTICE +++ b/NOTICE @@ -86,4 +86,4 @@ OTHER DEALINGS IN THE SOFTWARE. util/unlink.py -------------- -backport frop python3 Lib/test/support.py +backport from python3 Lib/test/support.py diff --git a/docs/site/index.html b/docs/site/index.html index d7312d822..31545d68d 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -118,7 +118,7 @@

Project Management

  • Forum
  • Mailing list -

    Project maintenance guidelines are avaible on the wiki

    +

    Project maintenance guidelines are available on the wiki

    Irc

    The Gunicorn channel is on the Freenode IRC diff --git a/docs/source/2012-news.rst b/docs/source/2012-news.rst index 6ae37bfe6..ce4f7cc4d 100644 --- a/docs/source/2012-news.rst +++ b/docs/source/2012-news.rst @@ -75,7 +75,7 @@ Changelog - 2012 - fix tornado.wsgi.WSGIApplication calling error - **breaking change**: take the control on graceful reload back. - graceful can't be overrided anymore using the on_reload function. + graceful can't be overridden anymore using the on_reload function. 0.14.3 / 2012-05-15 ------------------- diff --git a/docs/source/2013-news.rst b/docs/source/2013-news.rst index 5f848e2cf..eb8cf556a 100644 --- a/docs/source/2013-news.rst +++ b/docs/source/2013-news.rst @@ -38,10 +38,10 @@ Changelog - 2013 - fix: give the initial global_conf to paster application - fix: fix 'Expect: 100-continue' support on Python 3 -New versionning: +New versioning: ++++++++++++++++ -With this release, the versionning of Gunicorn is changing. Gunicorn is +With this release, the versioning of Gunicorn is changing. Gunicorn is stable since a long time and there is no point to release a "1.0" now. It should have been done since a long time. 0.17 really meant it was the 17th stable version. From the beginning we have only 2 kind of @@ -49,7 +49,7 @@ releases: major release: releases with major changes or huge features added services releases: fixes and minor features added So from now we will -apply the following versionning ``.``. For example ``17.5`` is a +apply the following versioning ``.``. For example ``17.5`` is a service release. 0.17.4 / 2013-04-24 diff --git a/docs/source/2014-news.rst b/docs/source/2014-news.rst index 1a9af82bb..3eec18fcd 100644 --- a/docs/source/2014-news.rst +++ b/docs/source/2014-news.rst @@ -71,7 +71,7 @@ AioHttp worker Async worker ++++++++++++ -- fix :issue:`790`: StopIteration shouldn't be catched at this level. +- fix :issue:`790`: StopIteration shouldn't be caught at this level. Logging @@ -180,7 +180,7 @@ core - add: syslog logging can now be done to a unix socket - fix logging: don't try to redirect stdout/stderr to the logfile. - fix logging: don't propagate log -- improve logging: file option can be overriden by the gunicorn options +- improve logging: file option can be overridden by the gunicorn options `--error-logfile` and `--access-logfile` if they are given. - fix: don't override SERVER_* by the Host header - fix: handle_error diff --git a/docs/source/2019-news.rst b/docs/source/2019-news.rst index 771c28f0e..28b69216b 100644 --- a/docs/source/2019-news.rst +++ b/docs/source/2019-news.rst @@ -90,7 +90,7 @@ Changelog - 2019 - Simplify Paste Deployment documentation - Fix root logging: root and logger are same level. - Fixed typo in ssl_version documentation -- Documented systemd deployement unit examples +- Documented systemd deployment unit examples - Added systemd sd_notify support - Fixed typo in gthread.py - Added `tornado `_ 5 and 6 support diff --git a/docs/source/community.rst b/docs/source/community.rst index 249d82c5b..e16767447 100644 --- a/docs/source/community.rst +++ b/docs/source/community.rst @@ -15,7 +15,7 @@ for 3 different purposes: * `Mailing list `_ : Discussion of Gunicorn development, new features and project management. -Project maintenance guidelines are avaible on the `wiki `_ +Project maintenance guidelines are available on the `wiki `_ . IRC diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index 08bc121a1..6d80a3ac3 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -275,7 +275,7 @@ def log(self, lvl, msg, *args, **kwargs): self.error_log.log(lvl, msg, *args, **kwargs) def atoms(self, resp, req, environ, request_time): - """ Gets atoms for log formating. + """ Gets atoms for log formatting. """ status = resp.status if isinstance(status, str): diff --git a/gunicorn/workers/workertmp.py b/gunicorn/workers/workertmp.py index 65bbe54fa..cc79ecd6d 100644 --- a/gunicorn/workers/workertmp.py +++ b/gunicorn/workers/workertmp.py @@ -28,7 +28,7 @@ def __init__(self, cfg): if cfg.uid != os.geteuid() or cfg.gid != os.getegid(): util.chown(name, cfg.uid, cfg.gid) - # unlink the file so we don't leak tempory files + # unlink the file so we don't leak temporary files try: if not IS_CYGWIN: util.unlink(name) From e5a97150c95d68551a3cd3a3ecf356dcd20b667c Mon Sep 17 00:00:00 2001 From: Krystian Date: Fri, 4 Feb 2022 05:11:51 +0800 Subject: [PATCH 045/164] fix chdir documentation typo (#2656) --- docs/source/settings.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 4c8e9212f..874008997 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -976,9 +976,11 @@ Set the ``SO_REUSEPORT`` flag on the listening socket. **Command line:** ``--chdir`` -**Default:** ``'/Users/chainz/Documents/Projects/gunicorn/docs'`` +**Default:** ``'.'`` -Change directory to specified directory before loading apps. +Change directory to specified directory before loading apps. + +Default is the current directory. .. _daemon: From 835a4fc42089d12b41bf1564844d7b518cc5e324 Mon Sep 17 00:00:00 2001 From: Brett Randall Date: Thu, 27 Jan 2022 16:03:13 +1100 Subject: [PATCH 046/164] Ensure fd 0 stdin --- gunicorn/util.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/gunicorn/util.py b/gunicorn/util.py index a821e3572..af414093d 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -486,7 +486,10 @@ def daemonize(enable_stdio_inheritance=False): closerange(0, 3) fd_null = os.open(REDIRECT_TO, os.O_RDWR) + # PEP 446, make fd for /dev/null inheritable + os.set_inheritable(fd_null, True) + # expect fd_null to be always 0 here, but in-case not ... if fd_null != 0: os.dup2(fd_null, 0) @@ -494,13 +497,17 @@ def daemonize(enable_stdio_inheritance=False): os.dup2(fd_null, 2) else: - fd_null = os.open(REDIRECT_TO, os.O_RDWR) - # Always redirect stdin to /dev/null as we would # never expect to need to read interactive input. + os.close(0) + + fd_null = os.open(REDIRECT_TO, os.O_RDWR) + # PEP 446, make fd for /dev/null inheritable + os.set_inheritable(fd_null, True) + + # expect fd_null to be always 0 here, but in-case not ... if fd_null != 0: - os.close(0) os.dup2(fd_null, 0) # If stdout and stderr are still connected to From eedc3a38b66b4c5a5ae975baefdc9894288294e4 Mon Sep 17 00:00:00 2001 From: Brett Randall Date: Sun, 30 Jan 2022 17:58:52 +1100 Subject: [PATCH 047/164] Refactoring: common stdin Date: Mon, 31 Jan 2022 07:18:45 +1100 Subject: [PATCH 048/164] Revert "Refactoring: common stdin Date: Mon, 31 Jan 2022 07:26:26 +1100 Subject: [PATCH 049/164] Undo changes that make -R branch logic behave the same as non -R branch, that is, to close(0) and open /dev/null as fd=0 instead of fd=3. (Partially) Revert "Ensure fd 0 stdin Date: Mon, 7 Feb 2022 07:39:34 +1100 Subject: [PATCH 050/164] Updated THANKS. --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index 2b226f35b..cc48fa5d0 100644 --- a/THANKS +++ b/THANKS @@ -109,6 +109,7 @@ Kyle Mulka Lars Hansson Leonardo Santagada Levi Gross +licunlong Łukasz Kucharski Mahmoud Hashemi Malthe Borch From a16b8975a921287d6d203a58dbadd8be182eda4f Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 2 Nov 2021 19:57:35 +0100 Subject: [PATCH 051/164] GitHub Action to run tox Because Travis CI seems to be on vacation... https://travis-ci.org/github/benoitc/gunicorn --- .github/workflows/tox.yml | 42 +++++++++++++++++++++++++++++++++ appveyor.yml | 34 ++++++++++++++++---------- tests/workers/test_geventlet.py | 5 ++++ tox.ini | 9 +++---- 4 files changed, 73 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/tox.yml diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml new file mode 100644 index 000000000..611a0938c --- /dev/null +++ b/.github/workflows/tox.yml @@ -0,0 +1,42 @@ +name: tox +on: [push, pull_request] +jobs: + tox-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: pip install --upgrade pip + - run: tox -e lint + + tox-docs-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: pip install --upgrade pip + - run: tox -e docs-lint + + tox-pycodestyle: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: pip install --upgrade pip + - run: tox -e pycodestyle + + tox: + strategy: + fail-fast: false + matrix: # All OSes pass except Windows because tests need Unix-only fcntl, grp, pwd, etc. + os: [ubuntu-latest] # [macos-latest, ubuntu-latest, windows-latest] + python: ['3.10'] # ['3.6', '3.7', '3.8', '3.9', '3.10', 'pypy-3.7'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - run: pip install --upgrade pip + - run: pip install tox + - run: tox -e py diff --git a/appveyor.yml b/appveyor.yml index 0bcf6c6ca..5eb48e9cd 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,17 +2,23 @@ version: '{branch}.{build}' environment: matrix: - TOXENV: lint - PYTHON: "C:\\Python37-x64" - - TOXENV: py35 - PYTHON: "C:\\Python35-x64" - - TOXENV: py36 - PYTHON: "C:\\Python36-x64" - - TOXENV: py37 - PYTHON: "C:\\Python37-x64" - - TOXENV: py38 PYTHON: "C:\\Python38-x64" - - TOXENV: py39 - PYTHON: "C:\\Python39-x64" + - TOXENV: docs-lint + PYTHON: "C:\\Python38-x64" + - TOXENV: pycodestyle + PYTHON: "C:\\Python38-x64" + # Windows is not ready for testing!!! + # Python's fcntl, grp, pwd, os.geteuid(), and socket.AF_UNIX are all Unix-only. + #- TOXENV: py35 + # PYTHON: "C:\\Python35-x64" + #- TOXENV: py36 + # PYTHON: "C:\\Python36-x64" + #- TOXENV: py37 + # PYTHON: "C:\\Python37-x64" + #- TOXENV: py38 + # PYTHON: "C:\\Python38-x64" + #- TOXENV: py39 + # PYTHON: "C:\\Python39-x64" matrix: allow_failures: - TOXENV: py35 @@ -20,11 +26,13 @@ matrix: - TOXENV: py37 - TOXENV: py38 - TOXENV: py39 -init: SET "PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" +init: + - SET "PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" install: - pip install tox -build: off -test_script: tox +build: false +test_script: + - tox cache: # Not including the .tox directory since it takes longer to download/extract # the cache archive than for tox to clean install from the pip cache. diff --git a/tests/workers/test_geventlet.py b/tests/workers/test_geventlet.py index 815dcec34..030505a62 100644 --- a/tests/workers/test_geventlet.py +++ b/tests/workers/test_geventlet.py @@ -2,6 +2,11 @@ # # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. +import pytest + +@pytest.mark.xfail( + reason="TypeError: cannot set 'is_timeout' of immutable type 'TimeoutError'" +) def test_import(): __import__('gunicorn.workers.geventlet') diff --git a/tox.ini b/tox.ini index 6000eb617..64bec90d3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,16 +1,17 @@ [tox] -envlist = py35, py36, py37, py38, py39, pypy3, lint +envlist = py35, py36, py37, py38, py39, py310, pypy3, lint, docs-lint, pycodestyle skipsdist = True [testenv] usedevelop = True -commands = py.test --cov=gunicorn {posargs} +commands = pytest --cov=gunicorn {posargs} deps = -rrequirements_test.txt [testenv:lint] commands = pylint -j0 \ + --disable=consider-using-f-string,consider-using-from-import,consider-using-with,deprecated-method,unspecified-encoding \ gunicorn \ tests/test_arbiter.py \ tests/test_config.py \ @@ -46,5 +47,5 @@ deps = pycodestyle [pycodestyle] -max-line-length = 120 -ignore = E129,W503,W504,W606 +max-line-length = 127 +ignore = E122,E126,E128,E129,E226,E302,E303,E501,E722,E741,W291,W293,W503,W504,W606 From f587bfaf0f76b4cb7fd27c9d3c714c5622048d35 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 3 Nov 2021 16:17:45 +0100 Subject: [PATCH 052/164] GitHub Action to run tox --- .github/workflows/tox.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 611a0938c..ca0cb26f5 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -7,6 +7,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install --upgrade pip + - run: pip install tox - run: tox -e lint tox-docs-lint: @@ -15,6 +16,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install --upgrade pip + - run: pip install tox - run: tox -e docs-lint tox-pycodestyle: @@ -23,6 +25,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install --upgrade pip + - run: pip install tox - run: tox -e pycodestyle tox: From 1feb7c59a227ca18851914df529543acb8b7f8b4 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 4 Feb 2022 14:43:38 +0100 Subject: [PATCH 053/164] Revert the xfail for Python 3.10 --- tests/workers/test_geventlet.py | 5 ----- tox.ini | 2 -- 2 files changed, 7 deletions(-) diff --git a/tests/workers/test_geventlet.py b/tests/workers/test_geventlet.py index 030505a62..815dcec34 100644 --- a/tests/workers/test_geventlet.py +++ b/tests/workers/test_geventlet.py @@ -2,11 +2,6 @@ # # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -import pytest - -@pytest.mark.xfail( - reason="TypeError: cannot set 'is_timeout' of immutable type 'TimeoutError'" -) def test_import(): __import__('gunicorn.workers.geventlet') diff --git a/tox.ini b/tox.ini index 64bec90d3..4ee134352 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,6 @@ deps = [testenv:lint] commands = pylint -j0 \ - --disable=consider-using-f-string,consider-using-from-import,consider-using-with,deprecated-method,unspecified-encoding \ gunicorn \ tests/test_arbiter.py \ tests/test_config.py \ @@ -48,4 +47,3 @@ deps = [pycodestyle] max-line-length = 127 -ignore = E122,E126,E128,E129,E226,E302,E303,E501,E722,E741,W291,W293,W503,W504,W606 From eaebf6d72bbdc43a9a16a704f5e66050d57c06f2 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 4 Feb 2022 14:48:24 +0100 Subject: [PATCH 054/164] Revert the xfail for Python 3.10 --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4ee134352..196bac39d 100644 --- a/tox.ini +++ b/tox.ini @@ -46,4 +46,5 @@ deps = pycodestyle [pycodestyle] -max-line-length = 127 +max-line-length = 120 +ignore = E129,W503,W504,W606 From 80a62afc1e410f92edf451cc7e7469925e46378c Mon Sep 17 00:00:00 2001 From: Brett Randall Date: Mon, 7 Feb 2022 08:43:10 +1100 Subject: [PATCH 055/164] Updated THANKS. --- THANKS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/THANKS b/THANKS index cc48fa5d0..117134970 100644 --- a/THANKS +++ b/THANKS @@ -39,6 +39,7 @@ Chris Adams Chris Forbes Chris Lamb Chris Streeter +Christian Clauss Christoph Heer Christos Stavrakakis CMGS @@ -103,6 +104,7 @@ Konstantin Kapustin kracekumar Kristian Glass Kristian Øllegaard +Krystian Krzysztof Urbaniak Kyle Kelley Kyle Mulka From 19fb762bd523cce7fd05cb4f72166eaf682b677a Mon Sep 17 00:00:00 2001 From: Brett Randall Date: Mon, 7 Feb 2022 13:24:50 +1100 Subject: [PATCH 056/164] Split main tox test build and linters into two workflows. --- .github/workflows/lint.yml | 29 +++++++++++++++++++++++++++++ .github/workflows/tox.yml | 27 --------------------------- 2 files changed, 29 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..e2a6225ea --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,29 @@ +name: lint +on: [push, pull_request] +jobs: + tox-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: pip install --upgrade pip + - run: pip install tox + - run: tox -e lint + + tox-docs-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: pip install --upgrade pip + - run: pip install tox + - run: tox -e docs-lint + + tox-pycodestyle: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: pip install --upgrade pip + - run: pip install tox + - run: tox -e pycodestyle diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index ca0cb26f5..f6199e20c 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -1,33 +1,6 @@ name: tox on: [push, pull_request] jobs: - tox-lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install --upgrade pip - - run: pip install tox - - run: tox -e lint - - tox-docs-lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install --upgrade pip - - run: pip install tox - - run: tox -e docs-lint - - tox-pycodestyle: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install --upgrade pip - - run: pip install tox - - run: tox -e pycodestyle - tox: strategy: fail-fast: false From 71d6388f0129f268a14c2cf587c74b966aaa9cce Mon Sep 17 00:00:00 2001 From: Brett Randall Date: Mon, 7 Feb 2022 13:40:12 +1100 Subject: [PATCH 057/164] Replaced Travis CI badge with 2x gh actions badges for test and lint --- README.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index a6a27b850..569f9db47 100644 --- a/README.rst +++ b/README.rst @@ -9,9 +9,13 @@ Gunicorn :alt: Supported Python versions :target: https://pypi.python.org/pypi/gunicorn -.. image:: https://travis-ci.org/benoitc/gunicorn.svg?branch=master +.. image:: https://github.com/benoitc/gunicorn/actions/workflows/tox.yml/badge.svg :alt: Build Status - :target: https://travis-ci.org/benoitc/gunicorn + :target: https://github.com/benoitc/gunicorn/actions/workflows/tox.yml + +.. image:: https://github.com/benoitc/gunicorn/actions/workflows/lint.yml/badge.svg + :alt: Lint Status + :target: https://github.com/benoitc/gunicorn/actions/workflows/lint.yml Gunicorn 'Green Unicorn' is a Python WSGI HTTP Server for UNIX. It's a pre-fork worker model ported from Ruby's Unicorn_ project. The Gunicorn server is broadly From 362a52bd8458e7cf1d89da60aaa9cb1da05d2784 Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Fri, 11 Feb 2022 19:26:56 +0200 Subject: [PATCH 058/164] Added parameter to ssl_context hook for constructing default context Signed-off-by: Tero Saarni --- examples/example_config.py | 30 ++++++++++++------------------ gunicorn/config.py | 11 +++++++---- gunicorn/sock.py | 6 +++--- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/examples/example_config.py b/examples/example_config.py index 83c3ffbcb..5a399a497 100644 --- a/examples/example_config.py +++ b/examples/example_config.py @@ -215,32 +215,26 @@ def worker_int(worker): def worker_abort(worker): worker.log.info("worker received SIGABRT signal") -def ssl_context(conf): +def ssl_context(conf, default_ssl_context_factory): import ssl - def set_defaults(context): - context.verify_mode = conf.cert_reqs - context.minimum_version = ssl.TLSVersion.TLSv1_3 - if conf.ciphers: - context.set_ciphers(conf.ciphers) - if conf.ca_certs: - context.load_verify_locations(cafile=conf.ca_certs) + # The default SSLContext returned by the factory function is initialized + # with the TLS parameters from config, including TLS certificates and other + # parameters. + context = default_ssl_context_factory() - # Return different server certificate depending which hostname the client - # uses. Requires Python 3.7 or later. + # The SSLContext can be further customized, for example by enforcing + # minimum TLS version. + context.minimum_version = ssl.TLSVersion.TLSv1_3 + + # Server can also return different server certificate depending which + # hostname the client uses. Requires Python 3.7 or later. def sni_callback(socket, server_hostname, context): if server_hostname == "foo.127.0.0.1.nip.io": - new_context = ssl.SSLContext() + new_context = default_ssl_context_factory() new_context.load_cert_chain(certfile="foo.pem", keyfile="foo-key.pem") - set_defaults(new_context) socket.context = new_context - context = ssl.SSLContext(conf.ssl_version) context.sni_callback = sni_callback - set_defaults(context) - - # Load fallback certificate that will be returned when there is no match - # or client did not set TLS SNI (server_hostname == None) - context.load_cert_chain(certfile=conf.certfile, keyfile=conf.keyfile) return context diff --git a/gunicorn/config.py b/gunicorn/config.py index 3f397d126..a92653747 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1970,11 +1970,11 @@ def on_exit(server): class NewSSLContext(Setting): name = "ssl_context" section = "Server Hooks" - validator = validate_callable(1) + validator = validate_callable(2) type = callable - def ssl_context(config): - return None + def ssl_context(config, default_ssl_context_factory): + return default_ssl_context_factory() default = staticmethod(ssl_context) desc = """\ @@ -1983,7 +1983,10 @@ def ssl_context(config): Allows fully customized SSL context to be used in place of the default context. - The callable needs to accept a single instance variable for the Config. + The callable needs to accept an instance variable for the Config and + a factory function that returns default SSLContext which is initialized + with certificates, private key, cert_reqs, and ciphers according to + config and can be further customized by the callable. The callable needs to return SSLContext object. """ diff --git a/gunicorn/sock.py b/gunicorn/sock.py index 1481c23bb..37631e6b1 100644 --- a/gunicorn/sock.py +++ b/gunicorn/sock.py @@ -212,8 +212,7 @@ def close_sockets(listeners, unlink=True): os.unlink(sock_name) def ssl_context(conf): - context = conf.ssl_context(conf) - if context is None: + def default_ssl_context_factory(): context = ssl.SSLContext(conf.ssl_version) context.load_cert_chain(certfile=conf.certfile, keyfile=conf.keyfile) context.verify_mode = conf.cert_reqs @@ -221,8 +220,9 @@ def ssl_context(conf): context.set_ciphers(conf.ciphers) if conf.ca_certs: context.load_verify_locations(cafile=conf.ca_certs) + return context - return context + return conf.ssl_context(conf, default_ssl_context_factory) def ssl_wrap_socket(sock, conf): return ssl_context(conf).wrap_socket(sock, server_side=True, From 99447b08271a5eb74485c88ce36e85f2c463a99d Mon Sep 17 00:00:00 2001 From: usr3 <50021155+usr3@users.noreply.github.com> Date: Sun, 13 Mar 2022 14:41:33 +0530 Subject: [PATCH 059/164] Fix typo in deploy.rst --- docs/source/deploy.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/deploy.rst b/docs/source/deploy.rst index 43bad9a80..809739f27 100644 --- a/docs/source/deploy.rst +++ b/docs/source/deploy.rst @@ -216,7 +216,7 @@ A tool that is starting to be common on linux systems is Systemd_. It is a system services manager that allows for strict process management, resources and permissions control. -Below are configurations files and instructions for using systemd to create +Below are configuration files and instructions for using systemd to create a unix socket for incoming Gunicorn requests. Systemd will listen on this socket and start gunicorn automatically in response to traffic. Later in this section are instructions for configuring Nginx to forward web traffic From cc3238c6ff5887919fdc9524f690275681970021 Mon Sep 17 00:00:00 2001 From: DwarfMaster Date: Sun, 13 Mar 2022 17:16:32 +0100 Subject: [PATCH 060/164] Prevent unnecessary setuid call --- gunicorn/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/util.py b/gunicorn/util.py index fb844c18a..c26811547 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -145,7 +145,7 @@ def set_owner_process(uid, gid, initgroups=False): elif gid != os.getgid(): os.setgid(gid) - if uid: + if uid and uid != os.getuid(): os.setuid(uid) From ac8bc3a4553e4954d69e8080b3c937dca409a69c Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Tue, 15 Mar 2022 19:05:30 +0200 Subject: [PATCH 061/164] Regenerated rst docs Signed-off-by: Tero Saarni --- docs/source/settings.rst | 95 ++++++++++++++++++++++++++++++++++------ gunicorn/config.py | 2 + 2 files changed, 84 insertions(+), 13 deletions(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index a5a27026f..90eed993a 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -310,8 +310,6 @@ file format. ``logconfig_dict`` ~~~~~~~~~~~~~~~~~~ -**Command line:** ``--log-config-dict`` - **Default:** ``{}`` The log config dictionary to use, using the standard Python @@ -332,7 +330,7 @@ For more context you can look at the default configuration dictionary for loggin **Command line:** ``--log-syslog-to SYSLOG_ADDR`` -**Default:** ``'unix:///var/run/syslog'`` +**Default:** ``'udp://localhost:514'`` Address to send syslog messages. @@ -921,19 +919,22 @@ The callable needs to accept a single instance variable for the Arbiter. ``ssl_context`` ~~~~~~~~~~~~~~~ -**Default:** +**Default:** .. code-block:: python - def ssl_context(config): - return None + def ssl_context(config, default_ssl_context_factory): + return default_ssl_context_factory() Called when SSLContext is needed. Allows fully customized SSL context to be used in place of the default context. -The callable needs to accept a single instance variable for the Config. +The callable needs to accept an instance variable for the Config and +a factory function that returns default SSLContext which is initialized +with certificates, private key, cert_reqs, and ciphers according to +config and can be further customized by the callable. The callable needs to return SSLContext object. Server Mechanics @@ -998,9 +999,7 @@ Set the ``SO_REUSEPORT`` flag on the listening socket. **Default:** ``'.'`` -Change directory to specified directory before loading apps. - -Default is the current directory. +Change directory to specified directory before loading apps. .. _daemon: @@ -1161,10 +1160,16 @@ temporary directory. **Default:** ``{'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-SSL': 'on'}`` A dictionary containing headers and values that the front-end proxy -uses to indicate HTTPS requests. These tell Gunicorn to set +uses to indicate HTTPS requests. If the source IP is permitted by +``forwarded-allow-ips`` (below), *and* at least one request header matches +a key-value pair listed in this dictionary, then Gunicorn will set ``wsgi.url_scheme`` to ``https``, so your application can tell that the request is secure. +If the other headers listed in this dictionary are not present in the request, they will be ignored, +but if the other headers are present and do not match the provided values, then +the request will fail to parse. See the note below for more detailed examples of this behaviour. + The dictionary should map upper-case header names to exact string values. The value comparisons are case-sensitive, unlike the header names, so make sure they're exactly what your front-end proxy sends @@ -1192,6 +1197,68 @@ you still trust the environment). By default, the value of the ``FORWARDED_ALLOW_IPS`` environment variable. If it is not defined, the default is ``"127.0.0.1"``. +.. note:: + + The interplay between the request headers, the value of ``forwarded_allow_ips``, and the value of + ``secure_scheme_headers`` is complex. Various scenarios are documented below to further elaborate. In each case, we + have a request from the remote address 134.213.44.18, and the default value of ``secure_scheme_headers``: + + .. code:: + + secure_scheme_headers = { + 'X-FORWARDED-PROTOCOL': 'ssl', + 'X-FORWARDED-PROTO': 'https', + 'X-FORWARDED-SSL': 'on' + } + + + .. list-table:: + :header-rows: 1 + :align: center + :widths: auto + + * - ``forwarded-allow-ips`` + - Secure Request Headers + - Result + - Explanation + * - .. code:: + + ["127.0.0.1"] + - .. code:: + + X-Forwarded-Proto: https + - .. code:: + + wsgi.url_scheme = "http" + - IP address was not allowed + * - .. code:: + + "*" + - + - .. code:: + + wsgi.url_scheme = "http" + - IP address allowed, but no secure headers provided + * - .. code:: + + "*" + - .. code:: + + X-Forwarded-Proto: https + - .. code:: + + wsgi.url_scheme = "https" + - IP address allowed, one request header matched + * - .. code:: + + ["134.213.44.18"] + - .. code:: + + X-Forwarded-Ssl: on + X-Forwarded-Proto: http + - ``InvalidSchemeHeaders()`` raised + - IP address allowed, but the two secure headers disagreed on if HTTPS was used + .. _pythonpath: ``pythonpath`` @@ -1364,8 +1431,9 @@ A positive integer generally in the ``2-4 x $(NUM_CORES)`` range. You'll want to vary this a bit to find the best for your particular application's work load. -By default, the value of the ``WEB_CONCURRENCY`` environment variable. -If it is not defined, the default is ``1``. +By default, the value of the ``WEB_CONCURRENCY`` environment variable, +which is set by some Platform-as-a-Service providers such as Heroku. If +it is not defined, the default is ``1``. .. _worker-class: @@ -1528,3 +1596,4 @@ set this to a higher value. .. note:: ``sync`` worker does not support persistent connections and will ignore this option. + diff --git a/gunicorn/config.py b/gunicorn/config.py index a92653747..392d687e6 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1509,6 +1509,8 @@ class LogConfigDict(Setting): Format: https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig + For more context you can look at the default configuration dictionary for logging, which can be found at ``gunicorn.glogging.CONFIG_DEFAULTS``. + .. versionadded:: 19.8 """ From 7d8f68c1a78a17dc4c53b157937957d7320161ee Mon Sep 17 00:00:00 2001 From: "Hazh. M. Adam" Date: Mon, 4 Apr 2022 12:27:55 +0800 Subject: [PATCH 062/164] Solving issue #2692 ValueError: count must be a positive integer (got 0) --- gunicorn/http/wsgi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 478677f4b..0fcff5a24 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -371,8 +371,8 @@ def sendfile(self, respiter): if self.is_chunked(): chunk_size = "%X\r\n" % nbytes self.sock.sendall(chunk_size.encode('utf-8')) - - self.sock.sendfile(respiter.filelike, count=nbytes) + if nbytes > 0: + self.sock.sendfile(respiter.filelike, count=nbytes) if self.is_chunked(): self.sock.sendall(b"\r\n") From f186fa3a27b55ff7b8517bade61aa453267c4643 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 22 Apr 2022 15:33:11 +0200 Subject: [PATCH 063/164] Make loggers function atomic Fixes https://github.com/benoitc/gunicorn/issues/2784: ``` Traceback (most recent call last): File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.7/site-packages/gunicorn/arbiter.py", line 589, in spawn_worker worker.init_process() File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.7/site-packages/uvicorn/workers.py", line 66, in init_process super(UvicornWorker, self).init_process() File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.7/site-packages/gunicorn/workers/base.py", line 116, in init_process self.log.close_on_exec() File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.7/site-packages/gunicorn/glogging.py", line 381, in close_on_exec for log in loggers(): File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.7/site-packages/gunicorn/glogging.py", line 94, in loggers return [logging.getLogger(name) for name in existing] File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.7/site-packages/gunicorn/glogging.py", line 94, in return [logging.getLogger(name) for name in existing] RuntimeError: dictionary changed size during iteration ``` --- gunicorn/glogging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index 6d80a3ac3..fec25a104 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -90,7 +90,7 @@ def loggers(): """ get list of all loggers """ root = logging.root - existing = root.manager.loggerDict.keys() + existing = list(root.manager.loggerDict.keys()) return [logging.getLogger(name) for name in existing] From 1754d33e4add5d3234f1f5b22363bd5868ae0dd4 Mon Sep 17 00:00:00 2001 From: pylipp Date: Tue, 24 May 2022 15:39:07 +0200 Subject: [PATCH 064/164] Add Python 3.10 classifier Upon pushing to PyPI, the badge on top of the readme should automatically update --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index fb220d90e..bffbb24e7 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', From 7ef281fa0388fb8ea2556a262f8e65dd2b69f20f Mon Sep 17 00:00:00 2001 From: David Moreau Simard Date: Sun, 29 May 2022 19:15:40 -0400 Subject: [PATCH 065/164] readme: IRC channel has moved to libera.chat There was a bit of drama with the freenode network and most channels have since migrated to libera.chat which is somewhat a spiritual successor. --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 569f9db47..fe9523b9a 100644 --- a/README.rst +++ b/README.rst @@ -22,7 +22,7 @@ worker model ported from Ruby's Unicorn_ project. The Gunicorn server is broadly compatible with various web frameworks, simply implemented, light on server resource usage, and fairly speedy. -Feel free to join us in `#gunicorn`_ on Freenode_. +Feel free to join us in `#gunicorn`_ on `Libera.chat`_. Documentation ------------- @@ -69,6 +69,6 @@ Gunicorn is released under the MIT License. See the LICENSE_ file for more details. .. _Unicorn: https://bogomips.org/unicorn/ -.. _`#gunicorn`: https://webchat.freenode.net/?channels=gunicorn -.. _Freenode: https://freenode.net/ +.. _`#gunicorn`: https://web.libera.chat/?channels=#gunicorn +.. _`Libera.chat`: https://libera.chat/ .. _LICENSE: https://github.com/benoitc/gunicorn/blob/master/LICENSE From 4dff334f18ea7e75ce106d129fe5f75362137154 Mon Sep 17 00:00:00 2001 From: Roffild Date: Tue, 5 Jul 2022 14:43:43 +0300 Subject: [PATCH 066/164] fix LevelLog in Arbiter.run() --- gunicorn/arbiter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 7ca2f6b79..0e7050759 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -230,8 +230,8 @@ def run(self): except SystemExit: raise except Exception: - self.log.info("Unhandled exception in main loop", - exc_info=True) + self.log.error("Unhandled exception in main loop", + exc_info=True) self.stop(False) if self.pidfile is not None: self.pidfile.unlink() From 69bf1d0cf44abffeeb469d4c36e1ff978b6e5769 Mon Sep 17 00:00:00 2001 From: focus zheng Date: Wed, 13 Jul 2022 15:44:30 +0800 Subject: [PATCH 067/164] remove changes by isort and lint --- gunicorn/http/message.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 17d22402b..6821eb7d7 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -31,6 +31,7 @@ def __init__(self, cfg, unreader, peer_addr): self.cfg = cfg self.unreader = unreader self.peer_addr = peer_addr + self.remote_addr = peer_addr self.version = None self.headers = [] self.trailers = [] From c9d01c801967596f66636cfc0c47660339d32631 Mon Sep 17 00:00:00 2001 From: focus zheng Date: Thu, 14 Jul 2022 10:19:18 +0800 Subject: [PATCH 068/164] get address from tuple --- gunicorn/http/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 6821eb7d7..0041cadc3 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -31,7 +31,7 @@ def __init__(self, cfg, unreader, peer_addr): self.cfg = cfg self.unreader = unreader self.peer_addr = peer_addr - self.remote_addr = peer_addr + self.remote_addr = peer_addr[0] self.version = None self.headers = [] self.trailers = [] From 9f159f7cb6653ea77f079dcdf7310c901273312f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Fri, 16 Sep 2022 15:15:19 -0400 Subject: [PATCH 069/164] docs: Add section for Gthread workers --- docs/source/design.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/source/design.rst b/docs/source/design.rst index 0d5454492..50f835b25 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -38,12 +38,6 @@ applications are programmed. closed after response has been sent (even if you manually add ``Keep-Alive`` or ``Connection: keep-alive`` header in your application). -The worker `gthread` is a threaded worker. It accepts connections in the -main loop, accepted connections are added to the thread pool as a -connection job. On keepalive connections are put back in the loop -waiting for an event. If no event happen after the keep alive timeout, -the connection is closed. - Async Workers ------------- @@ -59,6 +53,15 @@ installed and `setup `_. Other applications might not be compatible at all as they, e.g., rely on the original unpatched behavior. +Gthread Workers +--------------- + +The worker `gthread` is a threaded worker. It accepts connections in the +main loop. Accepted connections are added to the thread pool as a +connection job. On keepalive connections are put back in the loop +waiting for an event. If no event happens after the keepalive timeout, +the connection is closed. + Tornado Workers --------------- From 4cf30b79aaeade3a62cc1d774e24a1ab3a83d015 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 25 Sep 2022 16:31:58 +0200 Subject: [PATCH 070/164] build: harden tox.yml permissions Signed-off-by: Alex --- .github/workflows/tox.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index f6199e20c..455a6a075 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -1,5 +1,7 @@ name: tox on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) jobs: tox: strategy: From 137b77d37f358af5a0c58238a1a680b16812a6c9 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 25 Sep 2022 16:32:17 +0200 Subject: [PATCH 071/164] build: harden lint.yml permissions Signed-off-by: Alex --- .github/workflows/lint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e2a6225ea..8ec9391d2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,5 +1,7 @@ name: lint on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) jobs: tox-lint: runs-on: ubuntu-latest From 91fee380029c5e7d77d069252fbceb380d3cf7aa Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 29 Sep 2022 20:57:39 +0100 Subject: [PATCH 072/164] Remove Travis CI configuration --- .travis.yml | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2dd038725..000000000 --- a/.travis.yml +++ /dev/null @@ -1,34 +0,0 @@ -language: python -matrix: - include: - - python: 3.8 - env: TOXENV=lint - - python: 3.8 - env: TOXENV=docs-lint - - python: 3.5 - env: TOXENV=py35 - - python: 3.6 - env: TOXENV=py36 - - python: 3.7 - env: TOXENV=py37 - - python: 3.8 - env: TOXENV=py38 - - python: 3.9 - env: TOXENV=py39 - - python: pypy3 - env: - - TOXENV=pypy3 - # Embedded c-ares takes a long time to build and - # as-of 2020-01-04 there are no PyPy3 manylinux - # wheels for gevent on PyPI. - - GEVENTSETUP_EMBED_CARES=no - dist: xenial -install: pip install -U tox coverage -# TODO: https://github.com/tox-dev/tox/issues/149 -script: tox --recreate -after_success: - - if [ -f .coverage ]; then coverage report ; fi -cache: - directories: - - .tox - - $HOME/.cache/pip From 6766c086d5525bd44e5a4c488702787ee8fbfe70 Mon Sep 17 00:00:00 2001 From: Teko012 <112829523+Teko012@users.noreply.github.com> Date: Sat, 15 Oct 2022 14:21:56 +0200 Subject: [PATCH 073/164] Fix HTTP status code for Request Header Fields Too Large --- gunicorn/workers/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index a6d84bd23..a436326db 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -230,7 +230,9 @@ def handle_error(self, req, client, addr, exc): elif isinstance(exc, LimitRequestLine): mesg = "%s" % str(exc) elif isinstance(exc, LimitRequestHeaders): + reason = "Request Header Fields Too Large" mesg = "Error parsing headers: '%s'" % str(exc) + status_int = 431 elif isinstance(exc, InvalidProxyLine): mesg = "'%s'" % str(exc) elif isinstance(exc, ForbiddenProxyRequest): From b42af98e0ee75063329621c81bbf72ea8985787f Mon Sep 17 00:00:00 2001 From: Teko012 <112829523+Teko012@users.noreply.github.com> Date: Sat, 15 Oct 2022 14:28:21 +0200 Subject: [PATCH 074/164] Add contributor --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index 117134970..453da465d 100644 --- a/THANKS +++ b/THANKS @@ -168,6 +168,7 @@ Steven Cummings Sébastien Fievet Talha Malik TedWantsMore +Teko012 <112829523+Teko012@users.noreply.github.com> Thomas Grainger Thomas Steinacher Travis Cline From 008d81d0227cbc324f4e0814011e5d3cc4ec47e7 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sat, 3 Dec 2022 08:18:12 +0100 Subject: [PATCH 075/164] Update setup.py ensure domain email reflects the correct domain --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bffbb24e7..bfe5e45d5 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,7 @@ def run_tests(self): description='WSGI HTTP Server for UNIX', long_description=long_description, author='Benoit Chesneau', - author_email='benoitc@e-engura.com', + author_email='benoitc@gunicorn.org', license='MIT', url='https://gunicorn.org', project_urls={ From a41ee75c9edfb6ec164272004935bfda880acd2b Mon Sep 17 00:00:00 2001 From: Benny Mei Date: Sat, 10 Dec 2022 21:34:56 -0500 Subject: [PATCH 076/164] update irc channel docs --- docs/site/index.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/site/index.html b/docs/site/index.html index 31545d68d..17b95846b 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -120,9 +120,9 @@

    Project Management

    Project maintenance guidelines are available on the wiki

    -

    Irc

    -

    The Gunicorn channel is on the Freenode IRC - network. You can chat with the community on the #gunicorn channel.

    +

    IRC

    +

    The Gunicorn channel is on the Libera Chat IRC + network. You can chat with the community on the #gunicorn channel.

    Issue Tracking

    Bug reports, enhancement requests and tasks generally go in the Github From 61d643f9884477246e296a4dab6a28414c6af1b4 Mon Sep 17 00:00:00 2001 From: Benny Mei Date: Sat, 10 Dec 2022 21:40:12 -0500 Subject: [PATCH 077/164] add myself to THANKS file --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index 453da465d..b2cd8338f 100644 --- a/THANKS +++ b/THANKS @@ -26,6 +26,7 @@ Bartosz Oler Ben Cochran Ben Oswald Benjamin Gilbert +Benny Mei Benoit Chesneau Berker Peksag bninja From 1efc5d91fa3dd2fad54cb931a9f7473499de5165 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Wed, 21 Dec 2022 10:54:06 +0100 Subject: [PATCH 078/164] irc is on libera --- docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 50bb2abd8..24c4814d2 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,7 +7,7 @@ Gunicorn - WSGI server :Website: http://gunicorn.org :Source code: https://github.com/benoitc/gunicorn :Issue tracker: https://github.com/benoitc/gunicorn/issues -:IRC: ``#gunicorn`` on Freenode +:IRC: ``#gunicorn`` on Libera Chat :Usage questions: https://github.com/benoitc/gunicorn/issues Gunicorn 'Green Unicorn' is a Python WSGI HTTP Server for UNIX. It's a pre-fork From 2f9eb19b666868d324150072e7489788688c91cb Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Wed, 4 Jan 2023 11:42:56 +0100 Subject: [PATCH 079/164] prepare for release bump year in notice file. --- NOTICE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NOTICE b/NOTICE index 942a605d3..56e8b56f8 100644 --- a/NOTICE +++ b/NOTICE @@ -1,6 +1,6 @@ Gunicorn -2009-2018 (c) Benoît Chesneau +2009-2023 (c) Benoît Chesneau 2009-2015 (c) Paul J. Davis Gunicorn is released under the MIT license. See the LICENSE From 0ebb73aa240f0ecffe3e0922d54cfece19f5bfed Mon Sep 17 00:00:00 2001 From: JorisOnGithub Date: Mon, 16 Jan 2023 22:46:58 +0100 Subject: [PATCH 080/164] gthread: only read sockets when they are readable --- gunicorn/workers/gthread.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index d53181154..877c1d128 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -40,12 +40,15 @@ def __init__(self, cfg, sock, client, server): self.timeout = None self.parser = None + self.initialized = False # set the socket to non blocking self.sock.setblocking(False) def init(self): + self.initialized = True self.sock.setblocking(True) + if self.parser is None: # wrap the socket if needed if self.cfg.is_ssl: @@ -120,23 +123,28 @@ def accept(self, server, listener): # initialize the connection object conn = TConn(self.cfg, sock, client, server) self.nr_conns += 1 - # enqueue the job - self.enqueue_req(conn) + + # wait until socket is readable + with self._lock: + self.poller.register(conn.sock, selectors.EVENT_READ, + partial(self.on_client_socket_readable, conn)) except EnvironmentError as e: if e.errno not in (errno.EAGAIN, errno.ECONNABORTED, errno.EWOULDBLOCK): raise - def reuse_connection(self, conn, client): + def on_client_socket_readable(self, conn, client): with self._lock: # unregister the client from the poller self.poller.unregister(client) - # remove the connection from keepalive - try: - self._keep.remove(conn) - except ValueError: - # race condition - return + + if conn.initialized: + # remove the connection from keepalive + try: + self._keep.remove(conn) + except ValueError: + # race condition + return # submit the connection to a worker self.enqueue_req(conn) @@ -249,7 +257,7 @@ def finish_request(self, fs): # add the socket to the event loop self.poller.register(conn.sock, selectors.EVENT_READ, - partial(self.reuse_connection, conn)) + partial(self.on_client_socket_readable, conn)) else: self.nr_conns -= 1 conn.close() From 0b2c6fd99ebfac0eb3e4baaa6def107845cd5fe5 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Tue, 17 Jan 2023 20:22:06 +0100 Subject: [PATCH 081/164] bump uses --- .github/workflows/tox.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 455a6a075..07fc612ec 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -11,8 +11,8 @@ jobs: python: ['3.10'] # ['3.6', '3.7', '3.8', '3.9', '3.10', 'pypy-3.7'] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - run: pip install --upgrade pip From 48eda22a4b3399b0a65677619b4938503cc207cb Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Tue, 17 Jan 2023 20:45:47 +0100 Subject: [PATCH 082/164] Update tox.yml --- .github/workflows/tox.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 07fc612ec..8edaaa6ae 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -17,4 +17,5 @@ jobs: python-version: ${{ matrix.python }} - run: pip install --upgrade pip - run: pip install tox + - run: pip install -e . - run: tox -e py From 30baeebf337850a7ce3cb8099ffa248cef5831ef Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 17 Jan 2023 21:53:23 +0100 Subject: [PATCH 083/164] Revert "get address from tuple" This reverts commit c9d01c801967596f66636cfc0c47660339d32631. --- gunicorn/http/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 0041cadc3..6821eb7d7 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -31,7 +31,7 @@ def __init__(self, cfg, unreader, peer_addr): self.cfg = cfg self.unreader = unreader self.peer_addr = peer_addr - self.remote_addr = peer_addr[0] + self.remote_addr = peer_addr self.version = None self.headers = [] self.trailers = [] From f5d73aa73d53ef7838c128b7c3a582c536670c5e Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Sun, 22 Jan 2023 13:02:59 -0500 Subject: [PATCH 084/164] GA workflows cleanup Working Matrix for both lint/tox Fixing tox v4 issues Adjust pylint to working version --- .github/workflows/lint.yml | 41 ++++++++++++++++---------------------- .github/workflows/tox.yml | 21 ++++++++++--------- tox.ini | 11 +++++----- 3 files changed, 35 insertions(+), 38 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8ec9391d2..d66a54586 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,29 +3,22 @@ on: [push, pull_request] permissions: contents: read # to fetch code (actions/checkout) jobs: - tox-lint: + lint: + name: tox-${{ matrix.toxenv }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + toxenv: [lint, docs-lint, pycodestyle] + python-version: [ "3.10" ] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install --upgrade pip - - run: pip install tox - - run: tox -e lint - - tox-docs-lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install --upgrade pip - - run: pip install tox - - run: tox -e docs-lint - - tox-pycodestyle: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install --upgrade pip - - run: pip install tox - - run: tox -e pycodestyle + - uses: actions/checkout@v3 + - name: Using Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox + - run: tox -e ${{ matrix.toxenv }} diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 8edaaa6ae..0dc9bc729 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -4,18 +4,21 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: tox: + name: ${{ matrix.os }} / ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false - matrix: # All OSes pass except Windows because tests need Unix-only fcntl, grp, pwd, etc. - os: [ubuntu-latest] # [macos-latest, ubuntu-latest, windows-latest] - python: ['3.10'] # ['3.6', '3.7', '3.8', '3.9', '3.10', 'pypy-3.7'] - runs-on: ${{ matrix.os }} + matrix: + os: [ubuntu-latest, macos-latest] # All OSes pass except Windows because tests need Unix-only fcntl, grp, pwd, etc. + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.7", "pypy-3.8" ] steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - name: Using Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python }} - - run: pip install --upgrade pip - - run: pip install tox - - run: pip install -e . + python-version: ${{ matrix.python-version }} + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox - run: tox -e py diff --git a/tox.ini b/tox.ini index 196bac39d..64f58d662 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,10 @@ [tox] -envlist = py35, py36, py37, py38, py39, py310, pypy3, lint, docs-lint, pycodestyle -skipsdist = True +envlist = py{37,38,39,310,311,py3}, lint, docs-lint, pycodestyle +skipsdist = false +; Can't set skipsdist and use_develop in tox v4 to true due to https://github.com/tox-dev/tox/issues/2730 [testenv] -usedevelop = True +use_develop = true commands = pytest --cov=gunicorn {posargs} deps = -rrequirements_test.txt @@ -25,10 +26,10 @@ commands = tests/test_util.py \ tests/test_valid_requests.py deps = - pylint + pylint<2.7 [testenv:docs-lint] -whitelist_externals = +allowlist_externals = rst-lint bash grep From 2ea4699fe743299080659ede4436417e84398cd7 Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Sun, 22 Jan 2023 21:20:11 -0500 Subject: [PATCH 085/164] Fixing errors reported by pycodestyle --- gunicorn/arbiter.py | 10 ++--- gunicorn/config.py | 63 +++++++++++++++-------------- gunicorn/debug.py | 2 +- gunicorn/glogging.py | 75 +++++++++++++++++------------------ gunicorn/http/message.py | 8 ++-- gunicorn/sock.py | 2 +- gunicorn/systemd.py | 3 +- gunicorn/util.py | 2 +- gunicorn/workers/geventlet.py | 1 - gunicorn/workers/ggevent.py | 2 +- gunicorn/workers/gtornado.py | 2 +- 11 files changed, 86 insertions(+), 84 deletions(-) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 0e7050759..7b9ed76f5 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -154,7 +154,7 @@ def start(self): self.LISTENERS = sock.create_sockets(self.cfg, self.log, fds) - listeners_str = ",".join([str(l) for l in self.LISTENERS]) + listeners_str = ",".join([str(lnr) for lnr in self.LISTENERS]) self.log.debug("Arbiter booted") self.log.info("Listening at: %s (%s)", listeners_str, self.pid) self.log.info("Using worker: %s", self.cfg.worker_class_str) @@ -421,7 +421,7 @@ def reexec(self): environ['LISTEN_FDS'] = str(len(self.LISTENERS)) else: environ['GUNICORN_FD'] = ','.join( - str(l.fileno()) for l in self.LISTENERS) + str(lnr.fileno()) for lnr in self.LISTENERS) os.chdir(self.START_CTX['cwd']) @@ -454,11 +454,11 @@ def reload(self): # do we need to change listener ? if old_address != self.cfg.address: # close all listeners - for l in self.LISTENERS: - l.close() + for lnr in self.LISTENERS: + lnr.close() # init new listeners self.LISTENERS = sock.create_sockets(self.cfg, self.log) - listeners_str = ",".join([str(l) for l in self.LISTENERS]) + listeners_str = ",".join([str(lnr) for lnr in self.LISTENERS]) self.log.info("Listening at: %s", listeners_str) # do some actions on reload diff --git a/gunicorn/config.py b/gunicorn/config.py index bc24b7000..2d7d8be6d 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -448,7 +448,7 @@ def _validate_callable(val): raise TypeError(str(e)) except AttributeError: raise TypeError("Can not load '%s' from '%s'" - "" % (obj_name, mod_name)) + "" % (obj_name, mod_name)) if not callable(val): raise TypeError("Value is not callable: %s" % val) if arity != -1 and arity != util.get_arity(val): @@ -563,6 +563,7 @@ class ConfigFile(Setting): prefix. """ + class WSGIApp(Setting): name = "wsgi_app" section = "Config File" @@ -575,6 +576,7 @@ class WSGIApp(Setting): .. versionadded:: 20.1.0 """ + class Bind(Setting): name = "bind" action = "append" @@ -1273,69 +1275,70 @@ class ForwardedAllowIPS(Setting): By default, the value of the ``FORWARDED_ALLOW_IPS`` environment variable. If it is not defined, the default is ``"127.0.0.1"``. - + .. note:: - + The interplay between the request headers, the value of ``forwarded_allow_ips``, and the value of - ``secure_scheme_headers`` is complex. Various scenarios are documented below to further elaborate. In each case, we - have a request from the remote address 134.213.44.18, and the default value of ``secure_scheme_headers``: - + ``secure_scheme_headers`` is complex. Various scenarios are documented below to further elaborate. + In each case, we have a request from the remote address 134.213.44.18, and the default value of + ``secure_scheme_headers``: + .. code:: - + secure_scheme_headers = { 'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-SSL': 'on' } - - - .. list-table:: + + + .. list-table:: :header-rows: 1 :align: center :widths: auto - + * - ``forwarded-allow-ips`` - Secure Request Headers - Result - Explanation - * - .. code:: - + * - .. code:: + ["127.0.0.1"] - .. code:: - + X-Forwarded-Proto: https - - .. code:: - + - .. code:: + wsgi.url_scheme = "http" - IP address was not allowed - * - .. code:: - + * - .. code:: + "*" - - - .. code:: - + - .. code:: + wsgi.url_scheme = "http" - IP address allowed, but no secure headers provided - * - .. code:: - + * - .. code:: + "*" - .. code:: - + X-Forwarded-Proto: https - - .. code:: - + - .. code:: + wsgi.url_scheme = "https" - IP address allowed, one request header matched - * - .. code:: - + * - .. code:: + ["134.213.44.18"] - .. code:: - + X-Forwarded-Ssl: on X-Forwarded-Proto: http - ``InvalidSchemeHeaders()`` raised - IP address allowed, but the two secure headers disagreed on if HTTPS was used - + """ @@ -1617,6 +1620,7 @@ class StatsdHost(Setting): .. versionadded:: 19.1 """ + # Datadog Statsd (dogstatsd) tags. https://docs.datadoghq.com/developers/dogstatsd/ class DogstatsdTags(Setting): name = "dogstatsd_tags" @@ -1632,6 +1636,7 @@ class DogstatsdTags(Setting): .. versionadded:: 20 """ + class StatsdPrefix(Setting): name = "statsd_prefix" section = "Logging" diff --git a/gunicorn/debug.py b/gunicorn/debug.py index 996fe1b4b..a492df9e4 100644 --- a/gunicorn/debug.py +++ b/gunicorn/debug.py @@ -28,7 +28,7 @@ def __call__(self, frame, event, arg): if '__file__' in frame.f_globals: filename = frame.f_globals['__file__'] if (filename.endswith('.pyc') or - filename.endswith('.pyo')): + filename.endswith('.pyo')): filename = filename[:-1] name = frame.f_globals['__name__'] line = linecache.getline(filename, lineno) diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index fec25a104..70d992707 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -44,46 +44,45 @@ "local7": 23 } - CONFIG_DEFAULTS = dict( - version=1, - disable_existing_loggers=False, - - root={"level": "INFO", "handlers": ["console"]}, - loggers={ - "gunicorn.error": { - "level": "INFO", - "handlers": ["error_console"], - "propagate": True, - "qualname": "gunicorn.error" - }, - - "gunicorn.access": { - "level": "INFO", - "handlers": ["console"], - "propagate": True, - "qualname": "gunicorn.access" - } + version=1, + disable_existing_loggers=False, + + root={"level": "INFO", "handlers": ["console"]}, + loggers={ + "gunicorn.error": { + "level": "INFO", + "handlers": ["error_console"], + "propagate": True, + "qualname": "gunicorn.error" }, - handlers={ - "console": { - "class": "logging.StreamHandler", - "formatter": "generic", - "stream": "ext://sys.stdout" - }, - "error_console": { - "class": "logging.StreamHandler", - "formatter": "generic", - "stream": "ext://sys.stderr" - }, + + "gunicorn.access": { + "level": "INFO", + "handlers": ["console"], + "propagate": True, + "qualname": "gunicorn.access" + } + }, + handlers={ + "console": { + "class": "logging.StreamHandler", + "formatter": "generic", + "stream": "ext://sys.stdout" }, - formatters={ - "generic": { - "format": "%(asctime)s [%(process)d] [%(levelname)s] %(message)s", - "datefmt": "[%Y-%m-%d %H:%M:%S %z]", - "class": "logging.Formatter" - } + "error_console": { + "class": "logging.StreamHandler", + "formatter": "generic", + "stream": "ext://sys.stderr" + }, + }, + formatters={ + "generic": { + "format": "%(asctime)s [%(process)d] [%(levelname)s] %(message)s", + "datefmt": "[%Y-%m-%d %H:%M:%S %z]", + "class": "logging.Formatter" } + } ) @@ -299,7 +298,7 @@ def atoms(self, resp, req, environ, request_time): 'a': environ.get('HTTP_USER_AGENT', '-'), 'T': request_time.seconds, 'D': (request_time.seconds * 1000000) + request_time.microseconds, - 'M': (request_time.seconds * 1000) + int(request_time.microseconds/1000), + 'M': (request_time.seconds * 1000) + int(request_time.microseconds / 1000), 'L': "%d.%06d" % (request_time.seconds, request_time.microseconds), 'p': "<%s>" % os.getpid() } @@ -437,7 +436,7 @@ def _set_syslog_handler(self, log, cfg, fmt, name): # finally setup the syslog handler h = logging.handlers.SysLogHandler(address=addr, - facility=facility, socktype=socktype) + facility=facility, socktype=socktype) h.setFormatter(fmt) h._gunicorn = True diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 0041cadc3..b600ceabf 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -41,7 +41,7 @@ def __init__(self, cfg, unreader, peer_addr): # set headers limits self.limit_request_fields = cfg.limit_request_fields if (self.limit_request_fields <= 0 - or self.limit_request_fields > MAX_HEADERS): + or self.limit_request_fields > MAX_HEADERS): self.limit_request_fields = MAX_HEADERS self.limit_request_field_size = cfg.limit_request_field_size if self.limit_request_field_size < 0: @@ -71,7 +71,7 @@ def parse_headers(self, data): secure_scheme_headers = {} if ('*' in cfg.forwarded_allow_ips or not isinstance(self.peer_addr, tuple) - or self.peer_addr[0] in cfg.forwarded_allow_ips): + or self.peer_addr[0] in cfg.forwarded_allow_ips): secure_scheme_headers = cfg.secure_scheme_headers # Parse headers into key/value pairs paying attention @@ -173,7 +173,7 @@ def __init__(self, cfg, unreader, peer_addr, req_number=1): # get max request line size self.limit_request_line = cfg.limit_request_line if (self.limit_request_line < 0 - or self.limit_request_line >= MAX_REQUEST_LINE): + or self.limit_request_line >= MAX_REQUEST_LINE): self.limit_request_line = MAX_REQUEST_LINE self.req_number = req_number @@ -276,7 +276,7 @@ def proxy_protocol_access_check(self): # check in allow list if ("*" not in self.cfg.proxy_allow_ips and isinstance(self.peer_addr, tuple) and - self.peer_addr[0] not in self.cfg.proxy_allow_ips): + self.peer_addr[0] not in self.cfg.proxy_allow_ips): raise ForbiddenProxyRequest(self.peer_addr[0]) def parse_proxy_protocol(self, line): diff --git a/gunicorn/sock.py b/gunicorn/sock.py index d45867709..da5705d16 100644 --- a/gunicorn/sock.py +++ b/gunicorn/sock.py @@ -39,7 +39,7 @@ def __getattr__(self, name): def set_options(self, sock, bound=False): sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if (self.conf.reuse_port - and hasattr(socket, 'SO_REUSEPORT')): # pragma: no cover + and hasattr(socket, 'SO_REUSEPORT')): # pragma: no cover try: sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) except socket.error as err: diff --git a/gunicorn/systemd.py b/gunicorn/systemd.py index cea482204..5bc1a7449 100644 --- a/gunicorn/systemd.py +++ b/gunicorn/systemd.py @@ -58,7 +58,6 @@ def sd_notify(state, logger, unset_environment=False): child processes. """ - addr = os.environ.get('NOTIFY_SOCKET') if addr is None: # not run in a service, just a noop @@ -69,7 +68,7 @@ def sd_notify(state, logger, unset_environment=False): addr = '\0' + addr[1:] sock.connect(addr) sock.sendall(state.encode('utf-8')) - except: + except Exception: logger.debug("Exception while invoking sd_notify()", exc_info=True) finally: if unset_environment: diff --git a/gunicorn/util.py b/gunicorn/util.py index c26811547..b7cdd4e26 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -97,7 +97,7 @@ def load_class(uri, default="gunicorn.workers.sync.SyncWorker", try: mod = importlib.import_module('.'.join(components)) - except: + except Exception: exc = traceback.format_exc() msg = "class uri %r invalid or not found: \n\n[%s]" raise RuntimeError(msg % (uri, exc)) diff --git a/gunicorn/workers/geventlet.py b/gunicorn/workers/geventlet.py index ea82f3d62..4daf6d1ca 100644 --- a/gunicorn/workers/geventlet.py +++ b/gunicorn/workers/geventlet.py @@ -66,7 +66,6 @@ def _eventlet_socket_sendfile(self, file, offset=0, count=None): file.seek(offset + total_sent) - def _eventlet_serve(sock, handle, concurrency): """ Serve requests forever. diff --git a/gunicorn/workers/ggevent.py b/gunicorn/workers/ggevent.py index 3941814f5..0a844db3a 100644 --- a/gunicorn/workers/ggevent.py +++ b/gunicorn/workers/ggevent.py @@ -41,7 +41,7 @@ def patch(self): sockets = [] for s in self.sockets: sockets.append(socket.socket(s.FAMILY, socket.SOCK_STREAM, - fileno=s.sock.fileno())) + fileno=s.sock.fileno())) self.sockets = sockets def notify(self): diff --git a/gunicorn/workers/gtornado.py b/gunicorn/workers/gtornado.py index 9dd3d7bc4..2383c0d39 100644 --- a/gunicorn/workers/gtornado.py +++ b/gunicorn/workers/gtornado.py @@ -108,7 +108,7 @@ def run(self): if tornado.version_info[0] < 6: if not isinstance(app, tornado.web.Application) or \ - isinstance(app, tornado.wsgi.WSGIApplication): + isinstance(app, tornado.wsgi.WSGIApplication): app = WSGIContainer(app) elif not isinstance(app, WSGIContainer): app = WSGIContainer(app) From 4d532c960b433641b206da76437c17ebd6424cc0 Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Thu, 26 Jan 2023 17:34:09 -0500 Subject: [PATCH 086/164] Update THANKS --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index b2cd8338f..524ab48b6 100644 --- a/THANKS +++ b/THANKS @@ -157,6 +157,7 @@ Rik Ronan Amicel Ryan Peck Saeed Gharedaghi +Samuel Matos Sergey Rublev Shane Reustle shouse-cars From d04ddfd117c3f9741c68ace4f50c3cb3f61e29ce Mon Sep 17 00:00:00 2001 From: Klaas van Schelven Date: Wed, 8 Feb 2023 12:04:53 +0100 Subject: [PATCH 087/164] Docs: add link to configuration file See #2136 (not a full solution, but at least one more path to understanding added) --- docs/source/configure.rst | 1 + gunicorn/config.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/configure.rst b/docs/source/configure.rst index 0a8e7fe7e..dc9ba62da 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -65,6 +65,7 @@ usual:: There is also a ``--version`` flag available to the command line scripts that isn't mentioned in the list of :ref:`settings `. +.. _configuration_file: Configuration File ================== diff --git a/gunicorn/config.py b/gunicorn/config.py index 2d7d8be6d..f8938c7af 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -548,7 +548,7 @@ class ConfigFile(Setting): validator = validate_string default = "./gunicorn.conf.py" desc = """\ - The Gunicorn config file. + :ref:`The Gunicorn config file`. A string of the form ``PATH``, ``file:PATH``, or ``python:MODULE_NAME``. From 72faa11e27ffb82fe84c5438e0c40c0d64120bfd Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Wed, 22 Mar 2023 21:53:05 +0200 Subject: [PATCH 088/164] Replace pkg_resources.parse_version with packaging.version.parse This is a step towards removing the dependency on pkg_resources, which is part of setuptools, and thus makes setuptools a runtime dependency. --- gunicorn/workers/geventlet.py | 2 +- gunicorn/workers/ggevent.py | 2 +- setup.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/gunicorn/workers/geventlet.py b/gunicorn/workers/geventlet.py index 4daf6d1ca..e80867462 100644 --- a/gunicorn/workers/geventlet.py +++ b/gunicorn/workers/geventlet.py @@ -11,7 +11,7 @@ except ImportError: raise RuntimeError("eventlet worker requires eventlet 0.24.1 or higher") else: - from pkg_resources import parse_version + from packaging.version import parse as parse_version if parse_version(eventlet.__version__) < parse_version('0.24.1'): raise RuntimeError("eventlet worker requires eventlet 0.24.1 or higher") diff --git a/gunicorn/workers/ggevent.py b/gunicorn/workers/ggevent.py index 0a844db3a..fa38fdbd3 100644 --- a/gunicorn/workers/ggevent.py +++ b/gunicorn/workers/ggevent.py @@ -14,7 +14,7 @@ except ImportError: raise RuntimeError("gevent worker requires gevent 1.4 or higher") else: - from pkg_resources import parse_version + from packaging.version import parse as parse_version if parse_version(gevent.__version__) < parse_version('1.4'): raise RuntimeError("gevent worker requires gevent 1.4 or higher") diff --git a/setup.py b/setup.py index bfe5e45d5..f2fd73c44 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ def run_tests(self): # is the first version to support Python 3.4 which we require as a # floor. 'setuptools>=3.0', + 'packaging', ] extras_require = { From b8d6b1e97cb318ce16cf7ed0db471fbb01481202 Mon Sep 17 00:00:00 2001 From: Tal Einat <532281+taleinat@users.noreply.github.com> Date: Wed, 22 Mar 2023 21:56:52 +0200 Subject: [PATCH 089/164] Add myself to THANKS --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index 524ab48b6..552675f00 100644 --- a/THANKS +++ b/THANKS @@ -168,6 +168,7 @@ Stephen DiCato Stephen Holsapple Steven Cummings Sébastien Fievet +Tal Einat <532281+taleinat@users.noreply.github.com> Talha Malik TedWantsMore Teko012 <112829523+Teko012@users.noreply.github.com> From 98653df9da1acdbf70bbccd6d1ac32eca9ac158c Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sun, 7 May 2023 14:37:53 +0200 Subject: [PATCH 090/164] Update setup.py let's show it supports 3.11 . close #2941 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index bfe5e45d5..c12c9fde4 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', From e7781d2c2c669a87532ec6dd5d04490085ef7559 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sun, 7 May 2023 18:02:49 +0200 Subject: [PATCH 091/164] gthread: consider intial connectino as idle this change consider the connection is iddle when started and remove it from the eventloop if no change appears in the interval. This prevents the connection queued for too much time. --- gunicorn/workers/gthread.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index 877c1d128..a6bceb323 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -122,10 +122,13 @@ def accept(self, server, listener): sock, client = listener.accept() # initialize the connection object conn = TConn(self.cfg, sock, client, server) + # set timeout to ensure it will not be in the loop too long + conn.set_timeout() + self.nr_conns += 1 - # wait until socket is readable with self._lock: + self._keep.append(conn) self.poller.register(conn.sock, selectors.EVENT_READ, partial(self.on_client_socket_readable, conn)) except EnvironmentError as e: From f19e31fec5c679686f528fa77703fcb5a828ab5f Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sun, 7 May 2023 19:46:02 +0200 Subject: [PATCH 092/164] utils.py: fix code reference fix #2632 --- gunicorn/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/util.py b/gunicorn/util.py index b7cdd4e26..e10da0fe4 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -460,7 +460,7 @@ def is_hoppish(header): def daemonize(enable_stdio_inheritance=False): """\ Standard daemonization of a process. - http://www.svbug.com/documentation/comp.unix.programmer-FAQ/faq_2.html#SEC16 + http://www.faqs.org/faqs/unix-faq/programmer/faq/ section 1.7 """ if 'GUNICORN_FD' not in os.environ: if os.fork(): From a7ead9831e3c43826ce40e01ba4e8c36b9e12644 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Sun, 7 May 2023 21:05:41 +0200 Subject: [PATCH 093/164] fix basic auth logging decoding of basic auth header. fix #2625 --- gunicorn/glogging.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index e614294d3..60745f986 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -469,8 +469,7 @@ def _get_user(self, environ): # so we need to convert it to a byte string auth = base64.b64decode(auth[1].strip().encode('utf-8')) # b64decode returns a byte string - auth = auth.decode('utf-8') - auth = auth.split(":", 1) + auth.split(b":", 1)[0].decode(UTF-8", "replace") except (TypeError, binascii.Error, UnicodeDecodeError) as exc: self.debug("Couldn't get username: %s", exc) return user From 0b5c4aea99f56427116d1092e92b6ee0098dd507 Mon Sep 17 00:00:00 2001 From: Florian Demmer Date: Fri, 20 May 2022 11:37:53 +0200 Subject: [PATCH 094/164] Fix LogRecord pre-formatting, closes #2801 --- docs/source/settings.rst | 2 +- gunicorn/config.py | 2 +- gunicorn/workers/ggevent.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index a322b6152..9fef88ec5 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -836,7 +836,7 @@ The callable needs to accept a single instance variable for the Arbiter. .. code-block:: python def pre_request(worker, req): - worker.log.debug("%s %s" % (req.method, req.path)) + worker.log.debug("%s %s", req.method, req.path) Called just before a worker processes the request. diff --git a/gunicorn/config.py b/gunicorn/config.py index a41bd21c0..64a125068 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1906,7 +1906,7 @@ class PreRequest(Setting): type = callable def pre_request(worker, req): - worker.log.debug("%s %s" % (req.method, req.path)) + worker.log.debug("%s %s", req.method, req.path) default = staticmethod(pre_request) desc = """\ Called just before a worker processes the request. diff --git a/gunicorn/workers/ggevent.py b/gunicorn/workers/ggevent.py index 0a844db3a..5c41f14ec 100644 --- a/gunicorn/workers/ggevent.py +++ b/gunicorn/workers/ggevent.py @@ -110,7 +110,7 @@ def run(self): gevent.sleep(1.0) # Force kill all active the handlers - self.log.warning("Worker graceful timeout (pid:%s)" % self.pid) + self.log.warning("Worker graceful timeout (pid:%s)", self.pid) for server in servers: server.stop(timeout=1) except Exception: From ed9729a8262f03288373f4138e8d03d338809653 Mon Sep 17 00:00:00 2001 From: Chandan CH Date: Mon, 8 May 2023 11:28:30 +0530 Subject: [PATCH 095/164] add missing double quote Opening double quote was missed during the recent hence, showing the below error: auth.split(b":", 1)[0].decode(UTF-8", "replace") SyntaxError: unterminated string literal (detected at line 472) --- gunicorn/glogging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index 60745f986..4e5fbbd5d 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -469,7 +469,7 @@ def _get_user(self, environ): # so we need to convert it to a byte string auth = base64.b64decode(auth[1].strip().encode('utf-8')) # b64decode returns a byte string - auth.split(b":", 1)[0].decode(UTF-8", "replace") + auth.split(b":", 1)[0].decode("UTF-8", "replace") except (TypeError, binascii.Error, UnicodeDecodeError) as exc: self.debug("Couldn't get username: %s", exc) return user From 2b98f5e7b7918155eb2b311b0d2ed938a34e4c9f Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Thu, 11 May 2023 16:32:14 +0300 Subject: [PATCH 096/164] Fix username parsing for basic auth --- gunicorn/glogging.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index 4e5fbbd5d..e5cda1967 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -469,10 +469,7 @@ def _get_user(self, environ): # so we need to convert it to a byte string auth = base64.b64decode(auth[1].strip().encode('utf-8')) # b64decode returns a byte string - auth.split(b":", 1)[0].decode("UTF-8", "replace") + user = auth.split(b":", 1)[0].decode("UTF-8") except (TypeError, binascii.Error, UnicodeDecodeError) as exc: self.debug("Couldn't get username: %s", exc) - return user - if len(auth) == 2: - user = auth[0] return user From d8c3b1490e0f0e065c09b8143e11005e5e48addb Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Thu, 11 May 2023 17:45:57 +0300 Subject: [PATCH 097/164] Deprecate ssl_version option This change defaults SSLContext to Python's ssl.create_default_context() and marks ssl_version option as deprecated. The option value will be ignored and warnign will be printed in stderr. The ssl_version option was depending on old method of setting TLS min/max version, which has not worked well anymore with modern Python versions. --- docs/gunicorn_ext.py | 4 +- docs/source/settings.rst | 97 ++++++++++++++++++++++++---------------- gunicorn/config.py | 71 ++++++++++++++--------------- gunicorn/sock.py | 4 +- tests/test_config.py | 35 --------------- tests/test_ssl.py | 8 ---- 6 files changed, 97 insertions(+), 122 deletions(-) diff --git a/docs/gunicorn_ext.py b/docs/gunicorn_ext.py index 5b4781479..4310162eb 100755 --- a/docs/gunicorn_ext.py +++ b/docs/gunicorn_ext.py @@ -48,7 +48,9 @@ def format_settings(app): def fmt_setting(s): - if callable(s.default): + if hasattr(s, "default_doc"): + val = s.default_doc + elif callable(s.default): val = inspect.getsource(s.default) val = "\n".join(" %s" % line for line in val.splitlines()) val = "\n\n.. code-block:: python\n\n" + val diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 81558ac7a..a23fb6923 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -32,7 +32,7 @@ Config File **Default:** ``'./gunicorn.conf.py'`` -The Gunicorn config file. +:ref:`The Gunicorn config file`. A string of the form ``PATH``, ``file:PATH``, or ``python:MODULE_NAME``. @@ -305,16 +305,6 @@ The log config file to use. Gunicorn uses the standard Python logging module's Configuration file format. -.. _logconfig-json: - -logiconfig_json -~~~~~~~~~ - -* ``--log-config-json FILE`` -* ``None`` - -The log config file written in JSON. - .. _logconfig-dict: ``logconfig_dict`` @@ -324,9 +314,9 @@ The log config file written in JSON. The log config dictionary to use, using the standard Python logging module's dictionary configuration format. This option -takes precedence over the :ref:`logconfig` and :ref:`logConfigJson` options, which uses the -older file configuration format and JSON respectively. - +takes precedence over the :ref:`logconfig` and :ref:`logConfigJson` options, +which uses the older file configuration format and JSON +respectively. Format: https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig @@ -334,6 +324,21 @@ For more context you can look at the default configuration dictionary for loggin .. versionadded:: 19.8 +.. _logconfig-json: + +``logconfig_json`` +~~~~~~~~~~~~~~~~~~ + +**Command line:** ``--log-config-json FILE`` + +**Default:** ``None`` + +The log config to read config from a JSON file + +Format: https://docs.python.org/3/library/logging.config.html#logging.config.jsonConfig + +.. versionadded:: 20.0 + .. _syslog-addr: ``syslog_addr`` @@ -519,7 +524,10 @@ SSL certificate file **Default:** ``<_SSLMethod.PROTOCOL_TLS: 2>`` -SSL version to use. +SSL version to use (see stdlib ssl module's). + +.. deprecated:: 20.2 + The option is deprecated and it is currently ignored. Use :ref:`ssl-context` instead. ============= ============ --ssl-version Description @@ -542,6 +550,9 @@ TLS_SERVER Auto-negotiate the highest protocol version like TLS, .. versionchanged:: 20.0 This setting now accepts string names based on ``ssl.PROTOCOL_`` constants. +.. versionchanged:: 20.0.1 + The default value has been changed from ``ssl.PROTOCOL_SSLv23`` to + ``ssl.PROTOCOL_TLS`` when Python >= 3.6 . .. _cert-reqs: @@ -554,15 +565,13 @@ TLS_SERVER Auto-negotiate the highest protocol version like TLS, Whether client certificate is required (see stdlib ssl module's) -**Options:** - -`--cert-reqs=0` --- no client veirifcation - -`--cert-reqs=1` --- ssl.CERT_OPTIONAL - -`--cert-reqs=2` --- ssl.CERT_REQUIRED - - +=========== =========================== +--cert-reqs Description +=========== =========================== +`0` no client veirifcation +`1` ssl.CERT_OPTIONAL +`2` ssl.CERT_REQUIRED +=========== =========================== .. _ca-certs: @@ -954,8 +963,7 @@ The callable needs to accept a single instance variable for the Arbiter. Called when SSLContext is needed. -Allows fully customized SSL context to be used in place of the default -context. +Allows customizing SSL context. The callable needs to accept an instance variable for the Config and a factory function that returns default SSLContext which is initialized @@ -963,6 +971,18 @@ with certificates, private key, cert_reqs, and ciphers according to config and can be further customized by the callable. The callable needs to return SSLContext object. +Following example shows a configuration file that sets the minimum TLS version to 1.3: + +.. code-block:: python + + def ssl_context(conf, default_ssl_context_factory): + import ssl + context = default_ssl_context_factory() + context.minimum_version = ssl.TLSVersion.TLSv1_3 + return context + +.. versionadded:: 20.2 + Server Mechanics ---------------- @@ -1107,7 +1127,7 @@ If not set, the default temporary directory will be used. **Command line:** ``-u USER`` or ``--user USER`` -**Default:** ``501`` +**Default:** ``os.geteuid()`` Switch worker processes to run as this user. @@ -1122,7 +1142,7 @@ change the worker process user. **Command line:** ``-g GROUP`` or ``--group GROUP`` -**Default:** ``20`` +**Default:** ``os.getegid()`` Switch worker process to run as this group. @@ -1226,8 +1246,9 @@ variable. If it is not defined, the default is ``"127.0.0.1"``. .. note:: The interplay between the request headers, the value of ``forwarded_allow_ips``, and the value of - ``secure_scheme_headers`` is complex. Various scenarios are documented below to further elaborate. In each case, we - have a request from the remote address 134.213.44.18, and the default value of ``secure_scheme_headers``: + ``secure_scheme_headers`` is complex. Various scenarios are documented below to further elaborate. + In each case, we have a request from the remote address 134.213.44.18, and the default value of + ``secure_scheme_headers``: .. code:: @@ -1238,7 +1259,7 @@ variable. If it is not defined, the default is ``"127.0.0.1"``. } - .. list-table:: + .. list-table:: :header-rows: 1 :align: center :widths: auto @@ -1247,35 +1268,35 @@ variable. If it is not defined, the default is ``"127.0.0.1"``. - Secure Request Headers - Result - Explanation - * - .. code:: + * - .. code:: ["127.0.0.1"] - .. code:: X-Forwarded-Proto: https - - .. code:: + - .. code:: wsgi.url_scheme = "http" - IP address was not allowed - * - .. code:: + * - .. code:: "*" - - - .. code:: + - .. code:: wsgi.url_scheme = "http" - IP address allowed, but no secure headers provided - * - .. code:: + * - .. code:: "*" - .. code:: X-Forwarded-Proto: https - - .. code:: + - .. code:: wsgi.url_scheme = "https" - IP address allowed, one request header matched - * - .. code:: + * - .. code:: ["134.213.44.18"] - .. code:: diff --git a/gunicorn/config.py b/gunicorn/config.py index 51d1ac07c..97c125882 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -364,25 +364,9 @@ def validate_pos_int(val): def validate_ssl_version(val): - ssl_versions = {} - for protocol in [p for p in dir(ssl) if p.startswith("PROTOCOL_")]: - ssl_versions[protocol[9:]] = getattr(ssl, protocol) - if val in ssl_versions: - # string matching PROTOCOL_... - return ssl_versions[val] - - try: - intval = validate_pos_int(val) - if intval in ssl_versions.values(): - # positive int matching a protocol int constant - return intval - except (ValueError, TypeError): - # negative integer or not an integer - # drop this in favour of the more descriptive ValueError below - pass - - raise ValueError("Invalid ssl_version: %s. Valid options: %s" - % (val, ', '.join(ssl_versions))) + if val != SSLVersion.default: + sys.stderr.write("Warning: option `ssl_version` is deprecated and it is ignored. Use ssl_context instead.\n") + return val def validate_string(val): @@ -736,7 +720,7 @@ class WorkerConnections(Setting): desc = """\ The maximum number of simultaneous clients. - This setting only affects the Eventlet and Gevent worker types. + This setting only affects the ``gthread``, ``eventlet`` and ``gevent`` worker types. """ @@ -1066,6 +1050,7 @@ class Chdir(Setting): cli = ["--chdir"] validator = validate_chdir default = util.getcwd() + default_doc = "``'.'``" desc = """\ Change directory to specified directory before loading apps. """ @@ -1157,6 +1142,7 @@ class User(Setting): meta = "USER" validator = validate_user default = os.geteuid() + default_doc = "``os.geteuid()``" desc = """\ Switch worker processes to run as this user. @@ -1173,6 +1159,7 @@ class Group(Setting): meta = "GROUP" validator = validate_group default = os.getegid() + default_doc = "``os.getegid()``" desc = """\ Switch worker process to run as this group. @@ -2019,14 +2006,25 @@ def ssl_context(config, default_ssl_context_factory): desc = """\ Called when SSLContext is needed. - Allows fully customized SSL context to be used in place of the default - context. + Allows customizing SSL context. The callable needs to accept an instance variable for the Config and a factory function that returns default SSLContext which is initialized with certificates, private key, cert_reqs, and ciphers according to config and can be further customized by the callable. The callable needs to return SSLContext object. + + Following example shows a configuration file that sets the minimum TLS version to 1.3: + + .. code-block:: python + + def ssl_context(conf, default_ssl_context_factory): + import ssl + context = default_ssl_context_factory() + context.minimum_version = ssl.TLSVersion.TLSv1_3 + return context + + .. versionadded:: 20.2 """ class ProxyProtocol(Setting): @@ -2105,17 +2103,12 @@ class SSLVersion(Setting): else: default = ssl.PROTOCOL_SSLv23 - desc = """\ - SSL version to use (see stdlib ssl module's) - - .. versionchanged:: 20.0.1 - The default value has been changed from ``ssl.PROTOCOL_SSLv23`` to - ``ssl.PROTOCOL_TLS`` when Python >= 3.6 . - - """ default = ssl.PROTOCOL_SSLv23 desc = """\ - SSL version to use. + SSL version to use (see stdlib ssl module's). + + .. deprecated:: 20.2 + The option is deprecated and it is currently ignored. Use :ref:`ssl-context` instead. ============= ============ --ssl-version Description @@ -2138,6 +2131,9 @@ class SSLVersion(Setting): .. versionchanged:: 20.0 This setting now accepts string names based on ``ssl.PROTOCOL_`` constants. + .. versionchanged:: 20.0.1 + The default value has been changed from ``ssl.PROTOCOL_SSLv23`` to + ``ssl.PROTOCOL_TLS`` when Python >= 3.6 . """ @@ -2149,13 +2145,14 @@ class CertReqs(Setting): default = ssl.CERT_NONE desc = """\ Whether client certificate is required (see stdlib ssl module's) - ============== =========================== - `--cert-reqs=0` --- no client veirifcation - - `--cert-reqs=1` --- ssl.CERT_OPTIONAL - `--cert-reqs=2` --- ssl.CERT_REQUIRED - ============== =========================== + =========== =========================== + --cert-reqs Description + =========== =========================== + `0` no client veirifcation + `1` ssl.CERT_OPTIONAL + `2` ssl.CERT_REQUIRED + =========== =========================== """ diff --git a/gunicorn/sock.py b/gunicorn/sock.py index 31b9919dc..22e25b33e 100644 --- a/gunicorn/sock.py +++ b/gunicorn/sock.py @@ -213,13 +213,11 @@ def close_sockets(listeners, unlink=True): def ssl_context(conf): def default_ssl_context_factory(): - context = ssl.SSLContext(conf.ssl_version) + context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=conf.ca_certs) context.load_cert_chain(certfile=conf.certfile, keyfile=conf.keyfile) context.verify_mode = conf.cert_reqs if conf.ciphers: context.set_ciphers(conf.ciphers) - if conf.ca_certs: - context.load_verify_locations(cafile=conf.ca_certs) return context return conf.ssl_context(conf, default_ssl_context_factory) diff --git a/tests/test_config.py b/tests/test_config.py index 211ee017e..c094f6a21 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -453,41 +453,6 @@ def test_umask_config(options, expected): assert app.cfg.umask == expected -@pytest.mark.parametrize("options, expected", [ - (["--ssl-version", "SSLv23"], 2), - (["--ssl-version", "TLSv1"], 3), - (["--ssl-version", "2"], 2), - (["--ssl-version", "3"], 3), -]) -def test_ssl_version_named_constants_python3(options, expected): - _test_ssl_version(options, expected) - - -@pytest.mark.skipif(sys.version_info < (3, 6), - reason="requires python3.6+") -@pytest.mark.parametrize("options, expected", [ - (["--ssl-version", "TLS"], 2), - (["--ssl-version", "TLSv1_1"], 4), - (["--ssl-version", "TLSv1_2"], 5), - (["--ssl-version", "TLS_SERVER"], 17), -]) -def test_ssl_version_named_constants_python36(options, expected): - _test_ssl_version(options, expected) - - -@pytest.mark.parametrize("ssl_version", [ - "FOO", - "-99", - "99991234" -]) -def test_ssl_version_bad(ssl_version): - c = config.Config() - with pytest.raises(ValueError) as exc: - c.set("ssl_version", ssl_version) - assert 'Valid options' in str(exc.value) - assert "TLSv" in str(exc.value) - - def _test_ssl_version(options, expected): cmdline = ["prog_name"] cmdline.extend(options) diff --git a/tests/test_ssl.py b/tests/test_ssl.py index 97e05d86b..7d15de173 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -32,14 +32,6 @@ def test_certfile(): assert CertFile.default is None -def test_ssl_version(): - assert issubclass(SSLVersion, Setting) - assert SSLVersion.name == 'ssl_version' - assert SSLVersion.section == 'SSL' - assert SSLVersion.cli == ['--ssl-version'] - assert SSLVersion.default == ssl.PROTOCOL_SSLv23 - - def test_cacerts(): assert issubclass(CACerts, Setting) assert CACerts.name == 'ca_certs' From 6d4faab6d6bfa85e4f9afba02d933a09beee8a7f Mon Sep 17 00:00:00 2001 From: Ilya Konstantinov Date: Thu, 30 Jul 2020 10:33:55 -0400 Subject: [PATCH 098/164] Log access even when connection is closed --- gunicorn/workers/base_async.py | 18 ++++++++++-------- gunicorn/workers/gthread.py | 20 +++++++++++--------- gunicorn/workers/sync.py | 18 ++++++++++-------- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/gunicorn/workers/base_async.py b/gunicorn/workers/base_async.py index 73c3f6c1b..0b7d65701 100644 --- a/gunicorn/workers/base_async.py +++ b/gunicorn/workers/base_async.py @@ -109,14 +109,16 @@ def handle_request(self, listener_name, req, sock, addr): if self.is_already_handled(respiter): return False try: - if isinstance(respiter, environ['wsgi.file_wrapper']): - resp.write_file(respiter) - else: - for item in respiter: - resp.write(item) - resp.close() - request_time = datetime.now() - request_start - self.log.access(resp, req, environ, request_time) + try: + if isinstance(respiter, environ['wsgi.file_wrapper']): + resp.write_file(respiter) + else: + for item in respiter: + resp.write(item) + resp.close() + finally: + request_time = datetime.now() - request_start + self.log.access(resp, req, environ, request_time) finally: if hasattr(respiter, "close"): respiter.close() diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index f0e5bbf5f..dab22fb1b 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -333,15 +333,17 @@ def handle_request(self, req, conn): respiter = self.wsgi(environ, resp.start_response) try: - if isinstance(respiter, environ['wsgi.file_wrapper']): - resp.write_file(respiter) - else: - for item in respiter: - resp.write(item) - - resp.close() - request_time = datetime.now() - request_start - self.log.access(resp, req, environ, request_time) + try: + if isinstance(respiter, environ['wsgi.file_wrapper']): + resp.write_file(respiter) + else: + for item in respiter: + resp.write(item) + + resp.close() + finally: + request_time = datetime.now() - request_start + self.log.access(resp, req, environ, request_time) finally: if hasattr(respiter, "close"): respiter.close() diff --git a/gunicorn/workers/sync.py b/gunicorn/workers/sync.py index 4bdf0003e..b6350e86d 100644 --- a/gunicorn/workers/sync.py +++ b/gunicorn/workers/sync.py @@ -177,14 +177,16 @@ def handle_request(self, listener, req, client, addr): self.alive = False respiter = self.wsgi(environ, resp.start_response) try: - if isinstance(respiter, environ['wsgi.file_wrapper']): - resp.write_file(respiter) - else: - for item in respiter: - resp.write(item) - resp.close() - request_time = datetime.now() - request_start - self.log.access(resp, req, environ, request_time) + try: + if isinstance(respiter, environ['wsgi.file_wrapper']): + resp.write_file(respiter) + else: + for item in respiter: + resp.write(item) + resp.close() + finally: + request_time = datetime.now() - request_start + self.log.access(resp, req, environ, request_time) finally: if hasattr(respiter, "close"): respiter.close() From 858b743d072c7a6cbbf3eff1fffef08e76212e0e Mon Sep 17 00:00:00 2001 From: Ilya Priven Date: Thu, 11 May 2023 15:04:45 -0400 Subject: [PATCH 099/164] remove double finally --- gunicorn/workers/base_async.py | 18 ++++++++---------- gunicorn/workers/gthread.py | 20 +++++++++----------- gunicorn/workers/sync.py | 18 ++++++++---------- 3 files changed, 25 insertions(+), 31 deletions(-) diff --git a/gunicorn/workers/base_async.py b/gunicorn/workers/base_async.py index 0b7d65701..ec5d76449 100644 --- a/gunicorn/workers/base_async.py +++ b/gunicorn/workers/base_async.py @@ -109,17 +109,15 @@ def handle_request(self, listener_name, req, sock, addr): if self.is_already_handled(respiter): return False try: - try: - if isinstance(respiter, environ['wsgi.file_wrapper']): - resp.write_file(respiter) - else: - for item in respiter: - resp.write(item) - resp.close() - finally: - request_time = datetime.now() - request_start - self.log.access(resp, req, environ, request_time) + if isinstance(respiter, environ['wsgi.file_wrapper']): + resp.write_file(respiter) + else: + for item in respiter: + resp.write(item) + resp.close() finally: + request_time = datetime.now() - request_start + self.log.access(resp, req, environ, request_time) if hasattr(respiter, "close"): respiter.close() if resp.should_close(): diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index dab22fb1b..9a6ad0066 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -333,18 +333,16 @@ def handle_request(self, req, conn): respiter = self.wsgi(environ, resp.start_response) try: - try: - if isinstance(respiter, environ['wsgi.file_wrapper']): - resp.write_file(respiter) - else: - for item in respiter: - resp.write(item) - - resp.close() - finally: - request_time = datetime.now() - request_start - self.log.access(resp, req, environ, request_time) + if isinstance(respiter, environ['wsgi.file_wrapper']): + resp.write_file(respiter) + else: + for item in respiter: + resp.write(item) + + resp.close() finally: + request_time = datetime.now() - request_start + self.log.access(resp, req, environ, request_time) if hasattr(respiter, "close"): respiter.close() diff --git a/gunicorn/workers/sync.py b/gunicorn/workers/sync.py index b6350e86d..d68d15cb6 100644 --- a/gunicorn/workers/sync.py +++ b/gunicorn/workers/sync.py @@ -177,17 +177,15 @@ def handle_request(self, listener, req, client, addr): self.alive = False respiter = self.wsgi(environ, resp.start_response) try: - try: - if isinstance(respiter, environ['wsgi.file_wrapper']): - resp.write_file(respiter) - else: - for item in respiter: - resp.write(item) - resp.close() - finally: - request_time = datetime.now() - request_start - self.log.access(resp, req, environ, request_time) + if isinstance(respiter, environ['wsgi.file_wrapper']): + resp.write_file(respiter) + else: + for item in respiter: + resp.write(item) + resp.close() finally: + request_time = datetime.now() - request_start + self.log.access(resp, req, environ, request_time) if hasattr(respiter, "close"): respiter.close() except EnvironmentError: From 7f480daf071d134664ba4b525b2b821f704e0136 Mon Sep 17 00:00:00 2001 From: Maxwell G Date: Tue, 4 Apr 2023 18:50:22 +0000 Subject: [PATCH 100/164] replace pkg_resources.load_entry_point pkg_resources is deprecated. Use the corresponding importlib.metadata interface instead. Use the stdlib version on python >= 3.8 and use the importlib_metadata backport on older versions. --- gunicorn/util.py | 18 +++++++++++++++--- setup.py | 6 +----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/gunicorn/util.py b/gunicorn/util.py index b7cdd4e26..a330d9f19 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -22,7 +22,10 @@ import traceback import warnings -import pkg_resources +try: + import importlib.metadata as importlib_metadata +except ImportError: + import importlib_metadata from gunicorn.errors import AppImportError from gunicorn.workers import SUPPORTED_WORKERS @@ -54,6 +57,15 @@ def _setproctitle(title): pass +def load_entry_point(distribution, group, name): + dist_obj = importlib_metadata.distribution(distribution) + eps = [ep for ep in dist_obj.entry_points + if ep.group == group and ep.name == name] + if not eps: + raise ImportError("Entry point %r not found" % ((group, name),)) + return eps[0].load() + + def load_class(uri, default="gunicorn.workers.sync.SyncWorker", section="gunicorn.workers"): if inspect.isclass(uri): @@ -68,7 +80,7 @@ def load_class(uri, default="gunicorn.workers.sync.SyncWorker", name = default try: - return pkg_resources.load_entry_point(dist, section, name) + return load_entry_point(dist, section, name) except Exception: exc = traceback.format_exc() msg = "class uri %r invalid or not found: \n\n[%s]" @@ -85,7 +97,7 @@ def load_class(uri, default="gunicorn.workers.sync.SyncWorker", break try: - return pkg_resources.load_entry_point( + return load_entry_point( "gunicorn", section, uri ) except Exception: diff --git a/setup.py b/setup.py index f2fd73c44..434bdc265 100644 --- a/setup.py +++ b/setup.py @@ -70,11 +70,7 @@ def run_tests(self): install_requires = [ - # We depend on functioning pkg_resources.working_set.add_entry() and - # pkg_resources.load_entry_point(). These both work as of 3.0 which - # is the first version to support Python 3.4 which we require as a - # floor. - 'setuptools>=3.0', + 'importlib_metadata; python_version<"3.8"', 'packaging', ] From 48d670f0874765e9d812bda1a63461aa6936f632 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 17 May 2023 18:45:59 +0300 Subject: [PATCH 101/164] update pylint version, and fix linter issues --- .pylintrc | 6 +++--- gunicorn/config.py | 5 ++++- gunicorn/glogging.py | 19 +++++++++---------- gunicorn/http/wsgi.py | 2 +- gunicorn/sock.py | 10 +++++++--- gunicorn/util.py | 6 +++--- gunicorn/workers/base_async.py | 8 ++++---- gunicorn/workers/geventlet.py | 1 + gunicorn/workers/ggevent.py | 2 +- gunicorn/workers/gthread.py | 4 ++-- gunicorn/workers/gtornado.py | 3 +-- gunicorn/workers/sync.py | 10 +++++----- tests/test_arbiter.py | 2 +- tests/test_http.py | 2 +- tests/test_pidfile.py | 2 +- tests/test_sock.py | 2 +- tests/test_ssl.py | 2 +- tests/test_systemd.py | 2 +- tox.ini | 3 ++- 19 files changed, 49 insertions(+), 42 deletions(-) diff --git a/.pylintrc b/.pylintrc index af99b3e1a..bc2046c0c 100644 --- a/.pylintrc +++ b/.pylintrc @@ -12,7 +12,6 @@ ignore= disable= attribute-defined-outside-init, - bad-continuation, bad-mcs-classmethod-argument, bare-except, broad-except, @@ -25,12 +24,10 @@ disable= import-self, inconsistent-return-statements, invalid-name, - misplaced-comparison-constant, missing-docstring, no-else-return, no-member, no-self-argument, - no-self-use, no-staticmethod-decorator, not-callable, protected-access, @@ -53,3 +50,6 @@ disable= useless-import-alias, comparison-with-callable, try-except-raise, + consider-using-with, + consider-using-f-string, + unspecified-encoding diff --git a/gunicorn/config.py b/gunicorn/config.py index 97c125882..84e7619e4 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1510,7 +1510,8 @@ class LogConfigDict(Setting): Format: https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig - For more context you can look at the default configuration dictionary for logging, which can be found at ``gunicorn.glogging.CONFIG_DEFAULTS``. + For more context you can look at the default configuration dictionary for logging, + which can be found at ``gunicorn.glogging.CONFIG_DEFAULTS``. .. versionadded:: 19.8 """ @@ -1993,6 +1994,7 @@ def on_exit(server): The callable needs to accept a single instance variable for the Arbiter. """ + class NewSSLContext(Setting): name = "ssl_context" section = "Server Hooks" @@ -2027,6 +2029,7 @@ def ssl_context(conf, default_ssl_context_factory): .. versionadded:: 20.2 """ + class ProxyProtocol(Setting): name = "proxy_protocol" section = "Server Mechanics" diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index e5cda1967..b552e26a8 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -45,12 +45,11 @@ "local7": 23 } -CONFIG_DEFAULTS = dict( - version=1, - disable_existing_loggers=False, - - root={"level": "INFO", "handlers": ["console"]}, - loggers={ +CONFIG_DEFAULTS = { + "version": 1, + "disable_existing_loggers": False, + "root": {"level": "INFO", "handlers": ["console"]}, + "loggers": { "gunicorn.error": { "level": "INFO", "handlers": ["error_console"], @@ -65,7 +64,7 @@ "qualname": "gunicorn.access" } }, - handlers={ + "handlers": { "console": { "class": "logging.StreamHandler", "formatter": "generic", @@ -77,14 +76,14 @@ "stream": "ext://sys.stderr" }, }, - formatters={ + "formatters": { "generic": { "format": "%(asctime)s [%(process)d] [%(levelname)s] %(message)s", "datefmt": "[%Y-%m-%d %H:%M:%S %z]", "class": "logging.Formatter" } } -) +} def loggers(): @@ -418,7 +417,7 @@ def _set_handler(self, log, output, fmt, stream=None): if output == "-": h = logging.StreamHandler(stream) else: - util.check_is_writeable(output) + util.check_is_writable(output) h = logging.FileHandler(output) # make sure the user can reopen the file try: diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 5d91706c1..25715eab7 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -12,7 +12,7 @@ from gunicorn.http.message import HEADER_RE from gunicorn.http.errors import InvalidHeader, InvalidHeaderName from gunicorn import SERVER_SOFTWARE, SERVER -import gunicorn.util as util +from gunicorn import util # Send files in at most 1GB blocks as some operating systems can have problems # with sending files in blocks over 2GB. diff --git a/gunicorn/sock.py b/gunicorn/sock.py index 22e25b33e..7700146a8 100644 --- a/gunicorn/sock.py +++ b/gunicorn/sock.py @@ -204,6 +204,7 @@ def create_sockets(conf, log, fds=None): return listeners + def close_sockets(listeners, unlink=True): for sock in listeners: sock_name = sock.getsockname() @@ -211,6 +212,7 @@ def close_sockets(listeners, unlink=True): if unlink and _sock_type(sock_name) is UnixSocket: os.unlink(sock_name) + def ssl_context(conf): def default_ssl_context_factory(): context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=conf.ca_certs) @@ -222,7 +224,9 @@ def default_ssl_context_factory(): return conf.ssl_context(conf, default_ssl_context_factory) + def ssl_wrap_socket(sock, conf): - return ssl_context(conf).wrap_socket(sock, server_side=True, - suppress_ragged_eofs=conf.suppress_ragged_eofs, - do_handshake_on_connect=conf.do_handshake_on_connect) + return ssl_context(conf).wrap_socket(sock, + server_side=True, + suppress_ragged_eofs=conf.suppress_ragged_eofs, + do_handshake_on_connect=conf.do_handshake_on_connect) diff --git a/gunicorn/util.py b/gunicorn/util.py index b4a2d6eee..1b39ba73c 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -562,12 +562,12 @@ def seed(): random.seed('%s.%s' % (time.time(), os.getpid())) -def check_is_writeable(path): +def check_is_writable(path): try: - f = open(path, 'a') + with open(path, 'a') as f: + f.close() except IOError as e: raise RuntimeError("Error: '%s' isn't writable [%r]" % (path, e)) - f.close() def to_bytestring(value, encoding="utf8"): diff --git a/gunicorn/workers/base_async.py b/gunicorn/workers/base_async.py index ec5d76449..b059a7cb5 100644 --- a/gunicorn/workers/base_async.py +++ b/gunicorn/workers/base_async.py @@ -9,10 +9,10 @@ import ssl import sys -import gunicorn.http as http -import gunicorn.http.wsgi as wsgi -import gunicorn.util as util -import gunicorn.workers.base as base +from gunicorn import http +from gunicorn.http import wsgi +from gunicorn import util +from gunicorn.workers import base ALREADY_HANDLED = object() diff --git a/gunicorn/workers/geventlet.py b/gunicorn/workers/geventlet.py index 64595b535..c42ed1186 100644 --- a/gunicorn/workers/geventlet.py +++ b/gunicorn/workers/geventlet.py @@ -173,6 +173,7 @@ def run(self): eventlet.sleep(1.0) self.notify() + t = None try: with eventlet.Timeout(self.cfg.graceful_timeout) as t: for a in acceptors: diff --git a/gunicorn/workers/ggevent.py b/gunicorn/workers/ggevent.py index 9ece3b600..2125a32d0 100644 --- a/gunicorn/workers/ggevent.py +++ b/gunicorn/workers/ggevent.py @@ -59,7 +59,7 @@ def run(self): ssl_args = {} if self.cfg.is_ssl: - ssl_args = dict(ssl_context=ssl_context(self.cfg)) + ssl_args = {"ssl_context": ssl_context(self.cfg)} for s in self.sockets: s.setblocking(1) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index 9a6ad0066..fb37fd3f3 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -11,7 +11,7 @@ # closed. # pylint: disable=no-else-break -import concurrent.futures as futures +from concurrent import futures import errno import os import selectors @@ -124,7 +124,7 @@ def accept(self, server, listener): conn = TConn(self.cfg, sock, client, server) # set timeout to ensure it will not be in the loop too long conn.set_timeout() - + self.nr_conns += 1 # wait until socket is readable with self._lock: diff --git a/gunicorn/workers/gtornado.py b/gunicorn/workers/gtornado.py index 9f5937530..285061196 100644 --- a/gunicorn/workers/gtornado.py +++ b/gunicorn/workers/gtornado.py @@ -3,7 +3,6 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -import copy import os import sys @@ -112,7 +111,7 @@ def run(self): isinstance(app, tornado.wsgi.WSGIApplication): app = WSGIContainer(app) elif not isinstance(app, WSGIContainer) and \ - not isinstance(app, tornado.web.Application): + not isinstance(app, tornado.web.Application): app = WSGIContainer(app) # Monkey-patching HTTPConnection.finish to count the diff --git a/gunicorn/workers/sync.py b/gunicorn/workers/sync.py index d68d15cb6..39a209f06 100644 --- a/gunicorn/workers/sync.py +++ b/gunicorn/workers/sync.py @@ -12,11 +12,11 @@ import ssl import sys -import gunicorn.http as http -import gunicorn.http.wsgi as wsgi -import gunicorn.sock as sock -import gunicorn.util as util -import gunicorn.workers.base as base +from gunicorn import http +from gunicorn.http import wsgi +from gunicorn import sock +from gunicorn import util +from gunicorn.workers import base class StopWaiting(Exception): diff --git a/tests/test_arbiter.py b/tests/test_arbiter.py index dc059edb7..e856282c8 100644 --- a/tests/test_arbiter.py +++ b/tests/test_arbiter.py @@ -4,7 +4,7 @@ # See the NOTICE for more information. import os -import unittest.mock as mock +from unittest import mock import gunicorn.app.base import gunicorn.arbiter diff --git a/tests/test_http.py b/tests/test_http.py index 33481266d..b6ca46b22 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -3,7 +3,7 @@ import io import t import pytest -import unittest.mock as mock +from unittest import mock from gunicorn import util from gunicorn.http.body import Body, LengthReader, EOFReader diff --git a/tests/test_pidfile.py b/tests/test_pidfile.py index fdcd0a24f..ecbc052ff 100644 --- a/tests/test_pidfile.py +++ b/tests/test_pidfile.py @@ -4,7 +4,7 @@ # See the NOTICE for more information. import errno -import unittest.mock as mock +from unittest import mock import gunicorn.pidfile diff --git a/tests/test_sock.py b/tests/test_sock.py index 36205d840..adc348c6f 100644 --- a/tests/test_sock.py +++ b/tests/test_sock.py @@ -3,7 +3,7 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -import unittest.mock as mock +from unittest import mock from gunicorn import sock diff --git a/tests/test_ssl.py b/tests/test_ssl.py index 7d15de173..a31c1fe0f 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -8,7 +8,7 @@ import pytest from gunicorn.config import ( - KeyFile, CertFile, SSLVersion, CACerts, SuppressRaggedEOFs, + KeyFile, CertFile, CACerts, SuppressRaggedEOFs, DoHandshakeOnConnect, Setting, Ciphers, ) diff --git a/tests/test_systemd.py b/tests/test_systemd.py index d2c78aa2c..ff8b959a0 100644 --- a/tests/test_systemd.py +++ b/tests/test_systemd.py @@ -5,7 +5,7 @@ from contextlib import contextmanager import os -import unittest.mock as mock +from unittest import mock import pytest diff --git a/tox.ini b/tox.ini index 64f58d662..080a43f53 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,7 @@ deps = [testenv:lint] commands = pylint -j0 \ + --max-line-length=120 \ gunicorn \ tests/test_arbiter.py \ tests/test_config.py \ @@ -26,7 +27,7 @@ commands = tests/test_util.py \ tests/test_valid_requests.py deps = - pylint<2.7 + pylint=2.17.4 [testenv:docs-lint] allowlist_externals = From cc15967cff8252a136ac13d584980d9794c44d35 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 17 May 2023 18:48:55 +0300 Subject: [PATCH 102/164] tox.ini deps --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 080a43f53..a7b7d38c6 100644 --- a/tox.ini +++ b/tox.ini @@ -27,7 +27,7 @@ commands = tests/test_util.py \ tests/test_valid_requests.py deps = - pylint=2.17.4 + pylint==2.17.4 [testenv:docs-lint] allowlist_externals = From dd0aebfc874a2b746ad9cc5aa3d9d5fa3e3b6956 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 17 May 2023 18:55:15 +0300 Subject: [PATCH 103/164] add to THANKS :) --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index 552675f00..9f4c6b6b9 100644 --- a/THANKS +++ b/THANKS @@ -22,6 +22,7 @@ Andrew Svetlov Anil V Antoine Girard Anton Vlasenko +Artur Kruchinin Bartosz Oler Ben Cochran Ben Oswald From fa94f705293f2cfb083676324a1fec1dcca095f8 Mon Sep 17 00:00:00 2001 From: Jason Myers Date: Tue, 30 May 2023 20:42:13 -0500 Subject: [PATCH 104/164] Updating Content-Length Handling Signed-off-by: Jason Myers --- gunicorn/http/message.py | 5 ++++- tests/requests/invalid/022.http | 3 +++ tests/requests/invalid/022.py | 5 +++++ tests/requests/invalid/023.http | 3 +++ tests/requests/invalid/023.py | 5 +++++ tests/requests/invalid/024.http | 3 +++ tests/requests/invalid/024.py | 5 +++++ 7 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/requests/invalid/022.http create mode 100644 tests/requests/invalid/022.py create mode 100644 tests/requests/invalid/023.http create mode 100644 tests/requests/invalid/023.py create mode 100644 tests/requests/invalid/024.http create mode 100644 tests/requests/invalid/024.py diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 64b2060c0..1f93c7145 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -139,7 +139,10 @@ def set_body_reader(self): self.body = Body(ChunkedReader(self, self.unreader)) elif content_length is not None: try: - content_length = int(content_length) + if str(content_length).isnumeric(): + content_length = int(content_length) + else: + raise InvalidHeader("CONTENT-LENGTH", req=self) except ValueError: raise InvalidHeader("CONTENT-LENGTH", req=self) diff --git a/tests/requests/invalid/022.http b/tests/requests/invalid/022.http new file mode 100644 index 000000000..521c7a06e --- /dev/null +++ b/tests/requests/invalid/022.http @@ -0,0 +1,3 @@ +GET /first HTTP/1.0\r\n +Content-Length: -0\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/invalid/022.py b/tests/requests/invalid/022.py new file mode 100644 index 000000000..95b0581ae --- /dev/null +++ b/tests/requests/invalid/022.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeader + +cfg = Config() +request = InvalidHeader diff --git a/tests/requests/invalid/023.http b/tests/requests/invalid/023.http new file mode 100644 index 000000000..c672f7896 --- /dev/null +++ b/tests/requests/invalid/023.http @@ -0,0 +1,3 @@ +GET /first HTTP/1.0\r\n +Content-Length: 0_1\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/invalid/023.py b/tests/requests/invalid/023.py new file mode 100644 index 000000000..95b0581ae --- /dev/null +++ b/tests/requests/invalid/023.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeader + +cfg = Config() +request = InvalidHeader diff --git a/tests/requests/invalid/024.http b/tests/requests/invalid/024.http new file mode 100644 index 000000000..31c062fa9 --- /dev/null +++ b/tests/requests/invalid/024.http @@ -0,0 +1,3 @@ +GET /first HTTP/1.0\r\n +Content-Length: +1\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/invalid/024.py b/tests/requests/invalid/024.py new file mode 100644 index 000000000..95b0581ae --- /dev/null +++ b/tests/requests/invalid/024.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeader + +cfg = Config() +request = InvalidHeader From 2f17eb508e21125795ef6c3648f1bc1dec26d5d5 Mon Sep 17 00:00:00 2001 From: Kurt McKee Date: Thu, 29 Jun 2023 09:50:49 -0500 Subject: [PATCH 105/164] Add a Dependabot config to keep GitHub action versions updated --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..8ac6b8c49 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" From 378f0d04ec390a754cb819f095bf65bfcc04ed99 Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Mon, 10 Jul 2023 22:09:23 +0000 Subject: [PATCH 106/164] bump to 21.0.0 --- LICENSE | 2 +- docs/source/news.rst | 40 ++-------------------------------------- gunicorn/__init__.py | 2 +- 3 files changed, 4 insertions(+), 40 deletions(-) diff --git a/LICENSE b/LICENSE index 65865a92a..b244a2eda 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -2009-2018 (c) Benoît Chesneau +2009-2023 (c) Benoît Chesneau 2009-2015 (c) Paul J. Davis Permission is hereby granted, free of charge, to any person diff --git a/docs/source/news.rst b/docs/source/news.rst index 0fb996133..0f0183454 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -2,46 +2,10 @@ Changelog ========= -20.1.0 - 2021-02-12 +21.0.0 - 2023-07-11 =================== -- document WEB_CONCURRENCY is set by, at least, Heroku -- capture peername from accept: Avoid calls to getpeername by capturing the peer name returned by accept -- log a warning when a worker was terminated due to a signal -- fix tornado usage with latest versions of Django -- add support for python -m gunicorn -- fix systemd socket activation example -- allows to set wsgi application in configg file using ``wsgi_app`` -- document ``--timeout = 0`` -- always close a connection when the number of requests exceeds the max requests -- Disable keepalive during graceful shutdown -- kill tasks in the gthread workers during upgrade -- fix latency in gevent worker when accepting new requests -- fix file watcher: handle errors when new worker reboot and ensure the list of files is kept -- document the default name and path of the configuration file -- document how variable impact configuration -- document the ``$PORT`` environment variable -- added milliseconds option to request_time in access_log -- added PIP requirements to be used for example -- remove version from the Server header -- fix sendfile: use ``socket.sendfile`` instead of ``os.sendfile`` -- reloader: use absolute path to prevent empty to prevent0 ``InotifyError`` when a file - is added to the working directory -- Add --print-config option to print the resolved settings at startup. -- remove the ``--log-dict-config`` CLI flag because it never had a working format - (the ``logconfig_dict`` setting in configuration files continues to work) - -** Breaking changes ** - -- minimum version is Python 3.5 -- remove version from the Server header - -** Others ** - -- miscellaneous changes in the code base to be a better citizen with Python 3 -- remove dead code -- fix documentation generation - +- TBD History ======= diff --git a/gunicorn/__init__.py b/gunicorn/__init__.py index 29edada5f..684c8abe4 100644 --- a/gunicorn/__init__.py +++ b/gunicorn/__init__.py @@ -3,7 +3,7 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -version_info = (20, 1, 0) +version_info = (21, 0, 0) __version__ = ".".join([str(v) for v in version_info]) SERVER = "gunicorn" SERVER_SOFTWARE = "%s/%s" % (SERVER, __version__) From 1ff10ff99db5704105487bdf1775f0f89fbaac21 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 17 Jul 2023 21:10:25 +0200 Subject: [PATCH 107/164] remove useless dep --- requirements_test.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 03af49696..fad22e315 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,4 +1,3 @@ -aiohttp gevent eventlet coverage From f72acb6c0eb9b61c8c0d7cc1283f1210ac04cdfd Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 17 Jul 2023 21:20:34 +0200 Subject: [PATCH 108/164] add missing depedency for eventlet tests --- requirements_test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_test.txt b/requirements_test.txt index fad22e315..4c6b0250c 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,5 +1,6 @@ gevent eventlet +cryptography coverage pytest pytest-cov From 0304f006e61d0a398b25d314292b4fdd16de83a6 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 17 Jul 2023 21:36:31 +0200 Subject: [PATCH 109/164] add minitimal changelog --- docs/source/2023-news.rst | 20 ++++++++++++++++++++ docs/source/news.rst | 17 +++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 docs/source/2023-news.rst diff --git a/docs/source/2023-news.rst b/docs/source/2023-news.rst new file mode 100644 index 000000000..7b4e963ab --- /dev/null +++ b/docs/source/2023-news.rst @@ -0,0 +1,20 @@ +================ +Changelog - 2023 +================ + +21.0.0 - 2023-07-17 +=================== + +- support python 3.11 +- fix gevent and eventlet workers +- fix threads support (gththread): improve performance and unblock requests +- SSL: noaw use SSLContext object +- HTTP parser: miscellaneous fixes +- remove unecessary setuid calls +- fix testing +- improve logging +- miscellaneous fixes to core engine + +*** RELEASE NOTE *** + +We made this release major to start our new release cycle. More info will be provided on our discussion forum. diff --git a/docs/source/news.rst b/docs/source/news.rst index 0f0183454..14eae72d3 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -2,10 +2,22 @@ Changelog ========= -21.0.0 - 2023-07-11 +21.0.0 - 2023-07-17 =================== -- TBD +- support python 3.11 +- fix gevent and eventlet workers +- fix threads support (gththread): improve performance and unblock requests +- SSL: noaw use SSLContext object +- HTTP parser: miscellaneous fixes +- remove unecessary setuid calls +- fix testing +- improve logging +- miscellaneous fixes to core engine + +*** RELEASE NOTE *** + +We made this release major to start our new release cycle. More info will be provided on our discussion forum. History ======= @@ -13,6 +25,7 @@ History .. toctree:: :titlesonly: + 2023-news 2021-news 2020-news 2019-news From cac38b42863786928d7c615bc0cb5620eb41c4af Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 17 Jul 2023 22:31:15 +0200 Subject: [PATCH 110/164] bump doc version --- docs/site/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/site/index.html b/docs/site/index.html index 17b95846b..3b5fd5940 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -16,7 +16,7 @@

    Latest version: 20.1.0 + href="https://docs.gunicorn.org/en/stable/">21.0.0
    From f628dd9730f965b1917397ea1846c68844b1fe7a Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 17 Jul 2023 22:40:52 +0200 Subject: [PATCH 111/164] fix import error --- gunicorn/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/util.py b/gunicorn/util.py index 1b39ba73c..751deea71 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -24,7 +24,7 @@ try: import importlib.metadata as importlib_metadata -except ImportError: +except (ModuleNotFoundError, ImportError): import importlib_metadata from gunicorn.errors import AppImportError From 471a6f80f0a255a41e0effe9896751db0a814999 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 17 Jul 2023 22:43:27 +0200 Subject: [PATCH 112/164] remove cryptography from requirements --- requirements_test.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 4c6b0250c..fad22e315 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,5 @@ gevent eventlet -cryptography coverage pytest pytest-cov From 91cb3dc67cb3a711336de8f2bc1f18dfe9ad57bf Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 17 Jul 2023 22:45:55 +0200 Subject: [PATCH 113/164] Revert "remove cryptography from requirements" This reverts commit 471a6f80f0a255a41e0effe9896751db0a814999. --- requirements_test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_test.txt b/requirements_test.txt index fad22e315..4c6b0250c 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,5 +1,6 @@ gevent eventlet +cryptography coverage pytest pytest-cov From 543628eb12ed1204eabfa844b242315086e68f52 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 17 Jul 2023 22:47:52 +0200 Subject: [PATCH 114/164] remove pypy-3.7 from tests --- .github/workflows/tox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 0dc9bc729..71e0ca760 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -10,7 +10,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] # All OSes pass except Windows because tests need Unix-only fcntl, grp, pwd, etc. - python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.7", "pypy-3.8" ] + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.8" ] steps: - uses: actions/checkout@v3 - name: Using Python ${{ matrix.python-version }} From b6eb01ba521104b46fbefc0c8393f0227e03bf9e Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 17 Jul 2023 23:03:59 +0200 Subject: [PATCH 115/164] add readthedocs.yaml --- docs/.readthedocs.yaml | 22 ++++++++++++++++++++++ docs/source/settings.rst | 5 +++-- 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 docs/.readthedocs.yaml diff --git a/docs/.readthedocs.yaml b/docs/.readthedocs.yaml new file mode 100644 index 000000000..f3aa1f5af --- /dev/null +++ b/docs/.readthedocs.yaml @@ -0,0 +1,22 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt diff --git a/docs/source/settings.rst b/docs/source/settings.rst index a23fb6923..4be760c28 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -320,7 +320,8 @@ respectively. Format: https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig -For more context you can look at the default configuration dictionary for logging, which can be found at ``gunicorn.glogging.CONFIG_DEFAULTS``. +For more context you can look at the default configuration dictionary for logging, +which can be found at ``gunicorn.glogging.CONFIG_DEFAULTS``. .. versionadded:: 19.8 @@ -346,7 +347,7 @@ Format: https://docs.python.org/3/library/logging.config.html#logging.config.jso **Command line:** ``--log-syslog-to SYSLOG_ADDR`` -**Default:** ``'udp://localhost:514'`` +**Default:** ``'unix:///var/run/syslog'`` Address to send syslog messages. From 1dd24e6e3cbbd4491f5340ce605f1636cde4056d Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 17 Jul 2023 23:08:54 +0200 Subject: [PATCH 116/164] fix .readthedocs.yaml path --- docs/.readthedocs.yaml => .readthedocs.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/.readthedocs.yaml => .readthedocs.yaml (100%) diff --git a/docs/.readthedocs.yaml b/.readthedocs.yaml similarity index 100% rename from docs/.readthedocs.yaml rename to .readthedocs.yaml From 033dca60cdc7413abaf5dcf15d16a90fb18a86b5 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 17 Jul 2023 23:14:57 +0200 Subject: [PATCH 117/164] fix doc path to conf.py in readthedocs.yaml --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f3aa1f5af..0ff559627 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,7 +13,7 @@ build: # Build documentation in the docs/ directory with Sphinx sphinx: - configuration: docs/conf.py + configuration: docs/source/conf.py # We recommend specifying your dependencies to enable reproducible builds: # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html From 547f8561d9eb3d1831cf3827aa23e24e4f833e59 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 17 Jul 2023 23:19:49 +0200 Subject: [PATCH 118/164] bump 21.0.1: fix doc --- docs/site/index.html | 2 +- docs/source/2023-news.rst | 5 +++++ docs/source/news.rst | 5 +++++ gunicorn/__init__.py | 2 +- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/site/index.html b/docs/site/index.html index 3b5fd5940..f2d8ce356 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -16,7 +16,7 @@
    Latest version: 21.0.0 + href="https://docs.gunicorn.org/en/stable/">21.0.1
    diff --git a/docs/source/2023-news.rst b/docs/source/2023-news.rst index 7b4e963ab..8b472c5e9 100644 --- a/docs/source/2023-news.rst +++ b/docs/source/2023-news.rst @@ -2,6 +2,11 @@ Changelog - 2023 ================ +21.0.1 - 2023-07-17 +=================== + +- fix documentation build + 21.0.0 - 2023-07-17 =================== diff --git a/docs/source/news.rst b/docs/source/news.rst index 14eae72d3..c9f20fe9c 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -2,6 +2,11 @@ Changelog ========= +21.0.1 - 2023-07-17 +=================== + +- fix documentation build + 21.0.0 - 2023-07-17 =================== diff --git a/gunicorn/__init__.py b/gunicorn/__init__.py index 684c8abe4..693061995 100644 --- a/gunicorn/__init__.py +++ b/gunicorn/__init__.py @@ -3,7 +3,7 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -version_info = (21, 0, 0) +version_info = (21, 0, 1) __version__ = ".".join([str(v) for v in version_info]) SERVER = "gunicorn" SERVER_SOFTWARE = "%s/%s" % (SERVER, __version__) From 86d85cb3699c41a2c958c604344b62022e5a4c38 Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 18 Jul 2023 14:10:36 +0200 Subject: [PATCH 119/164] fix gthread worker under Python 3.8 and sup exception is ValueError when fd has already been cleared by the system. fix #3029 --- gunicorn/workers/gthread.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index fb37fd3f3..dbbb7c3ae 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -180,6 +180,9 @@ def murder_keepalived(self): except KeyError: # already removed by the system, continue pass + except ValueError: + # already removed by the system continue + pass # close the socket conn.close() From 60b78e3d98030383b36a2b78fe599b9c241ca5f7 Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 18 Jul 2023 14:33:53 +0200 Subject: [PATCH 120/164] fix setup.cfg: use new license_files property `license_file` propert in setup.cfg is deprecated and should be replaced by the `license_files` property. fix #3027 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index b880b5d9c..c8b3a81b2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,4 +4,4 @@ testpaths = tests/ addopts = --assert=plain --cov=gunicorn --cov-report=xml [metadata] -license_file = LICENSE +license_files = ['LICENSE'] From 4e12ebe3342a52547cc764676975a0d488e98b6f Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 18 Jul 2023 14:41:05 +0200 Subject: [PATCH 121/164] bump to 21.1.0 --- docs/site/index.html | 2 +- docs/source/2023-news.rst | 5 +++++ docs/source/news.rst | 5 +++++ gunicorn/__init__.py | 2 +- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/site/index.html b/docs/site/index.html index f2d8ce356..b1a9d366f 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -16,7 +16,7 @@
    Latest version: 21.0.1 + href="https://docs.gunicorn.org/en/stable/">21.1.0
    diff --git a/docs/source/2023-news.rst b/docs/source/2023-news.rst index 8b472c5e9..fbfd8c7a4 100644 --- a/docs/source/2023-news.rst +++ b/docs/source/2023-news.rst @@ -2,6 +2,11 @@ Changelog - 2023 ================ +21.1.0 - 2023-07-18 +=================== + +- fix thread worker: fix socket removal from the queue + 21.0.1 - 2023-07-17 =================== diff --git a/docs/source/news.rst b/docs/source/news.rst index c9f20fe9c..aaf02bfb2 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -2,6 +2,11 @@ Changelog ========= +21.1.0 - 2023-07-18 +=================== + +- fix thread worker: fix socket removal from the queue + 21.0.1 - 2023-07-17 =================== diff --git a/gunicorn/__init__.py b/gunicorn/__init__.py index 693061995..cced3dc1c 100644 --- a/gunicorn/__init__.py +++ b/gunicorn/__init__.py @@ -3,7 +3,7 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -version_info = (21, 0, 1) +version_info = (21, 1, 0) __version__ = ".".join([str(v) for v in version_info]) SERVER = "gunicorn" SERVER_SOFTWARE = "%s/%s" % (SERVER, __version__) From bc905859a604a73dd2626b07cf12621f3ef36716 Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 19 Jul 2023 00:10:24 +0200 Subject: [PATCH 122/164] revert change considering connection as idle --- gunicorn/workers/gthread.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index dbbb7c3ae..c9c42345f 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -122,13 +122,10 @@ def accept(self, server, listener): sock, client = listener.accept() # initialize the connection object conn = TConn(self.cfg, sock, client, server) - # set timeout to ensure it will not be in the loop too long - conn.set_timeout() self.nr_conns += 1 # wait until socket is readable with self._lock: - self._keep.append(conn) self.poller.register(conn.sock, selectors.EVENT_READ, partial(self.on_client_socket_readable, conn)) except EnvironmentError as e: From ab9c8301cb9ae573ba597154ddeea16f0326fc15 Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 19 Jul 2023 13:31:10 +0200 Subject: [PATCH 123/164] bump to 21.2.0 --- docs/site/index.html | 2 +- docs/source/2023-news.rst | 8 ++++++-- docs/source/news.rst | 9 +++++++++ gunicorn/__init__.py | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/site/index.html b/docs/site/index.html index b1a9d366f..33e973c18 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -16,7 +16,7 @@
    Latest version: 21.1.0 + href="https://docs.gunicorn.org/en/stable/">21.2.0
    diff --git a/docs/source/2023-news.rst b/docs/source/2023-news.rst index fbfd8c7a4..80f022248 100644 --- a/docs/source/2023-news.rst +++ b/docs/source/2023-news.rst @@ -2,10 +2,14 @@ Changelog - 2023 ================ -21.1.0 - 2023-07-18 +21.2.0 - 2023-07-19 =================== -- fix thread worker: fix socket removal from the queue +- fix thread worker: revert change considering connection as idle . + +*** NOTE *** + +This is fixing the bad file description error. 21.0.1 - 2023-07-17 =================== diff --git a/docs/source/news.rst b/docs/source/news.rst index aaf02bfb2..73debcb9b 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -2,6 +2,15 @@ Changelog ========= +21.2.0 - 2023-07-19 +=================== + +- fix thread worker: revert change considering connection as idle . + +*** NOTE *** + +This is fixing the bad file description error. + 21.1.0 - 2023-07-18 =================== diff --git a/gunicorn/__init__.py b/gunicorn/__init__.py index cced3dc1c..adf5e89be 100644 --- a/gunicorn/__init__.py +++ b/gunicorn/__init__.py @@ -3,7 +3,7 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -version_info = (21, 1, 0) +version_info = (21, 2, 0) __version__ = ".".join([str(v) for v in version_info]) SERVER = "gunicorn" SERVER_SOFTWARE = "%s/%s" % (SERVER, __version__) From 760e864200ed8c21055f65ca609cf47012a102af Mon Sep 17 00:00:00 2001 From: Mathieu Dupuy Date: Mon, 4 Sep 2023 15:19:54 +0200 Subject: [PATCH 124/164] migrate to setup.cfg --- setup.cfg | 65 +++++++++++++++++++++++++++++ setup.py | 122 +----------------------------------------------------- 2 files changed, 67 insertions(+), 120 deletions(-) diff --git a/setup.cfg b/setup.cfg index c8b3a81b2..3cd2d7069 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,3 +5,68 @@ addopts = --assert=plain --cov=gunicorn --cov-report=xml [metadata] license_files = ['LICENSE'] +name = gunicorn +version = attr: gunicorn.__version__ +author = Benoit Chesneau +author_email = benoitc@gunicorn.org +license = MIT +description = WSGI HTTP Server for UNIX +url = https://gunicorn.org +long_description = file: README.rst +classifiers = + Development Status :: 5 - Production/Stable + Environment :: Other Environment + Intended Audience :: Developers + License :: OSI Approved :: MIT License + Operating System :: MacOS :: MacOS X + Operating System :: POSIX + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy + Topic :: Internet + Topic :: Utilities + Topic :: Software Development :: Libraries :: Python Modules + Topic :: Internet :: WWW/HTTP + Topic :: Internet :: WWW/HTTP :: WSGI + Topic :: Internet :: WWW/HTTP :: WSGI :: Server + Topic :: Internet :: WWW/HTTP :: Dynamic Content +project_urls = + Documentation = https://docs.gunicorn.org + Homepage = https://gunicorn.org + Issue tracker = https://github.com/benoitc/gunicorn/issues + Source code = https://github.com/benoitc/gunicorn + +[options] +packages = find: +zip_safe = False +install_requires = + importlib_metadata; python_version<"3.8" + packaging +include_package_data = True +python_requires = >=3.5 +tests_require = + gevent + eventlet + cryptography + coverage + pytest + pytest-cov + +[options.packages.find] +exclude = examples; tests + +[options.extras_require] +gevent = gevent>=1.4.0 +eventlet = eventlet>=0.24.1 +tornado = tornado>=0.2 +gthread = +setproctitle = setproctitle diff --git a/setup.py b/setup.py index dffd418cf..606849326 100644 --- a/setup.py +++ b/setup.py @@ -1,121 +1,3 @@ -# -*- coding: utf-8 - -# -# This file is part of gunicorn released under the MIT license. -# See the NOTICE for more information. +from setuptools import setup -import os -import sys - -from setuptools import setup, find_packages -from setuptools.command.test import test as TestCommand - -from gunicorn import __version__ - - -CLASSIFIERS = [ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Other Environment', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: POSIX', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Topic :: Internet', - 'Topic :: Utilities', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Internet :: WWW/HTTP :: WSGI', - 'Topic :: Internet :: WWW/HTTP :: WSGI :: Server', - 'Topic :: Internet :: WWW/HTTP :: Dynamic Content'] - -# read long description -with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as f: - long_description = f.read() - -# read dev requirements -fname = os.path.join(os.path.dirname(__file__), 'requirements_test.txt') -with open(fname) as f: - tests_require = [l.strip() for l in f.readlines()] - -class PyTestCommand(TestCommand): - user_options = [ - ("cov", None, "measure coverage") - ] - - def initialize_options(self): - TestCommand.initialize_options(self) - self.cov = None - - def finalize_options(self): - TestCommand.finalize_options(self) - self.test_args = ['tests'] - if self.cov: - self.test_args += ['--cov', 'gunicorn'] - self.test_suite = True - - def run_tests(self): - import pytest - errno = pytest.main(self.test_args) - sys.exit(errno) - - -install_requires = [ - 'importlib_metadata; python_version<"3.8"', - 'packaging', -] - -extras_require = { - 'gevent': ['gevent>=1.4.0'], - 'eventlet': ['eventlet>=0.24.1'], - 'tornado': ['tornado>=0.2'], - 'gthread': [], - 'setproctitle': ['setproctitle'], -} - -setup( - name='gunicorn', - version=__version__, - - description='WSGI HTTP Server for UNIX', - long_description=long_description, - author='Benoit Chesneau', - author_email='benoitc@gunicorn.org', - license='MIT', - url='https://gunicorn.org', - project_urls={ - 'Documentation': 'https://docs.gunicorn.org', - 'Homepage': 'https://gunicorn.org', - 'Issue tracker': 'https://github.com/benoitc/gunicorn/issues', - 'Source code': 'https://github.com/benoitc/gunicorn', - }, - - python_requires='>=3.5', - install_requires=install_requires, - classifiers=CLASSIFIERS, - zip_safe=False, - packages=find_packages(exclude=['examples', 'tests']), - include_package_data=True, - - tests_require=tests_require, - cmdclass={'test': PyTestCommand}, - - entry_points=""" - [console_scripts] - gunicorn=gunicorn.app.wsgiapp:run - - [paste.server_runner] - main=gunicorn.app.pasterapp:serve - """, - extras_require=extras_require, -) +setup() From 7033f27e286ddcbb40c3ae548710516d3cba0cc2 Mon Sep 17 00:00:00 2001 From: Mathieu Dupuy Date: Mon, 4 Sep 2023 15:22:37 +0200 Subject: [PATCH 125/164] remove exclusion of tests and examples dir setuptools already exclude them by default in the flat-layout setup, see https://setuptools.pypa.io/en/latest/userguide/package_discovery.html\#flat-layout --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 3cd2d7069..e8a320074 100644 --- a/setup.cfg +++ b/setup.cfg @@ -61,9 +61,6 @@ tests_require = pytest pytest-cov -[options.packages.find] -exclude = examples; tests - [options.extras_require] gevent = gevent>=1.4.0 eventlet = eventlet>=0.24.1 From fdd23e82926d7d10ec4a8e65e42b5184a4ee20ce Mon Sep 17 00:00:00 2001 From: Mathieu Dupuy Date: Tue, 12 Sep 2023 00:43:37 +0200 Subject: [PATCH 126/164] migrate to pyproject.toml --- pyproject.toml | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 65 ---------------------------------------- setup.py | 3 -- 3 files changed, 80 insertions(+), 68 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..ec64a3285 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,80 @@ +[build-system] +requires = ["setuptools>=61.2"] +build-backend = "setuptools.build_meta" + +[project] +name = "gunicorn" +authors = [{name = "Benoit Chesneau", email = "benoitc@gunicorn.org"}] +license = {text = "MIT"} +description = "WSGI HTTP Server for UNIX" +readme = "README.rst" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Other Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Internet", + "Topic :: Utilities", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Internet :: WWW/HTTP :: WSGI", + "Topic :: Internet :: WWW/HTTP :: WSGI :: Server", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content", +] +requires-python = ">=3.5" +dependencies = [ + 'importlib_metadata; python_version<"3.8"', + "packaging", +] +dynamic = ["version"] + +[project.urls] +Homepage = "https://gunicorn.org" +Documentation = "https://docs.gunicorn.org" +"Issue tracker" = "https://github.com/benoitc/gunicorn/issues" +"Source code" = "https://github.com/benoitc/gunicorn" + +[project.optional-dependencies] +gevent = ["gevent>=1.4.0"] +eventlet = ["eventlet>=0.24.1"] +tornado = ["tornado>=0.2"] +gthread = [] +setproctitle = ["setproctitle"] +testing = [ + "gevent", + "eventlet", + "cryptography", + "coverage", + "pytest", + "pytest-cov", +] + +[tool.pytest.ini_options] +norecursedirs = ["examples", "lib", "local", "src"] +testpaths = ["tests/"] +addopts = "--assert=plain --cov=gunicorn --cov-report=xml" + +[tool.setuptools] +zip-safe = false +include-package-data = true +license-files = ["['LICENSE']"] + +[tool.setuptools.packages] +find = {namespaces = false} + +[tool.setuptools.dynamic] +version = {attr = "gunicorn.__version__"} diff --git a/setup.cfg b/setup.cfg index e8a320074..07322e337 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,68 +2,3 @@ norecursedirs = examples lib local src testpaths = tests/ addopts = --assert=plain --cov=gunicorn --cov-report=xml - -[metadata] -license_files = ['LICENSE'] -name = gunicorn -version = attr: gunicorn.__version__ -author = Benoit Chesneau -author_email = benoitc@gunicorn.org -license = MIT -description = WSGI HTTP Server for UNIX -url = https://gunicorn.org -long_description = file: README.rst -classifiers = - Development Status :: 5 - Production/Stable - Environment :: Other Environment - Intended Audience :: Developers - License :: OSI Approved :: MIT License - Operating System :: MacOS :: MacOS X - Operating System :: POSIX - Programming Language :: Python - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: Implementation :: CPython - Programming Language :: Python :: Implementation :: PyPy - Topic :: Internet - Topic :: Utilities - Topic :: Software Development :: Libraries :: Python Modules - Topic :: Internet :: WWW/HTTP - Topic :: Internet :: WWW/HTTP :: WSGI - Topic :: Internet :: WWW/HTTP :: WSGI :: Server - Topic :: Internet :: WWW/HTTP :: Dynamic Content -project_urls = - Documentation = https://docs.gunicorn.org - Homepage = https://gunicorn.org - Issue tracker = https://github.com/benoitc/gunicorn/issues - Source code = https://github.com/benoitc/gunicorn - -[options] -packages = find: -zip_safe = False -install_requires = - importlib_metadata; python_version<"3.8" - packaging -include_package_data = True -python_requires = >=3.5 -tests_require = - gevent - eventlet - cryptography - coverage - pytest - pytest-cov - -[options.extras_require] -gevent = gevent>=1.4.0 -eventlet = eventlet>=0.24.1 -tornado = tornado>=0.2 -gthread = -setproctitle = setproctitle diff --git a/setup.py b/setup.py deleted file mode 100644 index 606849326..000000000 --- a/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from setuptools import setup - -setup() From 7d69222b55f190d0b4f3bea500ddb8e3790c9d95 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 21:28:16 +0000 Subject: [PATCH 127/164] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/lint.yml | 2 +- .github/workflows/tox.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d66a54586..795946e4e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: toxenv: [lint, docs-lint, pycodestyle] python-version: [ "3.10" ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Using Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 71e0ca760..d253148f5 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -12,7 +12,7 @@ jobs: os: [ubuntu-latest, macos-latest] # All OSes pass except Windows because tests need Unix-only fcntl, grp, pwd, etc. python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.8" ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Using Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From afe068021215d99083a84261dcba4e5dc61b4b8a Mon Sep 17 00:00:00 2001 From: Ben Cail Date: Mon, 23 Oct 2023 16:58:12 -0400 Subject: [PATCH 128/164] Document Python 3.7 requirement --- README.rst | 2 +- docs/source/index.rst | 2 +- docs/source/install.rst | 2 +- setup.py | 4 +--- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index fe9523b9a..4a4029dd3 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ The documentation is hosted at https://docs.gunicorn.org. Installation ------------ -Gunicorn requires **Python 3.x >= 3.5**. +Gunicorn requires **Python 3.x >= 3.7**. Install from PyPI:: diff --git a/docs/source/index.rst b/docs/source/index.rst index 24c4814d2..3f89ce1eb 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -23,7 +23,7 @@ Features * Simple Python configuration * Multiple worker configurations * Various server hooks for extensibility -* Compatible with Python 3.x >= 3.5 +* Compatible with Python 3.x >= 3.7 Contents diff --git a/docs/source/install.rst b/docs/source/install.rst index 66f2479e7..2367086df 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -4,7 +4,7 @@ Installation .. highlight:: bash -:Requirements: **Python 3.x >= 3.5** +:Requirements: **Python 3.x >= 3.7** To install the latest released version of Gunicorn:: diff --git a/setup.py b/setup.py index dffd418cf..0d73584d2 100644 --- a/setup.py +++ b/setup.py @@ -21,8 +21,6 @@ 'Operating System :: POSIX', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', @@ -100,7 +98,7 @@ def run_tests(self): 'Source code': 'https://github.com/benoitc/gunicorn', }, - python_requires='>=3.5', + python_requires='>=3.7', install_requires=install_requires, classifiers=CLASSIFIERS, zip_safe=False, From 4ce82358e88357982676153c0141236d32c1d36b Mon Sep 17 00:00:00 2001 From: Jelmer Draaijer Date: Tue, 31 Oct 2023 08:42:48 +0100 Subject: [PATCH 129/164] Add Python 3.12 to test matrix and add classifiers --- .github/workflows/tox.yml | 2 +- setup.py | 1 + tox.ini | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index d253148f5..2c0d737da 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -10,7 +10,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] # All OSes pass except Windows because tests need Unix-only fcntl, grp, pwd, etc. - python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.8" ] + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.8" ] steps: - uses: actions/checkout@v4 - name: Using Python ${{ matrix.python-version }} diff --git a/setup.py b/setup.py index dffd418cf..77413acd0 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', diff --git a/tox.ini b/tox.ini index a7b7d38c6..aedd522fd 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{37,38,39,310,311,py3}, lint, docs-lint, pycodestyle +envlist = py{37,38,39,310,311,312,py3}, lint, docs-lint, pycodestyle skipsdist = false ; Can't set skipsdist and use_develop in tox v4 to true due to https://github.com/tox-dev/tox/issues/2730 From 237f3e6f5ccfee0355cb468b0f4257092c4d5327 Mon Sep 17 00:00:00 2001 From: sblondon Date: Sat, 11 Nov 2023 23:02:07 +0100 Subject: [PATCH 130/164] Remove Python2 note Python2 is not supported anymore. --- docs/source/design.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/source/design.rst b/docs/source/design.rst index cb609cf2f..60e8a39aa 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -143,10 +143,6 @@ signal, as the application code will be shared among workers but loaded only in the worker processes (unlike when using the preload setting, which loads the code in the master process). -.. note:: - Under Python 2.x, you need to install the 'futures' package to use this - feature. - .. _Greenlets: https://github.com/python-greenlet/greenlet .. _Eventlet: http://eventlet.net/ .. _Gevent: http://www.gevent.org/ From 0dd6b6350e2811462cc472fc6fddf275412006e4 Mon Sep 17 00:00:00 2001 From: Rami <72725910+ramikg@users.noreply.github.com> Date: Tue, 14 Nov 2023 11:54:14 +0200 Subject: [PATCH 131/164] Use `utime` instead of `fchmod` in WorkerTmp.notify --- gunicorn/workers/workertmp.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gunicorn/workers/workertmp.py b/gunicorn/workers/workertmp.py index cc79ecd6d..080bcb399 100644 --- a/gunicorn/workers/workertmp.py +++ b/gunicorn/workers/workertmp.py @@ -39,11 +39,8 @@ def __init__(self, cfg): os.close(fd) raise - self.spinner = 0 - def notify(self): - self.spinner = (self.spinner + 1) % 2 - os.fchmod(self._tmp.fileno(), self.spinner) + os.utime(self._tmp.fileno()) def last_update(self): return os.fstat(self._tmp.fileno()).st_ctime From 7acd83bfb6404794bf8871dd075440c95f8dcb36 Mon Sep 17 00:00:00 2001 From: Mathieu Dupuy Date: Sat, 2 Dec 2023 11:03:13 +0100 Subject: [PATCH 132/164] pyproject.toml: fix license-files field --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ec64a3285..f4afa7214 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ addopts = "--assert=plain --cov=gunicorn --cov-report=xml" [tool.setuptools] zip-safe = false include-package-data = true -license-files = ["['LICENSE']"] +license-files = ["LICENSE"] [tool.setuptools.packages] find = {namespaces = false} From 40232284934c32939c0e4e78caad1987c3773e08 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 7 Dec 2023 15:36:28 +0100 Subject: [PATCH 133/164] let's exception not bubble ensure we can catch correctly exceptions based on BaseException. Note: patch was origninally proposed by the pr #2923, but original author closed it. Fix #2923 --- gunicorn/workers/base_async.py | 2 +- gunicorn/workers/sync.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gunicorn/workers/base_async.py b/gunicorn/workers/base_async.py index b059a7cb5..6a79d7ed0 100644 --- a/gunicorn/workers/base_async.py +++ b/gunicorn/workers/base_async.py @@ -82,7 +82,7 @@ def handle(self, listener, client, addr): self.log.debug("Ignoring socket not connected") else: self.log.debug("Ignoring EPIPE") - except Exception as e: + except BaseException as e: self.handle_error(req, client, addr, e) finally: util.close(client) diff --git a/gunicorn/workers/sync.py b/gunicorn/workers/sync.py index 39a209f06..ddcd77270 100644 --- a/gunicorn/workers/sync.py +++ b/gunicorn/workers/sync.py @@ -154,7 +154,7 @@ def handle(self, listener, client, addr): self.log.debug("Ignoring socket not connected") else: self.log.debug("Ignoring EPIPE") - except Exception as e: + except BaseException as e: self.handle_error(req, client, addr, e) finally: util.close(client) From c3396b9786b07b10d71d2b224888c7cd32be3625 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 7 Dec 2023 10:38:30 +0100 Subject: [PATCH 134/164] github actions: cache and test run_module (-m) --- .github/workflows/lint.yml | 6 +++++- .github/workflows/tox.yml | 9 ++++++++- gunicorn/__main__.py | 6 +++++- gunicorn/app/wsgiapp.py | 4 ++-- pyproject.toml | 2 +- requirements_test.txt | 1 - tox.ini | 5 ++++- 7 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 795946e4e..b0a862cd1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,6 +2,9 @@ name: lint on: [push, pull_request] permissions: contents: read # to fetch code (actions/checkout) +env: + # note that some tools care only for the name, not the value + FORCE_COLOR: 1 jobs: lint: name: tox-${{ matrix.toxenv }} @@ -14,9 +17,10 @@ jobs: steps: - uses: actions/checkout@v4 - name: Using Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: pip - name: Install Dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index d253148f5..c188a4dba 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -2,6 +2,9 @@ name: tox on: [push, pull_request] permissions: contents: read # to fetch code (actions/checkout) +env: + # note that some tools care only for the name, not the value + FORCE_COLOR: 1 jobs: tox: name: ${{ matrix.os }} / ${{ matrix.python-version }} @@ -14,11 +17,15 @@ jobs: steps: - uses: actions/checkout@v4 - name: Using Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: requirements_test.txt + check-latest: true - name: Install Dependencies run: | python -m pip install --upgrade pip python -m pip install tox + - run: tox -e run-module - run: tox -e py diff --git a/gunicorn/__main__.py b/gunicorn/__main__.py index 49ba6960d..fda831570 100644 --- a/gunicorn/__main__.py +++ b/gunicorn/__main__.py @@ -4,4 +4,8 @@ # See the NOTICE for more information. from gunicorn.app.wsgiapp import run -run() + +if __name__ == "__main__": + # see config.py - argparse defaults to basename(argv[0]) == "__main__.py" + # todo: let runpy.run_module take care of argv[0] rewriting + run(prog="gunicorn") diff --git a/gunicorn/app/wsgiapp.py b/gunicorn/app/wsgiapp.py index 36cfba9d8..4e0031234 100644 --- a/gunicorn/app/wsgiapp.py +++ b/gunicorn/app/wsgiapp.py @@ -58,13 +58,13 @@ def load(self): return self.load_wsgiapp() -def run(): +def run(prog=None): """\ The ``gunicorn`` command line runner for launching Gunicorn with generic WSGI applications. """ from gunicorn.app.wsgiapp import WSGIApplication - WSGIApplication("%(prog)s [OPTIONS] [APP_MODULE]").run() + WSGIApplication("%(prog)s [OPTIONS] [APP_MODULE]", prog=prog).run() if __name__ == '__main__': diff --git a/pyproject.toml b/pyproject.toml index f4afa7214..ffe20515a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,13 +57,13 @@ setproctitle = ["setproctitle"] testing = [ "gevent", "eventlet", - "cryptography", "coverage", "pytest", "pytest-cov", ] [tool.pytest.ini_options] +# # can override these: python -m pytest --override-ini="addopts=" norecursedirs = ["examples", "lib", "local", "src"] testpaths = ["tests/"] addopts = "--assert=plain --cov=gunicorn --cov-report=xml" diff --git a/requirements_test.txt b/requirements_test.txt index 4c6b0250c..fad22e315 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,5 @@ gevent eventlet -cryptography coverage pytest pytest-cov diff --git a/tox.ini b/tox.ini index a7b7d38c6..83ce3374e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{37,38,39,310,311,py3}, lint, docs-lint, pycodestyle +envlist = py{37,38,39,310,311,py3}, lint, docs-lint, pycodestyle, run-module skipsdist = false ; Can't set skipsdist and use_develop in tox v4 to true due to https://github.com/tox-dev/tox/issues/2730 @@ -9,6 +9,9 @@ commands = pytest --cov=gunicorn {posargs} deps = -rrequirements_test.txt +[testenv:run-module] +commands = python3 -m gunicorn --version + [testenv:lint] commands = pylint -j0 \ From 611746edc91e202ddd92f97e560d670054458f7b Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 7 Dec 2023 20:02:47 +0100 Subject: [PATCH 135/164] CI: check entry points Fixes: fdd23e82926d7d10ec4a8e65e42b5184a4ee20ce --- .github/workflows/tox.yml | 1 + appveyor.yml | 5 +++++ pyproject.toml | 8 ++++++++ requirements_dev.txt | 5 +++++ tox.ini | 9 +++++++-- 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index c188a4dba..053b8d920 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -28,4 +28,5 @@ jobs: python -m pip install --upgrade pip python -m pip install tox - run: tox -e run-module + - run: tox -e run-entrypoint - run: tox -e py diff --git a/appveyor.yml b/appveyor.yml index 5eb48e9cd..85feb4dc3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,6 +7,11 @@ environment: PYTHON: "C:\\Python38-x64" - TOXENV: pycodestyle PYTHON: "C:\\Python38-x64" + # Windows cannot even import the module when they unconditionally import, see below. + #- TOXENV: run-module + # PYTHON: "C:\\Python38-x64" + #- TOXENV: run-entrypoint + # PYTHON: "C:\\Python38-x64" # Windows is not ready for testing!!! # Python's fcntl, grp, pwd, os.geteuid(), and socket.AF_UNIX are all Unix-only. #- TOXENV: py35 diff --git a/pyproject.toml b/pyproject.toml index ffe20515a..3ef076cc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,14 @@ testing = [ "pytest-cov", ] +[project.scripts] +# duplicates "python -m gunicorn" handling in __main__.py +gunicorn = "gunicorn.app.wsgiapp:run" + +# note the quotes around "paste.server_runner" to escape the dot +[project.entry-points."paste.server_runner"] +main = "gunicorn.app.pasterapp:serve" + [tool.pytest.ini_options] # # can override these: python -m pytest --override-ini="addopts=" norecursedirs = ["examples", "lib", "local", "src"] diff --git a/requirements_dev.txt b/requirements_dev.txt index a36971d58..1d8c01291 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,9 @@ -r requirements_test.txt +# setuptools v68.0 fails hard on invalid pyproject.toml +# which a developer would want to know +# otherwise, oldest known-working version is 61.2 +setuptools>=68.0 + sphinx sphinx_rtd_theme diff --git a/tox.ini b/tox.ini index 83ce3374e..b299a5a46 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{37,38,39,310,311,py3}, lint, docs-lint, pycodestyle, run-module +envlist = py{37,38,39,310,311,py3}, lint, docs-lint, pycodestyle, run-entrypoint, run-module skipsdist = false ; Can't set skipsdist and use_develop in tox v4 to true due to https://github.com/tox-dev/tox/issues/2730 @@ -9,8 +9,13 @@ commands = pytest --cov=gunicorn {posargs} deps = -rrequirements_test.txt +[testenv:run-entrypoint] +# entry point: console script (provided by setuptools from pyproject.toml) +commands = python -c 'import subprocess; cmd_out = subprocess.check_output(["gunicorn", "--version"])[:79].decode("utf-8", errors="replace"); print(cmd_out); assert cmd_out.startswith("gunicorn ")' + [testenv:run-module] -commands = python3 -m gunicorn --version +# runpy (provided by module.__main__) +commands = python -c 'import sys,subprocess; cmd_out = subprocess.check_output([sys.executable, "-m", "gunicorn", "--version"])[:79].decode("utf-8", errors="replace"); print(cmd_out); assert cmd_out.startswith("gunicorn ")' [testenv:lint] commands = From 559caf920537ece2ef058e1de5e36af44756bb19 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Wed, 6 Dec 2023 15:30:50 +0100 Subject: [PATCH 136/164] pytest: raise on malformed test fixtures and unbreak test depending on backslash escape --- tests/treq.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/treq.py b/tests/treq.py index ffe0691fd..05adb1461 100644 --- a/tests/treq.py +++ b/tests/treq.py @@ -51,7 +51,9 @@ def __init__(self, fname, expect): with open(self.fname, 'rb') as handle: self.data = handle.read() self.data = self.data.replace(b"\n", b"").replace(b"\\r\\n", b"\r\n") - self.data = self.data.replace(b"\\0", b"\000") + self.data = self.data.replace(b"\\0", b"\000").replace(b"\\n", b"\n").replace(b"\\t", b"\t") + if b"\\" in self.data: + raise AssertionError("Unexpected backslash in test data - only handling HTAB, NUL and CRLF") # Functions for sending data to the parser. # These functions mock out reading from a @@ -262,7 +264,8 @@ def same(self, req, sizer, matcher, exp): assert req.trailers == exp.get("trailers", []) -class badrequest(object): +class badrequest: + # FIXME: no good reason why this cannot match what the more extensive mechanism above def __init__(self, fname): self.fname = fname self.name = os.path.basename(fname) @@ -270,7 +273,9 @@ def __init__(self, fname): with open(self.fname) as handle: self.data = handle.read() self.data = self.data.replace("\n", "").replace("\\r\\n", "\r\n") - self.data = self.data.replace("\\0", "\000") + self.data = self.data.replace("\\0", "\000").replace("\\n", "\n").replace("\\t", "\t") + if "\\" in self.data: + raise AssertionError("Unexpected backslash in test data - only handling HTAB, NUL and CRLF") self.data = self.data.encode('latin1') def send(self): @@ -283,4 +288,6 @@ def send(self): def check(self, cfg): p = RequestParser(cfg, self.send(), None) - next(p) + # must fully consume iterator, otherwise EOF errors could go unnoticed + for _ in p: + pass From 2dbe49de99f681923330d98f5b872730c73943d0 Mon Sep 17 00:00:00 2001 From: Ben Kallus Date: Mon, 28 Aug 2023 22:32:36 -0400 Subject: [PATCH 137/164] RFC compliant header field+chunk validation * update HEADER_RE and HEADER_VALUE_RE to match the RFCs * update chunk length parsing to disallow 0x prefix and digit-separating underscores. --- gunicorn/http/body.py | 5 ++--- gunicorn/http/message.py | 2 +- gunicorn/http/wsgi.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/gunicorn/http/body.py b/gunicorn/http/body.py index aa1af2cb3..41fe334bc 100644 --- a/gunicorn/http/body.py +++ b/gunicorn/http/body.py @@ -86,10 +86,9 @@ def parse_chunk_size(self, unreader, data=None): line, rest_chunk = data[:idx], data[idx + 2:] chunk_size = line.split(b";", 1)[0].strip() - try: - chunk_size = int(chunk_size, 16) - except ValueError: + if any(n not in b"0123456789abcdefABCDEF" for n in chunk_size): raise InvalidChunkSize(chunk_size) + chunk_size = int(chunk_size, 16) if chunk_size == 0: try: diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 1f93c7145..0006fa61b 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -21,7 +21,7 @@ MAX_HEADERS = 32768 DEFAULT_MAX_HEADERFIELD_SIZE = 8190 -HEADER_RE = re.compile(r"[\x00-\x1F\x7F()<>@,;:\[\]={} \t\\\"]") +HEADER_RE = re.compile(r"[^!#$%&'*+\-.\^_`|~0-9a-zA-Z]") METH_RE = re.compile(r"[A-Z0-9$-_.]{3,20}") VERSION_RE = re.compile(r"HTTP/(\d+)\.(\d+)") diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 25715eab7..10c5a3dd5 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -18,7 +18,7 @@ # with sending files in blocks over 2GB. BLKSIZE = 0x3FFFFFFF -HEADER_VALUE_RE = re.compile(r'[\x00-\x1F\x7F]') +HEADER_VALUE_RE = re.compile(r'[^ \t\x21-\x7e\x80-\xff]') log = logging.getLogger(__name__) From 735e9e867af23acfbe8eca9ea2e546542d65e8c9 Mon Sep 17 00:00:00 2001 From: Ben Kallus Date: Mon, 4 Dec 2023 17:08:16 -0500 Subject: [PATCH 138/164] Disallow empty header names. --- gunicorn/http/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 0006fa61b..11ee4d15c 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -83,7 +83,7 @@ def parse_headers(self, data): # Parse initial header name : value pair. curr = lines.pop(0) header_length = len(curr) - if curr.find(":") < 0: + if curr.find(":") <= 0: raise InvalidHeader(curr.strip()) name, value = curr.split(":", 1) if self.cfg.strip_header_spaces: From 72238fcf8d29db62fdbb9d2481ced801e27fe139 Mon Sep 17 00:00:00 2001 From: Ben Kallus Date: Wed, 6 Dec 2023 17:28:40 -0500 Subject: [PATCH 139/164] RFC compliant request line and header parsing - Unify HEADER_RE and METH_RE - Replace CRLF with SP during obs-fold processing (See RFC 9112 Section 5.2, last paragraph) - Stop stripping header names. - Remove HTAB in OWS in header values that use obs-fold (See RFC 9112 Section 5.2, last paragraph) - Use fullmatch instead of search, which has problems with empty strings. (See GHSA-68xg-gqqm-vgj8) - Split proxy protocol line on space only. (See proxy protocol Section 2.1, bullet 3) - Use fullmatch for method and version (Thank you to Paul Dorn for noticing this.) - Replace calls to str.strip() with str.strip(' \t') - Split request line on SP only. Co-authored-by: Paul Dorn --- gunicorn/http/message.py | 33 +++++++++-------- gunicorn/http/wsgi.py | 23 ++++++------ tests/requests/invalid/003.http | 4 +-- tests/requests/invalid/003.py | 4 +-- tests/requests/valid/016.py | 64 ++++++++++++++++----------------- tests/requests/valid/031.http | 2 ++ tests/requests/valid/031.py | 7 ++++ 7 files changed, 74 insertions(+), 63 deletions(-) create mode 100644 tests/requests/valid/031.http create mode 100644 tests/requests/valid/031.py diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 11ee4d15c..db3d44bb2 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -21,8 +21,7 @@ MAX_HEADERS = 32768 DEFAULT_MAX_HEADERFIELD_SIZE = 8190 -HEADER_RE = re.compile(r"[^!#$%&'*+\-.\^_`|~0-9a-zA-Z]") -METH_RE = re.compile(r"[A-Z0-9$-_.]{3,20}") +TOKEN_RE = re.compile(r"[!#$%&'*+\-.\^_`|~0-9a-zA-Z]+") VERSION_RE = re.compile(r"HTTP/(\d+)\.(\d+)") @@ -63,8 +62,8 @@ def parse_headers(self, data): cfg = self.cfg headers = [] - # Split lines on \r\n keeping the \r\n on each line - lines = [bytes_to_str(line) + "\r\n" for line in data.split(b"\r\n")] + # Split lines on \r\n + lines = [bytes_to_str(line) for line in data.split(b"\r\n")] # handle scheme headers scheme_header = False @@ -80,30 +79,30 @@ def parse_headers(self, data): if len(headers) >= self.limit_request_fields: raise LimitRequestHeaders("limit request headers fields") - # Parse initial header name : value pair. + # Parse initial header name: value pair. curr = lines.pop(0) - header_length = len(curr) + header_length = len(curr) + len("\r\n") if curr.find(":") <= 0: - raise InvalidHeader(curr.strip()) + raise InvalidHeader(curr) name, value = curr.split(":", 1) if self.cfg.strip_header_spaces: name = name.rstrip(" \t").upper() else: name = name.upper() - if HEADER_RE.search(name): + if not TOKEN_RE.fullmatch(name): raise InvalidHeaderName(name) - name, value = name.strip(), [value.lstrip()] + value = [value.lstrip(" \t")] # Consume value continuation lines while lines and lines[0].startswith((" ", "\t")): curr = lines.pop(0) - header_length += len(curr) + header_length += len(curr) + len("\r\n") if header_length > self.limit_request_field_size > 0: raise LimitRequestHeaders("limit request headers " "fields size") - value.append(curr) - value = ''.join(value).rstrip() + value.append(curr.strip("\t ")) + value = " ".join(value) if header_length > self.limit_request_field_size > 0: raise LimitRequestHeaders("limit request headers fields size") @@ -156,7 +155,7 @@ def set_body_reader(self): def should_close(self): for (h, v) in self.headers: if h == "CONNECTION": - v = v.lower().strip() + v = v.lower().strip(" \t") if v == "close": return True elif v == "keep-alive": @@ -283,7 +282,7 @@ def proxy_protocol_access_check(self): raise ForbiddenProxyRequest(self.peer_addr[0]) def parse_proxy_protocol(self, line): - bits = line.split() + bits = line.split(" ") if len(bits) != 6: raise InvalidProxyLine(line) @@ -328,12 +327,12 @@ def parse_proxy_protocol(self, line): } def parse_request_line(self, line_bytes): - bits = [bytes_to_str(bit) for bit in line_bytes.split(None, 2)] + bits = [bytes_to_str(bit) for bit in line_bytes.split(b" ", 2)] if len(bits) != 3: raise InvalidRequestLine(bytes_to_str(line_bytes)) # Method - if not METH_RE.match(bits[0]): + if not TOKEN_RE.fullmatch(bits[0]): raise InvalidRequestMethod(bits[0]) self.method = bits[0].upper() @@ -349,7 +348,7 @@ def parse_request_line(self, line_bytes): self.fragment = parts.fragment or "" # Version - match = VERSION_RE.match(bits[2]) + match = VERSION_RE.fullmatch(bits[2]) if match is None: raise InvalidHTTPVersion(bits[2]) self.version = (int(match.group(1)), int(match.group(2))) diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 10c5a3dd5..7fca61422 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -9,7 +9,7 @@ import re import sys -from gunicorn.http.message import HEADER_RE +from gunicorn.http.message import TOKEN_RE from gunicorn.http.errors import InvalidHeader, InvalidHeaderName from gunicorn import SERVER_SOFTWARE, SERVER from gunicorn import util @@ -18,7 +18,9 @@ # with sending files in blocks over 2GB. BLKSIZE = 0x3FFFFFFF -HEADER_VALUE_RE = re.compile(r'[^ \t\x21-\x7e\x80-\xff]') +# RFC9110 5.5: field-vchar = VCHAR / obs-text +# RFC4234 B.1: VCHAR = 0x21-x07E = printable ASCII +HEADER_VALUE_RE = re.compile(r'[ \t\x21-\x7e\x80-\xff]*') log = logging.getLogger(__name__) @@ -249,31 +251,32 @@ def process_headers(self, headers): if not isinstance(name, str): raise TypeError('%r is not a string' % name) - if HEADER_RE.search(name): + if not TOKEN_RE.fullmatch(name): raise InvalidHeaderName('%r' % name) if not isinstance(value, str): raise TypeError('%r is not a string' % value) - if HEADER_VALUE_RE.search(value): + if not HEADER_VALUE_RE.fullmatch(value): raise InvalidHeader('%r' % value) - value = value.strip() - lname = name.lower().strip() + # RFC9110 5.5 + value = value.strip(" \t") + lname = name.lower() if lname == "content-length": self.response_length = int(value) elif util.is_hoppish(name): if lname == "connection": # handle websocket - if value.lower().strip() == "upgrade": + if value.lower() == "upgrade": self.upgrade = True elif lname == "upgrade": - if value.lower().strip() == "websocket": - self.headers.append((name.strip(), value)) + if value.lower() == "websocket": + self.headers.append((name, value)) # ignore hopbyhop headers continue - self.headers.append((name.strip(), value)) + self.headers.append((name, value)) def is_chunked(self): # Only use chunked responses when the client is diff --git a/tests/requests/invalid/003.http b/tests/requests/invalid/003.http index cd1ab7fcb..5a9eaafcd 100644 --- a/tests/requests/invalid/003.http +++ b/tests/requests/invalid/003.http @@ -1,2 +1,2 @@ --blargh /foo HTTP/1.1\r\n -\r\n \ No newline at end of file +GET\n/\nHTTP/1.1\r\n +\r\n diff --git a/tests/requests/invalid/003.py b/tests/requests/invalid/003.py index 86a0774e5..5a4ca8961 100644 --- a/tests/requests/invalid/003.py +++ b/tests/requests/invalid/003.py @@ -1,2 +1,2 @@ -from gunicorn.http.errors import InvalidRequestMethod -request = InvalidRequestMethod \ No newline at end of file +from gunicorn.http.errors import InvalidRequestLine +request = InvalidRequestLine diff --git a/tests/requests/valid/016.py b/tests/requests/valid/016.py index 139b27009..4e5144f8c 100644 --- a/tests/requests/valid/016.py +++ b/tests/requests/valid/016.py @@ -1,35 +1,35 @@ -certificate = """-----BEGIN CERTIFICATE-----\r\n - MIIFbTCCBFWgAwIBAgICH4cwDQYJKoZIhvcNAQEFBQAwcDELMAkGA1UEBhMCVUsx\r\n - ETAPBgNVBAoTCGVTY2llbmNlMRIwEAYDVQQLEwlBdXRob3JpdHkxCzAJBgNVBAMT\r\n - AkNBMS0wKwYJKoZIhvcNAQkBFh5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMu\r\n - dWswHhcNMDYwNzI3MTQxMzI4WhcNMDcwNzI3MTQxMzI4WjBbMQswCQYDVQQGEwJV\r\n - SzERMA8GA1UEChMIZVNjaWVuY2UxEzARBgNVBAsTCk1hbmNoZXN0ZXIxCzAJBgNV\r\n - BAcTmrsogriqMWLAk1DMRcwFQYDVQQDEw5taWNoYWVsIHBhcmQYJKoZIhvcNAQEB\r\n - BQADggEPADCCAQoCggEBANPEQBgl1IaKdSS1TbhF3hEXSl72G9J+WC/1R64fAcEF\r\n - W51rEyFYiIeZGx/BVzwXbeBoNUK41OK65sxGuflMo5gLflbwJtHBRIEKAfVVp3YR\r\n - gW7cMA/s/XKgL1GEC7rQw8lIZT8RApukCGqOVHSi/F1SiFlPDxuDfmdiNzL31+sL\r\n - 0iwHDdNkGjy5pyBSB8Y79dsSJtCW/iaLB0/n8Sj7HgvvZJ7x0fr+RQjYOUUfrePP\r\n - u2MSpFyf+9BbC/aXgaZuiCvSR+8Snv3xApQY+fULK/xY8h8Ua51iXoQ5jrgu2SqR\r\n - wgA7BUi3G8LFzMBl8FRCDYGUDy7M6QaHXx1ZWIPWNKsCAwEAAaOCAiQwggIgMAwG\r\n - 1UdEwEB/wQCMAAwEQYJYIZIAYb4QgHTTPAQDAgWgMA4GA1UdDwEB/wQEAwID6DAs\r\n - BglghkgBhvhCAQ0EHxYdVUsgZS1TY2llbmNlIFVzZXIgQ2VydGlmaWNhdGUwHQYD\r\n - VR0OBBYEFDTt/sf9PeMaZDHkUIldrDYMNTBZMIGaBgNVHSMEgZIwgY+AFAI4qxGj\r\n - loCLDdMVKwiljjDastqooXSkcjBwMQswCQYDVQQGEwJVSzERMA8GA1UEChMIZVNj\r\n - aWVuY2UxEjAQBgNVBAsTCUF1dGhvcml0eTELMAkGA1UEAxMCQ0ExLTArBgkqhkiG\r\n - 9w0BCQEWHmNhLW9wZXJhdG9yQGdyaWQtc3VwcG9ydC5hYy51a4IBADApBgNVHRIE\r\n - IjAggR5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMudWswGQYDVR0gBBIwEDAO\r\n - BgwrBgEEAdkvAQEBAQYwPQYJYIZIAYb4QgEEBDAWLmh0dHA6Ly9jYS5ncmlkLXN1\r\n - cHBvcnQuYWMudmT4sopwqlBWsvcHViL2NybC9jYWNybC5jcmwwPQYJYIZIAYb4Qg\r\n - EDBDAWLmh0dHA6Ly9jYS5ncmlkLXN1cHBvcnQuYWMudWsvcHViL2NybC9jYWNybC\r\n - 5jcmwwPwYDVR0fBDgwNjA0oDKgMIYuaHR0cDovL2NhLmdyaWQt5hYy51ay9wdWIv\r\n - Y3JsL2NhY3JsLmNybDANBgkqhkiG9w0BAQUFAAOCAQEAS/U4iiooBENGW/Hwmmd3\r\n - XCy6Zrt08YjKCzGNjorT98g8uGsqYjSxv/hmi0qlnlHs+k/3Iobc3LjS5AMYr5L8\r\n - UO7OSkgFFlLHQyC9JzPfmLCAugvzEbyv4Olnsr8hbxF1MbKZoQxUZtMVu29wjfXk\r\n - hTeApBv7eaKCWpSp7MCbvgzm74izKhu3vlDk9w6qVrxePfGgpKPqfHiOoGhFnbTK\r\n - wTC6o2xq5y0qZ03JonF7OJspEd3I5zKY3E+ov7/ZhW6DqT8UFvsAdjvQbXyhV8Eu\r\n - Yhixw1aKEPzNjNowuIseVogKOLXxWI5vAi5HgXdS0/ES5gDGsABo4fqovUKlgop3\r\n - RA==\r\n - -----END CERTIFICATE-----""".replace("\n\n", "\n") +certificate = """-----BEGIN CERTIFICATE----- + MIIFbTCCBFWgAwIBAgICH4cwDQYJKoZIhvcNAQEFBQAwcDELMAkGA1UEBhMCVUsx + ETAPBgNVBAoTCGVTY2llbmNlMRIwEAYDVQQLEwlBdXRob3JpdHkxCzAJBgNVBAMT + AkNBMS0wKwYJKoZIhvcNAQkBFh5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMu + dWswHhcNMDYwNzI3MTQxMzI4WhcNMDcwNzI3MTQxMzI4WjBbMQswCQYDVQQGEwJV + SzERMA8GA1UEChMIZVNjaWVuY2UxEzARBgNVBAsTCk1hbmNoZXN0ZXIxCzAJBgNV + BAcTmrsogriqMWLAk1DMRcwFQYDVQQDEw5taWNoYWVsIHBhcmQYJKoZIhvcNAQEB + BQADggEPADCCAQoCggEBANPEQBgl1IaKdSS1TbhF3hEXSl72G9J+WC/1R64fAcEF + W51rEyFYiIeZGx/BVzwXbeBoNUK41OK65sxGuflMo5gLflbwJtHBRIEKAfVVp3YR + gW7cMA/s/XKgL1GEC7rQw8lIZT8RApukCGqOVHSi/F1SiFlPDxuDfmdiNzL31+sL + 0iwHDdNkGjy5pyBSB8Y79dsSJtCW/iaLB0/n8Sj7HgvvZJ7x0fr+RQjYOUUfrePP + u2MSpFyf+9BbC/aXgaZuiCvSR+8Snv3xApQY+fULK/xY8h8Ua51iXoQ5jrgu2SqR + wgA7BUi3G8LFzMBl8FRCDYGUDy7M6QaHXx1ZWIPWNKsCAwEAAaOCAiQwggIgMAwG + 1UdEwEB/wQCMAAwEQYJYIZIAYb4QgHTTPAQDAgWgMA4GA1UdDwEB/wQEAwID6DAs + BglghkgBhvhCAQ0EHxYdVUsgZS1TY2llbmNlIFVzZXIgQ2VydGlmaWNhdGUwHQYD + VR0OBBYEFDTt/sf9PeMaZDHkUIldrDYMNTBZMIGaBgNVHSMEgZIwgY+AFAI4qxGj + loCLDdMVKwiljjDastqooXSkcjBwMQswCQYDVQQGEwJVSzERMA8GA1UEChMIZVNj + aWVuY2UxEjAQBgNVBAsTCUF1dGhvcml0eTELMAkGA1UEAxMCQ0ExLTArBgkqhkiG + 9w0BCQEWHmNhLW9wZXJhdG9yQGdyaWQtc3VwcG9ydC5hYy51a4IBADApBgNVHRIE + IjAggR5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMudWswGQYDVR0gBBIwEDAO + BgwrBgEEAdkvAQEBAQYwPQYJYIZIAYb4QgEEBDAWLmh0dHA6Ly9jYS5ncmlkLXN1 + cHBvcnQuYWMudmT4sopwqlBWsvcHViL2NybC9jYWNybC5jcmwwPQYJYIZIAYb4Qg + EDBDAWLmh0dHA6Ly9jYS5ncmlkLXN1cHBvcnQuYWMudWsvcHViL2NybC9jYWNybC + 5jcmwwPwYDVR0fBDgwNjA0oDKgMIYuaHR0cDovL2NhLmdyaWQt5hYy51ay9wdWIv + Y3JsL2NhY3JsLmNybDANBgkqhkiG9w0BAQUFAAOCAQEAS/U4iiooBENGW/Hwmmd3 + XCy6Zrt08YjKCzGNjorT98g8uGsqYjSxv/hmi0qlnlHs+k/3Iobc3LjS5AMYr5L8 + UO7OSkgFFlLHQyC9JzPfmLCAugvzEbyv4Olnsr8hbxF1MbKZoQxUZtMVu29wjfXk + hTeApBv7eaKCWpSp7MCbvgzm74izKhu3vlDk9w6qVrxePfGgpKPqfHiOoGhFnbTK + wTC6o2xq5y0qZ03JonF7OJspEd3I5zKY3E+ov7/ZhW6DqT8UFvsAdjvQbXyhV8Eu + Yhixw1aKEPzNjNowuIseVogKOLXxWI5vAi5HgXdS0/ES5gDGsABo4fqovUKlgop3 + RA== + -----END CERTIFICATE-----""".replace("\n", "") request = { "method": "GET", diff --git a/tests/requests/valid/031.http b/tests/requests/valid/031.http new file mode 100644 index 000000000..cd1ab7fcb --- /dev/null +++ b/tests/requests/valid/031.http @@ -0,0 +1,2 @@ +-blargh /foo HTTP/1.1\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/valid/031.py b/tests/requests/valid/031.py new file mode 100644 index 000000000..9691a002b --- /dev/null +++ b/tests/requests/valid/031.py @@ -0,0 +1,7 @@ +request = { + "method": "-BLARGH", + "uri": uri("/foo"), + "version": (1, 1), + "headers": [], + "body": b"" +} From f0c91cca484820d1034f3a5278c0662aed2a23ea Mon Sep 17 00:00:00 2001 From: Tomi Belan Date: Sun, 22 May 2022 00:42:55 +0200 Subject: [PATCH 140/164] Check SCRIPT_NAME is at the request path's beginning --- gunicorn/http/errors.py | 9 +++++++++ gunicorn/http/wsgi.py | 8 ++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/gunicorn/http/errors.py b/gunicorn/http/errors.py index 7839ef05a..05abd1ab0 100644 --- a/gunicorn/http/errors.py +++ b/gunicorn/http/errors.py @@ -22,6 +22,15 @@ def __str__(self): return "No more data after: %r" % self.buf +class ConfigurationProblem(ParseException): + def __init__(self, info): + self.info = info + self.code = 500 + + def __str__(self): + return "Configuration problem: %s" % self.info + + class InvalidRequestLine(ParseException): def __init__(self, req): self.req = req diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 7fca61422..bafed49eb 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -10,7 +10,7 @@ import sys from gunicorn.http.message import TOKEN_RE -from gunicorn.http.errors import InvalidHeader, InvalidHeaderName +from gunicorn.http.errors import ConfigurationProblem, InvalidHeader, InvalidHeaderName from gunicorn import SERVER_SOFTWARE, SERVER from gunicorn import util @@ -182,7 +182,11 @@ def create(req, sock, client, server, cfg): # set the path and script name path_info = req.path if script_name: - path_info = path_info.split(script_name, 1)[1] + if not path_info.startswith(script_name): + raise ConfigurationProblem( + "Request path %r does not start with SCRIPT_NAME %r" % + (path_info, script_name)) + path_info = path_info[len(script_name):] environ['PATH_INFO'] = util.unquote_to_wsgi_str(path_info) environ['SCRIPT_NAME'] = script_name From 13027ef797edba55967f366ec958a9a03b3d345b Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Wed, 6 Dec 2023 14:22:18 +0000 Subject: [PATCH 141/164] Create SECURITY.md --- SECURITY.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..3d1923540 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,22 @@ +# Security Policy + +## Reporting a Vulnerability + +**Please note that public Github issues are open for everyone to see!** + +If you believe you are found a problem in Gunicorn software, examples or documentation, we encourage you to send your report privately via email, or via Github using the *Report a vulnerability* button in the [Security](https://github.com/benoitc/gunicorn/security) section. + +## Supported Releases + +At this time, **only the latest release** receives any security attention whatsoever. + +| Version | Status | +| ------- | ------------------ | +| latest release | :white_check_mark: | +| 21.2.0 | :x: | +| 20.0.0 | :x: | +| < 20.0 | :x: | + +## Python Versions + +Gunicorn runs on Python 3.7+, we *highly recommend* the latest release of a [supported series](https://devguide.python.org/version/) and will not prioritize issues exclusively affecting in EoL environments. From 42dd4190ac01e5cba017948ab882c71017ea18d6 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 7 Dec 2023 18:46:13 +0100 Subject: [PATCH 142/164] test: verify TOKEN_RE against common HTTP Methods --- tests/test_http.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_http.py b/tests/test_http.py index b6ca46b22..0eb694601 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -10,6 +10,17 @@ from gunicorn.http.wsgi import Response from gunicorn.http.unreader import Unreader, IterUnreader, SocketUnreader from gunicorn.http.errors import InvalidHeader, InvalidHeaderName +from gunicorn.http.message import TOKEN_RE + + +def test_method_pattern(): + assert TOKEN_RE.fullmatch("GET") + assert TOKEN_RE.fullmatch("MKCALENDAR") + assert not TOKEN_RE.fullmatch("GET:") + assert not TOKEN_RE.fullmatch("GET;") + RFC9110_5_6_2_TOKEN_DELIM = r'"(),/:;<=>?@[\]{}' + for bad_char in RFC9110_5_6_2_TOKEN_DELIM: + assert not TOKEN_RE.match(bad_char) def assert_readline(payload, size, expected): From b2846783d799bdde24fa209d767bd4ffb54cc1be Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 7 Dec 2023 18:46:31 +0100 Subject: [PATCH 143/164] strict: header field validation: stop casefolding * refusing lowercase and ASCII 0x23 (#) had been partially enforced before * do not casefold by default, HTTP methods are case sensitive --- gunicorn/config.py | 46 +++++++++++++++++++++++++++- gunicorn/http/message.py | 26 +++++++++++++--- tests/requests/invalid/003b.http | 2 ++ tests/requests/invalid/003b.py | 2 ++ tests/requests/invalid/003c.http | 2 ++ tests/requests/invalid/003c.py | 2 ++ tests/requests/valid/031.http | 4 +-- tests/requests/valid/031compat.http | 2 ++ tests/requests/valid/031compat.py | 13 ++++++++ tests/requests/valid/031compat2.http | 2 ++ tests/requests/valid/031compat2.py | 12 ++++++++ 11 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 tests/requests/invalid/003b.http create mode 100644 tests/requests/invalid/003b.py create mode 100644 tests/requests/invalid/003c.http create mode 100644 tests/requests/invalid/003c.py create mode 100644 tests/requests/valid/031compat.http create mode 100644 tests/requests/valid/031compat.py create mode 100644 tests/requests/valid/031compat2.http create mode 100644 tests/requests/valid/031compat2.py diff --git a/gunicorn/config.py b/gunicorn/config.py index 84e7619e4..808a4b046 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -2254,5 +2254,49 @@ class StripHeaderSpaces(Setting): This is known to induce vulnerabilities and is not compliant with the HTTP/1.1 standard. See https://portswigger.net/research/http-desync-attacks-request-smuggling-reborn. - Use with care and only if necessary. + Use with care and only if necessary. May be removed in a future version. + + .. versionadded:: 20.0.1 + """ + + +class PermitUnconventionalHTTPMethod(Setting): + name = "permit_unconventional_http_method" + section = "Server Mechanics" + cli = ["--permit-unconventional-http-method"] + validator = validate_bool + action = "store_true" + default = False + desc = """\ + Permit HTTP methods not matching conventions, such as IANA registration guidelines + + This permits request methods of length less than 3 or more than 20, + methods with lowercase characters or methods containing the # character. + HTTP methods are case sensitive by definition, and merely uppercase by convention. + + This option is provided to diagnose backwards-incompatible changes. + + Use with care and only if necessary. May be removed in a future version. + + .. versionadded:: 22.0.0 """ + + +class CasefoldHTTPMethod(Setting): + name = "casefold_http_method" + section = "Server Mechanics" + cli = ["--casefold-http-method"] + validator = validate_bool + action = "store_true" + default = False + desc = """\ + Transform received HTTP methods to uppercase + + HTTP methods are case sensitive by definition, and merely uppercase by convention. + + This option is provided because previous versions of gunicorn defaulted to this behaviour. + + Use with care and only if necessary. May be removed in a future version. + + .. versionadded:: 22.0.0 + """ diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index db3d44bb2..2aa021c8d 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -21,7 +21,10 @@ MAX_HEADERS = 32768 DEFAULT_MAX_HEADERFIELD_SIZE = 8190 -TOKEN_RE = re.compile(r"[!#$%&'*+\-.\^_`|~0-9a-zA-Z]+") +# verbosely on purpose, avoid backslash ambiguity +RFC9110_5_6_2_TOKEN_SPECIALS = r"!#$%&'*+-.^_`|~" +TOKEN_RE = re.compile(r"[%s0-9a-zA-Z]+" % (re.escape(RFC9110_5_6_2_TOKEN_SPECIALS))) +METHOD_BADCHAR_RE = re.compile("[a-z#]") VERSION_RE = re.compile(r"HTTP/(\d+)\.(\d+)") @@ -331,10 +334,23 @@ def parse_request_line(self, line_bytes): if len(bits) != 3: raise InvalidRequestLine(bytes_to_str(line_bytes)) - # Method - if not TOKEN_RE.fullmatch(bits[0]): - raise InvalidRequestMethod(bits[0]) - self.method = bits[0].upper() + # Method: RFC9110 Section 9 + self.method = bits[0] + + # nonstandard restriction, suitable for all IANA registered methods + # partially enforced in previous gunicorn versions + if not self.cfg.permit_unconventional_http_method: + if METHOD_BADCHAR_RE.search(self.method): + raise InvalidRequestMethod(self.method) + if not 3 <= len(bits[0]) <= 20: + raise InvalidRequestMethod(self.method) + # standard restriction: RFC9110 token + if not TOKEN_RE.fullmatch(self.method): + raise InvalidRequestMethod(self.method) + # nonstandard and dangerous + # methods are merely uppercase by convention, no case-insensitive treatment is intended + if self.cfg.casefold_http_method: + self.method = self.method.upper() # URI self.uri = bits[1] diff --git a/tests/requests/invalid/003b.http b/tests/requests/invalid/003b.http new file mode 100644 index 000000000..e05fd9897 --- /dev/null +++ b/tests/requests/invalid/003b.http @@ -0,0 +1,2 @@ +bla:rgh /foo HTTP/1.1\r\n +\r\n diff --git a/tests/requests/invalid/003b.py b/tests/requests/invalid/003b.py new file mode 100644 index 000000000..86a0774e5 --- /dev/null +++ b/tests/requests/invalid/003b.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidRequestMethod +request = InvalidRequestMethod \ No newline at end of file diff --git a/tests/requests/invalid/003c.http b/tests/requests/invalid/003c.http new file mode 100644 index 000000000..98bd20bf1 --- /dev/null +++ b/tests/requests/invalid/003c.http @@ -0,0 +1,2 @@ +-bl /foo HTTP/1.1\r\n +\r\n diff --git a/tests/requests/invalid/003c.py b/tests/requests/invalid/003c.py new file mode 100644 index 000000000..1dac27c04 --- /dev/null +++ b/tests/requests/invalid/003c.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidRequestMethod +request = InvalidRequestMethod diff --git a/tests/requests/valid/031.http b/tests/requests/valid/031.http index cd1ab7fcb..ab3529da3 100644 --- a/tests/requests/valid/031.http +++ b/tests/requests/valid/031.http @@ -1,2 +1,2 @@ --blargh /foo HTTP/1.1\r\n -\r\n \ No newline at end of file +-BLARGH /foo HTTP/1.1\r\n +\r\n diff --git a/tests/requests/valid/031compat.http b/tests/requests/valid/031compat.http new file mode 100644 index 000000000..cd1ab7fcb --- /dev/null +++ b/tests/requests/valid/031compat.http @@ -0,0 +1,2 @@ +-blargh /foo HTTP/1.1\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/valid/031compat.py b/tests/requests/valid/031compat.py new file mode 100644 index 000000000..424b7cb47 --- /dev/null +++ b/tests/requests/valid/031compat.py @@ -0,0 +1,13 @@ +from gunicorn.config import Config + +cfg = Config() +cfg.set("permit_unconventional_http_method", True) +cfg.set("casefold_http_method", True) + +request = { + "method": "-BLARGH", + "uri": uri("/foo"), + "version": (1, 1), + "headers": [], + "body": b"" +} diff --git a/tests/requests/valid/031compat2.http b/tests/requests/valid/031compat2.http new file mode 100644 index 000000000..dcbf4f134 --- /dev/null +++ b/tests/requests/valid/031compat2.http @@ -0,0 +1,2 @@ +-blargh /foo HTTP/1.1\r\n +\r\n diff --git a/tests/requests/valid/031compat2.py b/tests/requests/valid/031compat2.py new file mode 100644 index 000000000..594a8b6a8 --- /dev/null +++ b/tests/requests/valid/031compat2.py @@ -0,0 +1,12 @@ +from gunicorn.config import Config + +cfg = Config() +cfg.set("permit_unconventional_http_method", True) + +request = { + "method": "-blargh", + "uri": uri("/foo"), + "version": (1, 1), + "headers": [], + "body": b"" +} From 72b8970dbf2bf3444eb2e8b12aeff1a3d5922a9a Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 7 Dec 2023 08:11:34 +0100 Subject: [PATCH 144/164] silently drop or refuse header names w/ underscore Ambiguous mappings open a bottomless pit of "what is user input and what is proxy input" confusion. Default to what everyone else has been doing for years now, silently drop. see also https://nginx.org/r/underscores_in_headers --- gunicorn/config.py | 44 ++++++++++++++++++++++++++ gunicorn/http/message.py | 17 ++++++++++ gunicorn/http/wsgi.py | 2 ++ tests/requests/invalid/040.http | 6 ++++ tests/requests/invalid/040.py | 7 ++++ tests/requests/invalid/chunked_07.http | 10 ++++++ tests/requests/invalid/chunked_07.py | 7 ++++ tests/requests/valid/040.http | 6 ++++ tests/requests/valid/040.py | 9 ++++++ tests/requests/valid/040_compat.http | 6 ++++ tests/requests/valid/040_compat.py | 16 ++++++++++ 11 files changed, 130 insertions(+) create mode 100644 tests/requests/invalid/040.http create mode 100644 tests/requests/invalid/040.py create mode 100644 tests/requests/invalid/chunked_07.http create mode 100644 tests/requests/invalid/chunked_07.py create mode 100644 tests/requests/valid/040.http create mode 100644 tests/requests/valid/040.py create mode 100644 tests/requests/valid/040_compat.http create mode 100644 tests/requests/valid/040_compat.py diff --git a/gunicorn/config.py b/gunicorn/config.py index 808a4b046..50baeb616 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -2300,3 +2300,47 @@ class CasefoldHTTPMethod(Setting): .. versionadded:: 22.0.0 """ + + +def validate_header_map_behaviour(val): + # FIXME: refactor all of this subclassing stdlib argparse + + if val is None: + return + + if not isinstance(val, str): + raise TypeError("Invalid type for casting: %s" % val) + if val.lower().strip() == "drop": + return "drop" + elif val.lower().strip() == "refuse": + return "refuse" + elif val.lower().strip() == "dangerous": + return "dangerous" + else: + raise ValueError("Invalid header map behaviour: %s" % val) + + +class HeaderMap(Setting): + name = "header_map" + section = "Server Mechanics" + cli = ["--header-map"] + validator = validate_header_map_behaviour + default = "drop" + desc = """\ + Configure how header field names are mapped into environ + + Headers containing underscores are permitted by RFC9110, + but gunicorn joining headers of different names into + the same environment variable will dangerously confuse applications as to which is which. + + The safe default ``drop`` is to silently drop headers that cannot be unambiguously mapped. + The value ``refuse`` will return an error if a request contains *any* such header. + The value ``dangerous`` matches the previous, not advisabble, behaviour of mapping different + header field names into the same environ name. + + Use with care and only if necessary and after considering if your problem could + instead be solved by specifically renaming or rewriting only the intended headers + on a proxy in front of Gunicorn. + + .. versionadded:: 22.0.0 + """ diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 2aa021c8d..59a780f6c 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -120,6 +120,23 @@ def parse_headers(self, data): scheme_header = True self.scheme = scheme + # ambiguous mapping allows fooling downstream, e.g. merging non-identical headers: + # X-Forwarded-For: 2001:db8::ha:cc:ed + # X_Forwarded_For: 127.0.0.1,::1 + # HTTP_X_FORWARDED_FOR = 2001:db8::ha:cc:ed,127.0.0.1,::1 + # Only modify after fixing *ALL* header transformations; network to wsgi env + if "_" in name: + if self.cfg.header_map == "dangerous": + # as if we did not know we cannot safely map this + pass + elif self.cfg.header_map == "drop": + # almost as if it never had been there + # but still counts against resource limits + continue + else: + # fail-safe fallthrough: refuse + raise InvalidHeaderName(name) + headers.append((name, value)) return headers diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index bafed49eb..6f3d9b68f 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -135,6 +135,8 @@ def create(req, sock, client, server, cfg): environ['CONTENT_LENGTH'] = hdr_value continue + # do not change lightly, this is a common source of security problems + # RFC9110 Section 17.10 discourages ambiguous or incomplete mappings key = 'HTTP_' + hdr_name.replace('-', '_') if key in environ: hdr_value = "%s,%s" % (environ[key], hdr_value) diff --git a/tests/requests/invalid/040.http b/tests/requests/invalid/040.http new file mode 100644 index 000000000..be03929d5 --- /dev/null +++ b/tests/requests/invalid/040.http @@ -0,0 +1,6 @@ +GET /keep/same/as?invalid/040 HTTP/1.0\r\n +Transfer_Encoding: tricked\r\n +Content-Length: 7\r\n +Content_Length: -1E23\r\n +\r\n +tricked\r\n diff --git a/tests/requests/invalid/040.py b/tests/requests/invalid/040.py new file mode 100644 index 000000000..643289fab --- /dev/null +++ b/tests/requests/invalid/040.py @@ -0,0 +1,7 @@ +from gunicorn.http.errors import InvalidHeaderName +from gunicorn.config import Config + +cfg = Config() +cfg.set("header_map", "refuse") + +request = InvalidHeaderName diff --git a/tests/requests/invalid/chunked_07.http b/tests/requests/invalid/chunked_07.http new file mode 100644 index 000000000..a78db48b7 --- /dev/null +++ b/tests/requests/invalid/chunked_07.http @@ -0,0 +1,10 @@ +POST /chunked_ambiguous_header_mapping HTTP/1.1\r\n +Transfer_Encoding: gzip\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +0\r\n +\r\n diff --git a/tests/requests/invalid/chunked_07.py b/tests/requests/invalid/chunked_07.py new file mode 100644 index 000000000..643289fab --- /dev/null +++ b/tests/requests/invalid/chunked_07.py @@ -0,0 +1,7 @@ +from gunicorn.http.errors import InvalidHeaderName +from gunicorn.config import Config + +cfg = Config() +cfg.set("header_map", "refuse") + +request = InvalidHeaderName diff --git a/tests/requests/valid/040.http b/tests/requests/valid/040.http new file mode 100644 index 000000000..be03929d5 --- /dev/null +++ b/tests/requests/valid/040.http @@ -0,0 +1,6 @@ +GET /keep/same/as?invalid/040 HTTP/1.0\r\n +Transfer_Encoding: tricked\r\n +Content-Length: 7\r\n +Content_Length: -1E23\r\n +\r\n +tricked\r\n diff --git a/tests/requests/valid/040.py b/tests/requests/valid/040.py new file mode 100644 index 000000000..7c2243d9f --- /dev/null +++ b/tests/requests/valid/040.py @@ -0,0 +1,9 @@ +request = { + "method": "GET", + "uri": uri("/keep/same/as?invalid/040"), + "version": (1, 0), + "headers": [ + ("CONTENT-LENGTH", "7") + ], + "body": b'tricked' +} diff --git a/tests/requests/valid/040_compat.http b/tests/requests/valid/040_compat.http new file mode 100644 index 000000000..be03929d5 --- /dev/null +++ b/tests/requests/valid/040_compat.http @@ -0,0 +1,6 @@ +GET /keep/same/as?invalid/040 HTTP/1.0\r\n +Transfer_Encoding: tricked\r\n +Content-Length: 7\r\n +Content_Length: -1E23\r\n +\r\n +tricked\r\n diff --git a/tests/requests/valid/040_compat.py b/tests/requests/valid/040_compat.py new file mode 100644 index 000000000..5f13487c4 --- /dev/null +++ b/tests/requests/valid/040_compat.py @@ -0,0 +1,16 @@ +from gunicorn.config import Config + +cfg = Config() +cfg.set("header_map", "dangerous") + +request = { + "method": "GET", + "uri": uri("/keep/same/as?invalid/040"), + "version": (1, 0), + "headers": [ + ("TRANSFER_ENCODING", "tricked"), + ("CONTENT-LENGTH", "7"), + ("CONTENT_LENGTH", "-1E23"), + ], + "body": b'tricked' +} From 0b10cbab1d6368fcab2d5a7b6fe359a6cecc81a7 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 7 Dec 2023 08:16:33 +0100 Subject: [PATCH 145/164] unconditionally log request error Somehow exception logging was conditional on successful request uri parsing. Add it back for the other branch. --- gunicorn/workers/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index f321dd2d4..f97d923c7 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -251,6 +251,8 @@ def handle_error(self, req, client, addr, exc): else: if hasattr(req, "uri"): self.log.exception("Error handling request %s", req.uri) + else: + self.log.exception("Error handling request (no URI read)") status_int = 500 reason = "Internal Server Error" mesg = "" From ac29c9b0a758d21f1e0fb3b3457239e523fa9f1d Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 7 Dec 2023 09:22:30 +0100 Subject: [PATCH 146/164] fail-safe on unsupported request framing If we promise wsgi.input_terminated, we better get it right - or not at all. * chunked encoding on HTTP <= 1.1 * chunked not last transfer coding * multiple chinked codings * any unknown codings (yes, this too! because we do not detect unusual syntax that is still chunked) * empty coding (plausibly harmless, but not see in real life anyway - refused, for the moment) --- gunicorn/config.py | 18 ++++++++++ gunicorn/http/errors.py | 9 +++++ gunicorn/http/message.py | 45 +++++++++++++++++++++++++ tests/requests/invalid/chunked_01.http | 12 +++++++ tests/requests/invalid/chunked_01.py | 2 ++ tests/requests/invalid/chunked_02.http | 9 +++++ tests/requests/invalid/chunked_02.py | 2 ++ tests/requests/invalid/chunked_03.http | 8 +++++ tests/requests/invalid/chunked_03.py | 2 ++ tests/requests/invalid/chunked_04.http | 11 ++++++ tests/requests/invalid/chunked_04.py | 2 ++ tests/requests/invalid/chunked_05.http | 11 ++++++ tests/requests/invalid/chunked_05.py | 2 ++ tests/requests/invalid/chunked_06.http | 9 +++++ tests/requests/invalid/chunked_06.py | 2 ++ tests/requests/invalid/chunked_08.http | 9 +++++ tests/requests/invalid/chunked_08.py | 2 ++ tests/requests/invalid/nonascii_01.http | 4 +++ tests/requests/invalid/nonascii_01.py | 5 +++ tests/requests/invalid/nonascii_02.http | 4 +++ tests/requests/invalid/nonascii_02.py | 5 +++ tests/requests/invalid/nonascii_04.http | 5 +++ tests/requests/invalid/nonascii_04.py | 5 +++ tests/requests/invalid/prefix_01.http | 2 ++ tests/requests/invalid/prefix_01.py | 2 ++ tests/requests/invalid/prefix_02.http | 2 ++ tests/requests/invalid/prefix_02.py | 2 ++ tests/requests/invalid/prefix_03.http | 4 +++ tests/requests/invalid/prefix_03.py | 5 +++ tests/requests/invalid/prefix_04.http | 5 +++ tests/requests/invalid/prefix_04.py | 5 +++ tests/requests/invalid/prefix_05.http | 4 +++ tests/requests/invalid/prefix_05.py | 5 +++ tests/requests/valid/025.http | 9 +++-- tests/requests/valid/025.py | 6 +++- tests/requests/valid/025compat.http | 18 ++++++++++ tests/requests/valid/025compat.py | 27 +++++++++++++++ tests/requests/valid/029.http | 2 +- tests/requests/valid/029.py | 2 +- tests/treq.py | 4 ++- 40 files changed, 281 insertions(+), 6 deletions(-) create mode 100644 tests/requests/invalid/chunked_01.http create mode 100644 tests/requests/invalid/chunked_01.py create mode 100644 tests/requests/invalid/chunked_02.http create mode 100644 tests/requests/invalid/chunked_02.py create mode 100644 tests/requests/invalid/chunked_03.http create mode 100644 tests/requests/invalid/chunked_03.py create mode 100644 tests/requests/invalid/chunked_04.http create mode 100644 tests/requests/invalid/chunked_04.py create mode 100644 tests/requests/invalid/chunked_05.http create mode 100644 tests/requests/invalid/chunked_05.py create mode 100644 tests/requests/invalid/chunked_06.http create mode 100644 tests/requests/invalid/chunked_06.py create mode 100644 tests/requests/invalid/chunked_08.http create mode 100644 tests/requests/invalid/chunked_08.py create mode 100644 tests/requests/invalid/nonascii_01.http create mode 100644 tests/requests/invalid/nonascii_01.py create mode 100644 tests/requests/invalid/nonascii_02.http create mode 100644 tests/requests/invalid/nonascii_02.py create mode 100644 tests/requests/invalid/nonascii_04.http create mode 100644 tests/requests/invalid/nonascii_04.py create mode 100644 tests/requests/invalid/prefix_01.http create mode 100644 tests/requests/invalid/prefix_01.py create mode 100644 tests/requests/invalid/prefix_02.http create mode 100644 tests/requests/invalid/prefix_02.py create mode 100644 tests/requests/invalid/prefix_03.http create mode 100644 tests/requests/invalid/prefix_03.py create mode 100644 tests/requests/invalid/prefix_04.http create mode 100644 tests/requests/invalid/prefix_04.py create mode 100644 tests/requests/invalid/prefix_05.http create mode 100644 tests/requests/invalid/prefix_05.py create mode 100644 tests/requests/valid/025compat.http create mode 100644 tests/requests/valid/025compat.py diff --git a/gunicorn/config.py b/gunicorn/config.py index 50baeb616..be9bb001c 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -2344,3 +2344,21 @@ class HeaderMap(Setting): .. versionadded:: 22.0.0 """ + + +class TolerateDangerousFraming(Setting): + name = "tolerate_dangerous_framing" + section = "Server Mechanics" + cli = ["--tolerate-dangerous-framing"] + validator = validate_bool + action = "store_true" + default = False + desc = """\ + Process requests with both Transfer-Encoding and Content-Length + + This is known to induce vulnerabilities, but not strictly forbidden by RFC9112. + + Use with care and only if necessary. May be removed in a future version. + + .. versionadded:: 22.0.0 + """ diff --git a/gunicorn/http/errors.py b/gunicorn/http/errors.py index 05abd1ab0..340f0473c 100644 --- a/gunicorn/http/errors.py +++ b/gunicorn/http/errors.py @@ -73,6 +73,15 @@ def __str__(self): return "Invalid HTTP header name: %r" % self.hdr +class UnsupportedTransferCoding(ParseException): + def __init__(self, hdr): + self.hdr = hdr + self.code = 501 + + def __str__(self): + return "Unsupported transfer coding: %r" % self.hdr + + class InvalidChunkSize(IOError): def __init__(self, data): self.data = data diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 59a780f6c..75b36e330 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -12,6 +12,7 @@ InvalidHeader, InvalidHeaderName, NoMoreData, InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, LimitRequestLine, LimitRequestHeaders, + UnsupportedTransferCoding, ) from gunicorn.http.errors import InvalidProxyLine, ForbiddenProxyRequest from gunicorn.http.errors import InvalidSchemeHeaders @@ -39,6 +40,7 @@ def __init__(self, cfg, unreader, peer_addr): self.trailers = [] self.body = None self.scheme = "https" if cfg.is_ssl else "http" + self.must_close = False # set headers limits self.limit_request_fields = cfg.limit_request_fields @@ -58,6 +60,9 @@ def __init__(self, cfg, unreader, peer_addr): self.unreader.unread(unused) self.set_body_reader() + def force_close(self): + self.must_close = True + def parse(self, unreader): raise NotImplementedError() @@ -152,9 +157,47 @@ def set_body_reader(self): content_length = value elif name == "TRANSFER-ENCODING": if value.lower() == "chunked": + # DANGER: transer codings stack, and stacked chunking is never intended + if chunked: + raise InvalidHeader("TRANSFER-ENCODING", req=self) chunked = True + elif value.lower() == "identity": + # does not do much, could still plausibly desync from what the proxy does + # safe option: nuke it, its never needed + if chunked: + raise InvalidHeader("TRANSFER-ENCODING", req=self) + elif value.lower() == "": + # lacking security review on this case + # offer the option to restore previous behaviour, but refuse by default, for now + self.force_close() + if not self.cfg.tolerate_dangerous_framing: + raise UnsupportedTransferCoding(value) + # DANGER: do not change lightly; ref: request smuggling + # T-E is a list and we *could* support correctly parsing its elements + # .. but that is only safe after getting all the edge cases right + # .. for which no real-world need exists, so best to NOT open that can of worms + else: + self.force_close() + # even if parser is extended, retain this branch: + # the "chunked not last" case remains to be rejected! + raise UnsupportedTransferCoding(value) if chunked: + # two potentially dangerous cases: + # a) CL + TE (TE overrides CL.. only safe if the recipient sees it that way too) + # b) chunked HTTP/1.0 (always faulty) + if self.version < (1, 1): + # framing wonky, see RFC 9112 Section 6.1 + self.force_close() + if not self.cfg.tolerate_dangerous_framing: + raise InvalidHeader("TRANSFER-ENCODING", req=self) + if content_length is not None: + # we cannot be certain the message framing we understood matches proxy intent + # -> whatever happens next, remaining input must not be trusted + self.force_close() + # either processing or rejecting is permitted in RFC 9112 Section 6.1 + if not self.cfg.tolerate_dangerous_framing: + raise InvalidHeader("CONTENT-LENGTH", req=self) self.body = Body(ChunkedReader(self, self.unreader)) elif content_length is not None: try: @@ -173,6 +216,8 @@ def set_body_reader(self): self.body = Body(EOFReader(self.unreader)) def should_close(self): + if self.must_close: + return True for (h, v) in self.headers: if h == "CONNECTION": v = v.lower().strip(" \t") diff --git a/tests/requests/invalid/chunked_01.http b/tests/requests/invalid/chunked_01.http new file mode 100644 index 000000000..7a8e55d2e --- /dev/null +++ b/tests/requests/invalid/chunked_01.http @@ -0,0 +1,12 @@ +POST /chunked_w_underscore_chunk_size HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\r\n +hello\r\n +6_0\r\n + world\r\n +0\r\n +\r\n +POST /after HTTP/1.1\r\n +Transfer-Encoding: identity\r\n +\r\n diff --git a/tests/requests/invalid/chunked_01.py b/tests/requests/invalid/chunked_01.py new file mode 100644 index 000000000..0571e1183 --- /dev/null +++ b/tests/requests/invalid/chunked_01.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidChunkSize +request = InvalidChunkSize diff --git a/tests/requests/invalid/chunked_02.http b/tests/requests/invalid/chunked_02.http new file mode 100644 index 000000000..9ae49e525 --- /dev/null +++ b/tests/requests/invalid/chunked_02.http @@ -0,0 +1,9 @@ +POST /chunked_with_prefixed_value HTTP/1.1\r\n +Content-Length: 12\r\n +Transfer-Encoding: \tchunked\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +\r\n diff --git a/tests/requests/invalid/chunked_02.py b/tests/requests/invalid/chunked_02.py new file mode 100644 index 000000000..1541eb70b --- /dev/null +++ b/tests/requests/invalid/chunked_02.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidHeader +request = InvalidHeader diff --git a/tests/requests/invalid/chunked_03.http b/tests/requests/invalid/chunked_03.http new file mode 100644 index 000000000..0bbbfe6ef --- /dev/null +++ b/tests/requests/invalid/chunked_03.http @@ -0,0 +1,8 @@ +POST /double_chunked HTTP/1.1\r\n +Transfer-Encoding: identity, chunked, identity, chunked\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +\r\n diff --git a/tests/requests/invalid/chunked_03.py b/tests/requests/invalid/chunked_03.py new file mode 100644 index 000000000..58a34600c --- /dev/null +++ b/tests/requests/invalid/chunked_03.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import UnsupportedTransferCoding +request = UnsupportedTransferCoding diff --git a/tests/requests/invalid/chunked_04.http b/tests/requests/invalid/chunked_04.http new file mode 100644 index 000000000..d47109e31 --- /dev/null +++ b/tests/requests/invalid/chunked_04.http @@ -0,0 +1,11 @@ +POST /chunked_twice HTTP/1.1\r\n +Transfer-Encoding: identity\r\n +Transfer-Encoding: chunked\r\n +Transfer-Encoding: identity\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +\r\n diff --git a/tests/requests/invalid/chunked_04.py b/tests/requests/invalid/chunked_04.py new file mode 100644 index 000000000..1541eb70b --- /dev/null +++ b/tests/requests/invalid/chunked_04.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidHeader +request = InvalidHeader diff --git a/tests/requests/invalid/chunked_05.http b/tests/requests/invalid/chunked_05.http new file mode 100644 index 000000000..014e85ac0 --- /dev/null +++ b/tests/requests/invalid/chunked_05.http @@ -0,0 +1,11 @@ +POST /chunked_HTTP_1.0 HTTP/1.0\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +0\r\n +Vary: *\r\n +Content-Type: text/plain\r\n +\r\n diff --git a/tests/requests/invalid/chunked_05.py b/tests/requests/invalid/chunked_05.py new file mode 100644 index 000000000..1541eb70b --- /dev/null +++ b/tests/requests/invalid/chunked_05.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidHeader +request = InvalidHeader diff --git a/tests/requests/invalid/chunked_06.http b/tests/requests/invalid/chunked_06.http new file mode 100644 index 000000000..ef70faab9 --- /dev/null +++ b/tests/requests/invalid/chunked_06.http @@ -0,0 +1,9 @@ +POST /chunked_not_last HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +Transfer-Encoding: gzip\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +\r\n diff --git a/tests/requests/invalid/chunked_06.py b/tests/requests/invalid/chunked_06.py new file mode 100644 index 000000000..58a34600c --- /dev/null +++ b/tests/requests/invalid/chunked_06.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import UnsupportedTransferCoding +request = UnsupportedTransferCoding diff --git a/tests/requests/invalid/chunked_08.http b/tests/requests/invalid/chunked_08.http new file mode 100644 index 000000000..8d4aaa6e9 --- /dev/null +++ b/tests/requests/invalid/chunked_08.http @@ -0,0 +1,9 @@ +POST /chunked_not_last HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +Transfer-Encoding: identity\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +\r\n diff --git a/tests/requests/invalid/chunked_08.py b/tests/requests/invalid/chunked_08.py new file mode 100644 index 000000000..1541eb70b --- /dev/null +++ b/tests/requests/invalid/chunked_08.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidHeader +request = InvalidHeader diff --git a/tests/requests/invalid/nonascii_01.http b/tests/requests/invalid/nonascii_01.http new file mode 100644 index 000000000..30d18cd6f --- /dev/null +++ b/tests/requests/invalid/nonascii_01.http @@ -0,0 +1,4 @@ +GETß /germans.. HTTP/1.1\r\n +Content-Length: 3\r\n +\r\n +ÄÄÄ diff --git a/tests/requests/invalid/nonascii_01.py b/tests/requests/invalid/nonascii_01.py new file mode 100644 index 000000000..0da10f426 --- /dev/null +++ b/tests/requests/invalid/nonascii_01.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidRequestMethod + +cfg = Config() +request = InvalidRequestMethod diff --git a/tests/requests/invalid/nonascii_02.http b/tests/requests/invalid/nonascii_02.http new file mode 100644 index 000000000..36a617039 --- /dev/null +++ b/tests/requests/invalid/nonascii_02.http @@ -0,0 +1,4 @@ +GETÿ /french.. HTTP/1.1\r\n +Content-Length: 3\r\n +\r\n +ÄÄÄ diff --git a/tests/requests/invalid/nonascii_02.py b/tests/requests/invalid/nonascii_02.py new file mode 100644 index 000000000..0da10f426 --- /dev/null +++ b/tests/requests/invalid/nonascii_02.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidRequestMethod + +cfg = Config() +request = InvalidRequestMethod diff --git a/tests/requests/invalid/nonascii_04.http b/tests/requests/invalid/nonascii_04.http new file mode 100644 index 000000000..be0b15661 --- /dev/null +++ b/tests/requests/invalid/nonascii_04.http @@ -0,0 +1,5 @@ +GET /french.. HTTP/1.1\r\n +Content-Lengthÿ: 3\r\n +Content-Length: 3\r\n +\r\n +ÄÄÄ diff --git a/tests/requests/invalid/nonascii_04.py b/tests/requests/invalid/nonascii_04.py new file mode 100644 index 000000000..d336fbc85 --- /dev/null +++ b/tests/requests/invalid/nonascii_04.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeaderName + +cfg = Config() +request = InvalidHeaderName diff --git a/tests/requests/invalid/prefix_01.http b/tests/requests/invalid/prefix_01.http new file mode 100644 index 000000000..f8bdeb352 --- /dev/null +++ b/tests/requests/invalid/prefix_01.http @@ -0,0 +1,2 @@ +GET\0PROXY /foo HTTP/1.1\r\n +\r\n diff --git a/tests/requests/invalid/prefix_01.py b/tests/requests/invalid/prefix_01.py new file mode 100644 index 000000000..86a0774e5 --- /dev/null +++ b/tests/requests/invalid/prefix_01.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidRequestMethod +request = InvalidRequestMethod \ No newline at end of file diff --git a/tests/requests/invalid/prefix_02.http b/tests/requests/invalid/prefix_02.http new file mode 100644 index 000000000..8a9b155c4 --- /dev/null +++ b/tests/requests/invalid/prefix_02.http @@ -0,0 +1,2 @@ +GET\0 /foo HTTP/1.1\r\n +\r\n diff --git a/tests/requests/invalid/prefix_02.py b/tests/requests/invalid/prefix_02.py new file mode 100644 index 000000000..86a0774e5 --- /dev/null +++ b/tests/requests/invalid/prefix_02.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidRequestMethod +request = InvalidRequestMethod \ No newline at end of file diff --git a/tests/requests/invalid/prefix_03.http b/tests/requests/invalid/prefix_03.http new file mode 100644 index 000000000..7803935cc --- /dev/null +++ b/tests/requests/invalid/prefix_03.http @@ -0,0 +1,4 @@ +GET /stuff/here?foo=bar HTTP/1.1\r\n +Content-Length: 0 1\r\n +\r\n +x diff --git a/tests/requests/invalid/prefix_03.py b/tests/requests/invalid/prefix_03.py new file mode 100644 index 000000000..95b0581ae --- /dev/null +++ b/tests/requests/invalid/prefix_03.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeader + +cfg = Config() +request = InvalidHeader diff --git a/tests/requests/invalid/prefix_04.http b/tests/requests/invalid/prefix_04.http new file mode 100644 index 000000000..712631c8f --- /dev/null +++ b/tests/requests/invalid/prefix_04.http @@ -0,0 +1,5 @@ +GET /stuff/here?foo=bar HTTP/1.1\r\n +Content-Length: 3 1\r\n +\r\n +xyz +abc123 diff --git a/tests/requests/invalid/prefix_04.py b/tests/requests/invalid/prefix_04.py new file mode 100644 index 000000000..95b0581ae --- /dev/null +++ b/tests/requests/invalid/prefix_04.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeader + +cfg = Config() +request = InvalidHeader diff --git a/tests/requests/invalid/prefix_05.http b/tests/requests/invalid/prefix_05.http new file mode 100644 index 000000000..120b6577d --- /dev/null +++ b/tests/requests/invalid/prefix_05.http @@ -0,0 +1,4 @@ +GET: /stuff/here?foo=bar HTTP/1.1\r\n +Content-Length: 3\r\n +\r\n +xyz diff --git a/tests/requests/invalid/prefix_05.py b/tests/requests/invalid/prefix_05.py new file mode 100644 index 000000000..0da10f426 --- /dev/null +++ b/tests/requests/invalid/prefix_05.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidRequestMethod + +cfg = Config() +request = InvalidRequestMethod diff --git a/tests/requests/valid/025.http b/tests/requests/valid/025.http index 62267add0..f8d7fae2e 100644 --- a/tests/requests/valid/025.http +++ b/tests/requests/valid/025.http @@ -1,5 +1,4 @@ POST /chunked_cont_h_at_first HTTP/1.1\r\n -Content-Length: -1\r\n Transfer-Encoding: chunked\r\n \r\n 5; some; parameters=stuff\r\n @@ -16,4 +15,10 @@ Content-Length: -1\r\n hello\r\n 6; blahblah; blah\r\n world\r\n -0\r\n \ No newline at end of file +0\r\n +\r\n +PUT /ignored_after_dangerous_framing HTTP/1.1\r\n +Content-Length: 3\r\n +\r\n +foo\r\n +\r\n diff --git a/tests/requests/valid/025.py b/tests/requests/valid/025.py index 12ea9ab76..33f5845cb 100644 --- a/tests/requests/valid/025.py +++ b/tests/requests/valid/025.py @@ -1,9 +1,13 @@ +from gunicorn.config import Config + +cfg = Config() +cfg.set("tolerate_dangerous_framing", True) + req1 = { "method": "POST", "uri": uri("/chunked_cont_h_at_first"), "version": (1, 1), "headers": [ - ("CONTENT-LENGTH", "-1"), ("TRANSFER-ENCODING", "chunked") ], "body": b"hello world" diff --git a/tests/requests/valid/025compat.http b/tests/requests/valid/025compat.http new file mode 100644 index 000000000..828f6fb71 --- /dev/null +++ b/tests/requests/valid/025compat.http @@ -0,0 +1,18 @@ +POST /chunked_cont_h_at_first HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +\r\n +5; some; parameters=stuff\r\n +hello\r\n +6; blahblah; blah\r\n + world\r\n +0\r\n +\r\n +PUT /chunked_cont_h_at_last HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +Content-Length: -1\r\n +\r\n +5; some; parameters=stuff\r\n +hello\r\n +6; blahblah; blah\r\n + world\r\n +0\r\n diff --git a/tests/requests/valid/025compat.py b/tests/requests/valid/025compat.py new file mode 100644 index 000000000..33f5845cb --- /dev/null +++ b/tests/requests/valid/025compat.py @@ -0,0 +1,27 @@ +from gunicorn.config import Config + +cfg = Config() +cfg.set("tolerate_dangerous_framing", True) + +req1 = { + "method": "POST", + "uri": uri("/chunked_cont_h_at_first"), + "version": (1, 1), + "headers": [ + ("TRANSFER-ENCODING", "chunked") + ], + "body": b"hello world" +} + +req2 = { + "method": "PUT", + "uri": uri("/chunked_cont_h_at_last"), + "version": (1, 1), + "headers": [ + ("TRANSFER-ENCODING", "chunked"), + ("CONTENT-LENGTH", "-1"), + ], + "body": b"hello world" +} + +request = [req1, req2] diff --git a/tests/requests/valid/029.http b/tests/requests/valid/029.http index c8611dbd3..5d029dd91 100644 --- a/tests/requests/valid/029.http +++ b/tests/requests/valid/029.http @@ -1,6 +1,6 @@ GET /stuff/here?foo=bar HTTP/1.1\r\n -Transfer-Encoding: chunked\r\n Transfer-Encoding: identity\r\n +Transfer-Encoding: chunked\r\n \r\n 5\r\n hello\r\n diff --git a/tests/requests/valid/029.py b/tests/requests/valid/029.py index f25449d15..64d026604 100644 --- a/tests/requests/valid/029.py +++ b/tests/requests/valid/029.py @@ -7,8 +7,8 @@ "uri": uri("/stuff/here?foo=bar"), "version": (1, 1), "headers": [ + ('TRANSFER-ENCODING', 'identity'), ('TRANSFER-ENCODING', 'chunked'), - ('TRANSFER-ENCODING', 'identity') ], "body": b"hello" } diff --git a/tests/treq.py b/tests/treq.py index 05adb1461..aeaae151f 100644 --- a/tests/treq.py +++ b/tests/treq.py @@ -248,8 +248,10 @@ def test_req(sn, sz, mt): def check(self, cfg, sender, sizer, matcher): cases = self.expect[:] p = RequestParser(cfg, sender(), None) - for req in p: + parsed_request_idx = -1 + for parsed_request_idx, req in enumerate(p): self.same(req, sizer, matcher, cases.pop(0)) + assert len(self.expect) == parsed_request_idx + 1 assert not cases def same(self, req, sizer, matcher, exp): From fd67112f40f48a9489f1f3132d8485ac11db9fbf Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 7 Dec 2023 09:31:00 +0100 Subject: [PATCH 147/164] Ignore secure_scheme_headers in Trailer section In common configuration unlikely a big security problem in itself you are just fooling the remote about https. However, it is offers an oracle for otherwise invisible proxy request headers, so it might help exploiting other vulnerabilities. --- gunicorn/http/body.py | 2 +- gunicorn/http/message.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/gunicorn/http/body.py b/gunicorn/http/body.py index 41fe334bc..2ae0eb848 100644 --- a/gunicorn/http/body.py +++ b/gunicorn/http/body.py @@ -51,7 +51,7 @@ def parse_trailers(self, unreader, data): if done: unreader.unread(buf.getvalue()[2:]) return b"" - self.req.trailers = self.req.parse_headers(buf.getvalue()[:idx]) + self.req.trailers = self.req.parse_headers(buf.getvalue()[:idx], from_trailer=True) unreader.unread(buf.getvalue()[idx + 4:]) def parse_chunked(self, unreader): diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 75b36e330..67fffd9e8 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -66,7 +66,7 @@ def force_close(self): def parse(self, unreader): raise NotImplementedError() - def parse_headers(self, data): + def parse_headers(self, data, from_trailer=False): cfg = self.cfg headers = [] @@ -76,9 +76,13 @@ def parse_headers(self, data): # handle scheme headers scheme_header = False secure_scheme_headers = {} - if ('*' in cfg.forwarded_allow_ips or - not isinstance(self.peer_addr, tuple) - or self.peer_addr[0] in cfg.forwarded_allow_ips): + if from_trailer: + # nonsense. either a request is https from the beginning + # .. or we are just behind a proxy who does not remove conflicting trailers + pass + elif ('*' in cfg.forwarded_allow_ips or + not isinstance(self.peer_addr, tuple) + or self.peer_addr[0] in cfg.forwarded_allow_ips): secure_scheme_headers = cfg.secure_scheme_headers # Parse headers into key/value pairs paying attention @@ -294,7 +298,7 @@ def parse(self, unreader): self.unreader.unread(data[2:]) return b"" - self.headers = self.parse_headers(data[:idx]) + self.headers = self.parse_headers(data[:idx], from_trailer=False) ret = data[idx + 4:] buf = None From f5501111a21d000f8c383277ef132ed882a4932f Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 7 Dec 2023 09:41:10 +0100 Subject: [PATCH 148/164] strict HTTP header field name validation Do the validation on the original, not the result from unicode case folding. Background: latin-1 0xDF is traditionally uppercased 0x53+0x53 which puts it back in ASCII --- gunicorn/http/message.py | 10 +++++++--- tests/requests/invalid/nonascii_03.http | 5 +++++ tests/requests/invalid/nonascii_03.py | 5 +++++ 3 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 tests/requests/invalid/nonascii_03.http create mode 100644 tests/requests/invalid/nonascii_03.py diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 67fffd9e8..4e1c2fd55 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -98,12 +98,16 @@ def parse_headers(self, data, from_trailer=False): raise InvalidHeader(curr) name, value = curr.split(":", 1) if self.cfg.strip_header_spaces: - name = name.rstrip(" \t").upper() - else: - name = name.upper() + name = name.rstrip(" \t") if not TOKEN_RE.fullmatch(name): raise InvalidHeaderName(name) + # this is still a dangerous place to do this + # but it is more correct than doing it before the pattern match: + # after we entered Unicode wonderland, 8bits could case-shift into ASCII: + # b"\xDF".decode("latin-1").upper().encode("ascii") == b"SS" + name = name.upper() + value = [value.lstrip(" \t")] # Consume value continuation lines diff --git a/tests/requests/invalid/nonascii_03.http b/tests/requests/invalid/nonascii_03.http new file mode 100644 index 000000000..eda5d010f --- /dev/null +++ b/tests/requests/invalid/nonascii_03.http @@ -0,0 +1,5 @@ +GET /germans.. HTTP/1.1\r\n +Content-Lengthß: 3\r\n +Content-Length: 3\r\n +\r\n +ÄÄÄ diff --git a/tests/requests/invalid/nonascii_03.py b/tests/requests/invalid/nonascii_03.py new file mode 100644 index 000000000..d336fbc85 --- /dev/null +++ b/tests/requests/invalid/nonascii_03.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeaderName + +cfg = Config() +request = InvalidHeaderName From 7ebe442d089a6fe0c51abb19a598b3d0d6a6d128 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 7 Dec 2023 09:56:49 +0100 Subject: [PATCH 149/164] strict HTTP version validation Note: This is unrelated to a reverse proxy potentially talking HTTP/3 to clients. This is about the HTTP protocol version spoken to Gunicorn, which is HTTP/1.0 or HTTP/1.1. Little legitimate need for processing HTTP 1 requests with ambiguous version numbers. Broadly refuse. Co-authored-by: Ben Kallus --- gunicorn/config.py | 20 ++++++++++++++++++++ gunicorn/http/message.py | 7 ++++++- tests/requests/invalid/prefix_06.http | 4 ++++ tests/requests/invalid/prefix_06.py | 5 +++++ tests/requests/invalid/version_01.http | 2 ++ tests/requests/invalid/version_01.py | 2 ++ tests/requests/invalid/version_02.http | 2 ++ tests/requests/invalid/version_02.py | 2 ++ 8 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 tests/requests/invalid/prefix_06.http create mode 100644 tests/requests/invalid/prefix_06.py create mode 100644 tests/requests/invalid/version_01.http create mode 100644 tests/requests/invalid/version_01.py create mode 100644 tests/requests/invalid/version_02.http create mode 100644 tests/requests/invalid/version_02.py diff --git a/gunicorn/config.py b/gunicorn/config.py index be9bb001c..e7e4fac54 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -2282,6 +2282,26 @@ class PermitUnconventionalHTTPMethod(Setting): """ +class PermitUnconventionalHTTPVersion(Setting): + name = "permit_unconventional_http_version" + section = "Server Mechanics" + cli = ["--permit-unconventional-http-version"] + validator = validate_bool + action = "store_true" + default = False + desc = """\ + Permit HTTP version not matching conventions of 2023 + + This disables the refusal of likely malformed request lines. + It is unusual to specify HTTP 1 versions other than 1.0 and 1.1. + + This option is provided to diagnose backwards-incompatible changes. + Use with care and only if necessary. May be removed in a future version. + + .. versionadded:: 22.0.0 + """ + + class CasefoldHTTPMethod(Setting): name = "casefold_http_method" section = "Server Mechanics" diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 4e1c2fd55..5e8d2427d 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -26,7 +26,8 @@ RFC9110_5_6_2_TOKEN_SPECIALS = r"!#$%&'*+-.^_`|~" TOKEN_RE = re.compile(r"[%s0-9a-zA-Z]+" % (re.escape(RFC9110_5_6_2_TOKEN_SPECIALS))) METHOD_BADCHAR_RE = re.compile("[a-z#]") -VERSION_RE = re.compile(r"HTTP/(\d+)\.(\d+)") +# usually 1.0 or 1.1 - RFC9112 permits restricting to single-digit versions +VERSION_RE = re.compile(r"HTTP/(\d)\.(\d)") class Message(object): @@ -438,6 +439,10 @@ def parse_request_line(self, line_bytes): if match is None: raise InvalidHTTPVersion(bits[2]) self.version = (int(match.group(1)), int(match.group(2))) + if not (1, 0) <= self.version < (2, 0): + # if ever relaxing this, carefully review Content-Encoding processing + if not self.cfg.permit_unconventional_http_version: + raise InvalidHTTPVersion(self.version) def set_body_reader(self): super().set_body_reader() diff --git a/tests/requests/invalid/prefix_06.http b/tests/requests/invalid/prefix_06.http new file mode 100644 index 000000000..536c586a3 --- /dev/null +++ b/tests/requests/invalid/prefix_06.http @@ -0,0 +1,4 @@ +GET /the/future HTTP/1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111.1\r\n +Content-Length: 7\r\n +\r\n +Old Man diff --git a/tests/requests/invalid/prefix_06.py b/tests/requests/invalid/prefix_06.py new file mode 100644 index 000000000..b2286d428 --- /dev/null +++ b/tests/requests/invalid/prefix_06.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHTTPVersion + +cfg = Config() +request = InvalidHTTPVersion diff --git a/tests/requests/invalid/version_01.http b/tests/requests/invalid/version_01.http new file mode 100644 index 000000000..0a222ce12 --- /dev/null +++ b/tests/requests/invalid/version_01.http @@ -0,0 +1,2 @@ +GET /foo HTTP/0.99\r\n +\r\n diff --git a/tests/requests/invalid/version_01.py b/tests/requests/invalid/version_01.py new file mode 100644 index 000000000..760840b69 --- /dev/null +++ b/tests/requests/invalid/version_01.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidHTTPVersion +request = InvalidHTTPVersion diff --git a/tests/requests/invalid/version_02.http b/tests/requests/invalid/version_02.http new file mode 100644 index 000000000..6d29ac5fe --- /dev/null +++ b/tests/requests/invalid/version_02.http @@ -0,0 +1,2 @@ +GET /foo HTTP/2.0\r\n +\r\n diff --git a/tests/requests/invalid/version_02.py b/tests/requests/invalid/version_02.py new file mode 100644 index 000000000..760840b69 --- /dev/null +++ b/tests/requests/invalid/version_02.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidHTTPVersion +request = InvalidHTTPVersion From b6c7414fd08196e87bee7c6687b4286922304e4e Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Tue, 12 Dec 2023 15:27:47 +0100 Subject: [PATCH 150/164] briefly document security fixes in 2023 news further information to be published in security advisories, published out of tree on Github --- docs/source/2023-news.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/source/2023-news.rst b/docs/source/2023-news.rst index 80f022248..971cfef38 100644 --- a/docs/source/2023-news.rst +++ b/docs/source/2023-news.rst @@ -2,6 +2,28 @@ Changelog - 2023 ================ +22.0.0 - TBDTBDTBD +================== + +- fix numerous security vulnerabilites in HTTP parser (closing some request smuggling vectors) +- parsing additional requests is no longer attempted past unsupported request framing +- on HTTP versions < 1.1 support for chunked transfer is refused (only used in exploits) +- requests conflicting configured or passed SCRIPT_NAME now produce a verbose error +- Trailer fields are no longer inspected for headers indicating secure scheme + +** Breaking changes ** + +- the limitations on valid characters in the HTTP method have been bounded to Internet Standards +- requests specifying unsupported transfer coding (order) are refused by default (rare) +- HTTP methods are no longer casefolded by default (IANA method registry contains none affacted) +- HTTP methods containing the number sign (#) are no longer accepted by default (rare) +- HTTP versions < 1.0 or >= 2.0 are no longer accepted by default (rare, only HTTP/1.1 is supported) +- HTTP versions consisting of multiple digits or containing a prefix/suffix are no longer accepted +- HTTP header field names Gunicorn cannot safely map to variables are silently dropped, as in other software +- HTTP headers with empty field name are refused by default (no legitimate use cases, used in exploits) +- requests with both Transfer-Encoding and Content-Length are refused by default (such a message might indicate an attempt to perform request smuggling) +- empty transfer codings are no longer permitted (reportedly seen with really old & broken proxies) + 21.2.0 - 2023-07-19 =================== From e710393d142edf56294c4490dbe08e5980160b19 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Sun, 17 Dec 2023 17:11:58 +0100 Subject: [PATCH 151/164] HTTP parser: stricter chunk-ext OBS handling chunk extensions are silently ignored before and after this change; its just the whitespace handling for the case without extensions that matters applying same strip(WS)->rstrip(BWS) replacement as already done in related cases half-way fix: could probably reject all BWS cases, rejecting only misplaced ones --- gunicorn/http/body.py | 5 ++++- tests/requests/invalid/chunked_09.http | 7 +++++++ tests/requests/invalid/chunked_09.py | 2 ++ tests/requests/invalid/chunked_10.http | 7 +++++++ tests/requests/invalid/chunked_10.py | 2 ++ tests/requests/invalid/chunked_11.http | 7 +++++++ tests/requests/invalid/chunked_11.py | 2 ++ tests/requests/valid/025.http | 2 +- 8 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 tests/requests/invalid/chunked_09.http create mode 100644 tests/requests/invalid/chunked_09.py create mode 100644 tests/requests/invalid/chunked_10.http create mode 100644 tests/requests/invalid/chunked_10.py create mode 100644 tests/requests/invalid/chunked_11.http create mode 100644 tests/requests/invalid/chunked_11.py diff --git a/gunicorn/http/body.py b/gunicorn/http/body.py index 2ae0eb848..78f03214a 100644 --- a/gunicorn/http/body.py +++ b/gunicorn/http/body.py @@ -85,7 +85,10 @@ def parse_chunk_size(self, unreader, data=None): data = buf.getvalue() line, rest_chunk = data[:idx], data[idx + 2:] - chunk_size = line.split(b";", 1)[0].strip() + # RFC9112 7.1.1: BWS before chunk-ext - but ONLY then + chunk_size, *chunk_ext = line.split(b";", 1) + if chunk_ext: + chunk_size = chunk_size.rstrip(b" \t") if any(n not in b"0123456789abcdefABCDEF" for n in chunk_size): raise InvalidChunkSize(chunk_size) chunk_size = int(chunk_size, 16) diff --git a/tests/requests/invalid/chunked_09.http b/tests/requests/invalid/chunked_09.http new file mode 100644 index 000000000..6207310cf --- /dev/null +++ b/tests/requests/invalid/chunked_09.http @@ -0,0 +1,7 @@ +POST /chunked_ows_without_ext HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\r\n +hello\r\n +0 \r\n +\r\n diff --git a/tests/requests/invalid/chunked_09.py b/tests/requests/invalid/chunked_09.py new file mode 100644 index 000000000..0571e1183 --- /dev/null +++ b/tests/requests/invalid/chunked_09.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidChunkSize +request = InvalidChunkSize diff --git a/tests/requests/invalid/chunked_10.http b/tests/requests/invalid/chunked_10.http new file mode 100644 index 000000000..db1e4c38d --- /dev/null +++ b/tests/requests/invalid/chunked_10.http @@ -0,0 +1,7 @@ +POST /chunked_ows_before HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\r\n +hello\r\n + 0\r\n +\r\n diff --git a/tests/requests/invalid/chunked_10.py b/tests/requests/invalid/chunked_10.py new file mode 100644 index 000000000..0571e1183 --- /dev/null +++ b/tests/requests/invalid/chunked_10.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidChunkSize +request = InvalidChunkSize diff --git a/tests/requests/invalid/chunked_11.http b/tests/requests/invalid/chunked_11.http new file mode 100644 index 000000000..da1f72e58 --- /dev/null +++ b/tests/requests/invalid/chunked_11.http @@ -0,0 +1,7 @@ +POST /chunked_ows_before HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\n;\r\n +hello\r\n +0\r\n +\r\n diff --git a/tests/requests/invalid/chunked_11.py b/tests/requests/invalid/chunked_11.py new file mode 100644 index 000000000..0571e1183 --- /dev/null +++ b/tests/requests/invalid/chunked_11.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidChunkSize +request = InvalidChunkSize diff --git a/tests/requests/valid/025.http b/tests/requests/valid/025.http index f8d7fae2e..214f3094c 100644 --- a/tests/requests/valid/025.http +++ b/tests/requests/valid/025.http @@ -3,7 +3,7 @@ Transfer-Encoding: chunked\r\n \r\n 5; some; parameters=stuff\r\n hello\r\n -6; blahblah; blah\r\n +6 \t;\tblahblah; blah\r\n world\r\n 0\r\n \r\n From e0c3390f1e2e51c8098f49b1a7d5957fadef512a Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Mon, 25 Dec 2023 18:39:18 +0000 Subject: [PATCH 152/164] Typo and email in Security.md Fixes: 13027ef797edba55967f366ec958a9a03b3d345b email duplicated from docs/source/community.rst --- SECURITY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 3d1923540..852025c0c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,7 +4,7 @@ **Please note that public Github issues are open for everyone to see!** -If you believe you are found a problem in Gunicorn software, examples or documentation, we encourage you to send your report privately via email, or via Github using the *Report a vulnerability* button in the [Security](https://github.com/benoitc/gunicorn/security) section. +If you believe you are found a problem in Gunicorn software, examples or documentation, we encourage you to send your report privately via [email](mailto:security@gunicorn.org?subject=Security%20issue%20in%20Gunicorn), or via Github using the *Report a vulnerability* button in the [Security](https://github.com/benoitc/gunicorn/security) section. ## Supported Releases @@ -19,4 +19,4 @@ At this time, **only the latest release** receives any security attention whatso ## Python Versions -Gunicorn runs on Python 3.7+, we *highly recommend* the latest release of a [supported series](https://devguide.python.org/version/) and will not prioritize issues exclusively affecting in EoL environments. +Gunicorn runs on Python 3.7+, we *highly recommend* the latest release of a [supported series](https://devguide.python.org/versions/) and will not prioritize issues exclusively affecting in EoL environments. From 660fd8d850f9424d5adcd50065e6060832a200d4 Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Thu, 28 Dec 2023 19:57:14 -0800 Subject: [PATCH 153/164] Fix references to non-existent 20.2 version in configuration settings Close #3043. --- gunicorn/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index e7e4fac54..3b57fad0f 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -2026,7 +2026,7 @@ def ssl_context(conf, default_ssl_context_factory): context.minimum_version = ssl.TLSVersion.TLSv1_3 return context - .. versionadded:: 20.2 + .. versionadded:: 21.0 """ @@ -2110,7 +2110,7 @@ class SSLVersion(Setting): desc = """\ SSL version to use (see stdlib ssl module's). - .. deprecated:: 20.2 + .. deprecated:: 21.0 The option is deprecated and it is currently ignored. Use :ref:`ssl-context` instead. ============= ============ From 0bb96d17c584be1a483a106098d5a40d17073b64 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 28 Dec 2023 21:21:01 +0100 Subject: [PATCH 154/164] CI: tests may hang on PyPy --- .github/workflows/tox.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 84e322fe5..eef5595c4 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -8,6 +8,8 @@ env: jobs: tox: name: ${{ matrix.os }} / ${{ matrix.python-version }} + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes + timeout-minutes: 10 runs-on: ${{ matrix.os }} strategy: fail-fast: false From b39c5b7ebb31a9db24db3805ddbc7f2dbbf676e1 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 28 Dec 2023 21:24:20 +0100 Subject: [PATCH 155/164] CI: style --- .github/workflows/tox.yml | 11 ++++++++++- tox.ini | 8 +++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index eef5595c4..29e09afc0 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -15,7 +15,16 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] # All OSes pass except Windows because tests need Unix-only fcntl, grp, pwd, etc. - python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.8" ] + python-version: + # CPython <= 3.7 is EoL since 2923-06-27 + - "3.7" + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + # PyPy <= 3.8 is EoL since 2023-06-16 + - "pypy-3.8" steps: - uses: actions/checkout@v4 - name: Using Python ${{ matrix.python-version }} diff --git a/tox.ini b/tox.ini index 4d4c19cd4..c1c2fd053 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,11 @@ [tox] -envlist = py{37,38,39,310,311,312,py3}, lint, docs-lint, pycodestyle, run-entrypoint, run-module +envlist = + py{37,38,39,310,311,312,py3}, + lint, + docs-lint, + pycodestyle, + run-entrypoint, + run-module, skipsdist = false ; Can't set skipsdist and use_develop in tox v4 to true due to https://github.com/tox-dev/tox/issues/2730 From 184e36f9dae7065d91e35ae1700dcbf4c556b7c2 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Fri, 8 Dec 2023 01:29:54 +0100 Subject: [PATCH 156/164] skip eventlet, not yet supported on python 3.12 will work again, should still be reverted when stdlib conflict resolved in eventlet --- tests/workers/test_geventlet.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/workers/test_geventlet.py b/tests/workers/test_geventlet.py index 815dcec34..06c7b5305 100644 --- a/tests/workers/test_geventlet.py +++ b/tests/workers/test_geventlet.py @@ -3,5 +3,15 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. +import pytest +import sys + def test_import(): + + try: + import eventlet + except AttributeError: + if (3,13) > sys.version_info >= (3, 12): + pytest.skip("Ignoring eventlet failures on Python 3.12") + raise __import__('gunicorn.workers.geventlet') From 89dcc5c5781ae27d14b8b14ccc4a70038a11989a Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 28 Dec 2023 21:31:10 +0100 Subject: [PATCH 157/164] CI: stop testing EoL PyPy --- .github/workflows/tox.yml | 5 +++-- appveyor.yml | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 29e09afc0..72d9884c5 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -16,7 +16,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] # All OSes pass except Windows because tests need Unix-only fcntl, grp, pwd, etc. python-version: - # CPython <= 3.7 is EoL since 2923-06-27 + # CPython <= 3.7 is EoL since 2023-06-27 - "3.7" - "3.8" - "3.9" @@ -24,7 +24,8 @@ jobs: - "3.11" - "3.12" # PyPy <= 3.8 is EoL since 2023-06-16 - - "pypy-3.8" + - "pypy-3.9" + - "pypy-3.10" steps: - uses: actions/checkout@v4 - name: Using Python ${{ matrix.python-version }} diff --git a/appveyor.yml b/appveyor.yml index 85feb4dc3..3cf11f0e9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,6 +24,12 @@ environment: # PYTHON: "C:\\Python38-x64" #- TOXENV: py39 # PYTHON: "C:\\Python39-x64" + #- TOXENV: py310 + # PYTHON: "C:\\Python310-x64" + #- TOXENV: py311 + # PYTHON: "C:\\Python311-x64" + #- TOXENV: py312 + # PYTHON: "C:\\Python312-x64" matrix: allow_failures: - TOXENV: py35 From 481c3f9522edc58806a3efc5b49be4f202cc7700 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Fri, 8 Dec 2023 07:02:46 +0100 Subject: [PATCH 158/164] remove setup.cfg - overridden by pyproject.toml --- requirements_test.txt | 2 +- setup.cfg | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 setup.cfg diff --git a/requirements_test.txt b/requirements_test.txt index fad22e315..b618d1a73 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,5 +1,5 @@ gevent eventlet coverage -pytest +pytest>=7.2.0 pytest-cov diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 07322e337..000000000 --- a/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[tool:pytest] -norecursedirs = examples lib local src -testpaths = tests/ -addopts = --assert=plain --cov=gunicorn --cov-report=xml From 5e30bfa6b1a3e1f2bde7feb514d1734d28f39231 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 16 Jul 2021 11:48:16 +0100 Subject: [PATCH 159/164] add changelog to project.urls (updated for PEP621) --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 0dabe03fe..f4ac02e99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,7 @@ requires = ["setuptools>=61.2"] build-backend = "setuptools.build_meta" [project] +# see https://packaging.python.org/en/latest/specifications/pyproject-toml/ name = "gunicorn" authors = [{name = "Benoit Chesneau", email = "benoitc@gunicorn.org"}] license = {text = "MIT"} @@ -46,6 +47,7 @@ Homepage = "https://gunicorn.org" Documentation = "https://docs.gunicorn.org" "Issue tracker" = "https://github.com/benoitc/gunicorn/issues" "Source code" = "https://github.com/benoitc/gunicorn" +Changelog = "https://docs.gunicorn.org/en/stable/news.html" [project.optional-dependencies] gevent = ["gevent>=1.4.0"] From f4703824c323fe6867dce0e2f11013b8de319353 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Thu, 28 Dec 2023 21:36:51 +0100 Subject: [PATCH 160/164] docs: promise 3.12 compat --- docs/source/2023-news.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/2023-news.rst b/docs/source/2023-news.rst index 971cfef38..c26071580 100644 --- a/docs/source/2023-news.rst +++ b/docs/source/2023-news.rst @@ -10,9 +10,11 @@ Changelog - 2023 - on HTTP versions < 1.1 support for chunked transfer is refused (only used in exploits) - requests conflicting configured or passed SCRIPT_NAME now produce a verbose error - Trailer fields are no longer inspected for headers indicating secure scheme +- support Python 3.12 ** Breaking changes ** +- minimum version is Python 3.7 - the limitations on valid characters in the HTTP method have been bounded to Internet Standards - requests specifying unsupported transfer coding (order) are refused by default (rare) - HTTP methods are no longer casefolded by default (IANA method registry contains none affacted) From deae2fc4c5f93bfce59be5363055d4cd4ab1b0b6 Mon Sep 17 00:00:00 2001 From: "Paul J. Dorn" Date: Fri, 29 Dec 2023 05:35:32 +0100 Subject: [PATCH 161/164] CI: back off the agressive timeout Precise number does not matter that much, so lets not stop potentially working tests. The point was to cut off well before 6 hours, so any small number will do. --- .github/workflows/tox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 72d9884c5..308fab11a 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -9,7 +9,7 @@ jobs: tox: name: ${{ matrix.os }} / ${{ matrix.python-version }} # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes - timeout-minutes: 10 + timeout-minutes: 20 runs-on: ${{ matrix.os }} strategy: fail-fast: false From 628a0bcb61ef3a211d67dfd68ad1ba161cccb3b8 Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Sun, 24 Mar 2024 03:12:40 +0900 Subject: [PATCH 162/164] chore: fix typos --- CONTRIBUTING.md | 2 +- docs/source/2021-news.rst | 2 +- docs/source/2023-news.rst | 8 ++++---- docs/source/news.rst | 2 +- docs/source/settings.rst | 4 ++-- gunicorn/arbiter.py | 2 +- gunicorn/config.py | 2 +- gunicorn/http/message.py | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7bd82abda..1231e715c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -139,7 +139,7 @@ The relevant maintainer for a pull request is assigned in 3 steps: * Step 1: Determine the subdirectory affected by the pull request. This might be src/registry, docs/source/api, or any other part of the repo. -* Step 2: Find the MAINTAINERS file which affects this directory. If the directory itself does not have a MAINTAINERS file, work your way up the the repo hierarchy until you find one. +* Step 2: Find the MAINTAINERS file which affects this directory. If the directory itself does not have a MAINTAINERS file, work your way up the repo hierarchy until you find one. * Step 3: The first maintainer listed is the primary maintainer who is assigned the Pull Request. The primary maintainer can reassign a Pull Request to other listed maintainers. diff --git a/docs/source/2021-news.rst b/docs/source/2021-news.rst index 7875f0a77..3057600de 100644 --- a/docs/source/2021-news.rst +++ b/docs/source/2021-news.rst @@ -16,7 +16,7 @@ Changelog - 2021 - fix tornado usage with latest versions of Django - add support for python -m gunicorn - fix systemd socket activation example -- allows to set wsgi application in configg file using `wsgi_app` +- allows to set wsgi application in config file using `wsgi_app` - document `--timeout = 0` - always close a connection when the number of requests exceeds the max requests - Disable keepalive during graceful shutdown diff --git a/docs/source/2023-news.rst b/docs/source/2023-news.rst index c26071580..1f943c8a2 100644 --- a/docs/source/2023-news.rst +++ b/docs/source/2023-news.rst @@ -5,7 +5,7 @@ Changelog - 2023 22.0.0 - TBDTBDTBD ================== -- fix numerous security vulnerabilites in HTTP parser (closing some request smuggling vectors) +- fix numerous security vulnerabilities in HTTP parser (closing some request smuggling vectors) - parsing additional requests is no longer attempted past unsupported request framing - on HTTP versions < 1.1 support for chunked transfer is refused (only used in exploits) - requests conflicting configured or passed SCRIPT_NAME now produce a verbose error @@ -17,7 +17,7 @@ Changelog - 2023 - minimum version is Python 3.7 - the limitations on valid characters in the HTTP method have been bounded to Internet Standards - requests specifying unsupported transfer coding (order) are refused by default (rare) -- HTTP methods are no longer casefolded by default (IANA method registry contains none affacted) +- HTTP methods are no longer casefolded by default (IANA method registry contains none affected) - HTTP methods containing the number sign (#) are no longer accepted by default (rare) - HTTP versions < 1.0 or >= 2.0 are no longer accepted by default (rare, only HTTP/1.1 is supported) - HTTP versions consisting of multiple digits or containing a prefix/suffix are no longer accepted @@ -46,9 +46,9 @@ This is fixing the bad file description error. - support python 3.11 - fix gevent and eventlet workers - fix threads support (gththread): improve performance and unblock requests -- SSL: noaw use SSLContext object +- SSL: now use SSLContext object - HTTP parser: miscellaneous fixes -- remove unecessary setuid calls +- remove unnecessary setuid calls - fix testing - improve logging - miscellaneous fixes to core engine diff --git a/docs/source/news.rst b/docs/source/news.rst index 73debcb9b..73cdc4cc7 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -29,7 +29,7 @@ This is fixing the bad file description error. - fix threads support (gththread): improve performance and unblock requests - SSL: noaw use SSLContext object - HTTP parser: miscellaneous fixes -- remove unecessary setuid calls +- remove unnecessary setuid calls - fix testing - improve logging - miscellaneous fixes to core engine diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 4be760c28..4e0c11877 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -210,7 +210,7 @@ H protocol s status B response length b response length or ``'-'`` (CLF format) -f referer +f referrer a user agent T request time in seconds M request time in milliseconds @@ -569,7 +569,7 @@ Whether client certificate is required (see stdlib ssl module's) =========== =========================== --cert-reqs Description =========== =========================== -`0` no client veirifcation +`0` no client verification `1` ssl.CERT_OPTIONAL `2` ssl.CERT_REQUIRED =========== =========================== diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 008a54efe..1cf436748 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -109,7 +109,7 @@ def setup(self, app): in sorted(self.cfg.settings.items(), key=lambda setting: setting[1])))) - # set enviroment' variables + # set environment' variables if self.cfg.env: for k, v in self.cfg.env.items(): os.environ[k] = v diff --git a/gunicorn/config.py b/gunicorn/config.py index 3b57fad0f..144acaecc 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -2233,7 +2233,7 @@ class PasteGlobalConf(Setting): The option can be specified multiple times. - The variables are passed to the the PasteDeploy entrypoint. Example:: + The variables are passed to the PasteDeploy entrypoint. Example:: $ gunicorn -b 127.0.0.1:8000 --paste development.ini --paste-global FOO=1 --paste-global BAR=2 diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 5e8d2427d..88ffa5a25 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -166,7 +166,7 @@ def set_body_reader(self): content_length = value elif name == "TRANSFER-ENCODING": if value.lower() == "chunked": - # DANGER: transer codings stack, and stacked chunking is never intended + # DANGER: transfer codings stack, and stacked chunking is never intended if chunked: raise InvalidHeader("TRANSFER-ENCODING", req=self) chunked = True From 0243ec39ef4fc1b479ff4e1659e165f0b980b571 Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Tue, 26 Mar 2024 10:15:11 -0400 Subject: [PATCH 163/164] fix(deps): exclude eventlet 0.36.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f4ac02e99..eaca1eac0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ Changelog = "https://docs.gunicorn.org/en/stable/news.html" [project.optional-dependencies] gevent = ["gevent>=1.4.0"] -eventlet = ["eventlet>=0.24.1"] +eventlet = ["eventlet>=0.24.1,!=0.36.0"] tornado = ["tornado>=0.2"] gthread = [] setproctitle = ["setproctitle"] From f63d59e4d73a8ee28748d2c700fb81c8780bc419 Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 17 Apr 2024 00:36:19 +0200 Subject: [PATCH 164/164] bump to 22.0 --- docs/site/index.html | 2 +- docs/source/2023-news.rst | 24 ------------------------ docs/source/news.rst | 32 ++++++++++++++++++++++++++++++++ gunicorn/__init__.py | 2 +- 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/docs/site/index.html b/docs/site/index.html index 33e973c18..7e099a586 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -16,7 +16,7 @@
    Latest version: 21.2.0 + href="https://docs.gunicorn.org/en/stable/">22.0.0
    diff --git a/docs/source/2023-news.rst b/docs/source/2023-news.rst index 1f943c8a2..286f15852 100644 --- a/docs/source/2023-news.rst +++ b/docs/source/2023-news.rst @@ -2,30 +2,6 @@ Changelog - 2023 ================ -22.0.0 - TBDTBDTBD -================== - -- fix numerous security vulnerabilities in HTTP parser (closing some request smuggling vectors) -- parsing additional requests is no longer attempted past unsupported request framing -- on HTTP versions < 1.1 support for chunked transfer is refused (only used in exploits) -- requests conflicting configured or passed SCRIPT_NAME now produce a verbose error -- Trailer fields are no longer inspected for headers indicating secure scheme -- support Python 3.12 - -** Breaking changes ** - -- minimum version is Python 3.7 -- the limitations on valid characters in the HTTP method have been bounded to Internet Standards -- requests specifying unsupported transfer coding (order) are refused by default (rare) -- HTTP methods are no longer casefolded by default (IANA method registry contains none affected) -- HTTP methods containing the number sign (#) are no longer accepted by default (rare) -- HTTP versions < 1.0 or >= 2.0 are no longer accepted by default (rare, only HTTP/1.1 is supported) -- HTTP versions consisting of multiple digits or containing a prefix/suffix are no longer accepted -- HTTP header field names Gunicorn cannot safely map to variables are silently dropped, as in other software -- HTTP headers with empty field name are refused by default (no legitimate use cases, used in exploits) -- requests with both Transfer-Encoding and Content-Length are refused by default (such a message might indicate an attempt to perform request smuggling) -- empty transfer codings are no longer permitted (reportedly seen with really old & broken proxies) - 21.2.0 - 2023-07-19 =================== diff --git a/docs/source/news.rst b/docs/source/news.rst index 73cdc4cc7..26c0fbb5f 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -2,6 +2,37 @@ Changelog ========= +22.0.0 - 2024-04-17 +=================== + +- use `utime` to notify workers liveness +- migrate setup to pyproject.toml +- fix numerous security vulnerabilities in HTTP parser (closing some request smuggling vectors) +- parsing additional requests is no longer attempted past unsupported request framing +- on HTTP versions < 1.1 support for chunked transfer is refused (only used in exploits) +- requests conflicting configured or passed SCRIPT_NAME now produce a verbose error +- Trailer fields are no longer inspected for headers indicating secure scheme +- support Python 3.12 + +** Breaking changes ** + +- minimum version is Python 3.7 +- the limitations on valid characters in the HTTP method have been bounded to Internet Standards +- requests specifying unsupported transfer coding (order) are refused by default (rare) +- HTTP methods are no longer casefolded by default (IANA method registry contains none affected) +- HTTP methods containing the number sign (#) are no longer accepted by default (rare) +- HTTP versions < 1.0 or >= 2.0 are no longer accepted by default (rare, only HTTP/1.1 is supported) +- HTTP versions consisting of multiple digits or containing a prefix/suffix are no longer accepted +- HTTP header field names Gunicorn cannot safely map to variables are silently dropped, as in other software +- HTTP headers with empty field name are refused by default (no legitimate use cases, used in exploits) +- requests with both Transfer-Encoding and Content-Length are refused by default (such a message might indicate an attempt to perform request smuggling) +- empty transfer codings are no longer permitted (reportedly seen with really old & broken proxies) + + +** SECURITY ** + +- fix CVE-2024-1135 + 21.2.0 - 2023-07-19 =================== @@ -44,6 +75,7 @@ History .. toctree:: :titlesonly: + 2024-news 2023-news 2021-news 2020-news diff --git a/gunicorn/__init__.py b/gunicorn/__init__.py index adf5e89be..70153f8e6 100644 --- a/gunicorn/__init__.py +++ b/gunicorn/__init__.py @@ -3,7 +3,7 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -version_info = (21, 2, 0) +version_info = (22, 0, 0) __version__ = ".".join([str(v) for v in version_info]) SERVER = "gunicorn" SERVER_SOFTWARE = "%s/%s" % (SERVER, __version__)