diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..8ac6b8c49 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..b0a862cd1 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,28 @@ +name: lint +on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) +env: + # note that some tools care only for the name, not the value + FORCE_COLOR: 1 +jobs: + lint: + name: tox-${{ matrix.toxenv }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + toxenv: [lint, docs-lint, pycodestyle] + python-version: [ "3.10" ] + steps: + - uses: actions/checkout@v4 + - name: Using Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox + - run: tox -e ${{ matrix.toxenv }} diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml new file mode 100644 index 000000000..308fab11a --- /dev/null +++ b/.github/workflows/tox.yml @@ -0,0 +1,44 @@ +name: tox +on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) +env: + # note that some tools care only for the name, not the value + FORCE_COLOR: 1 +jobs: + tox: + name: ${{ matrix.os }} / ${{ matrix.python-version }} + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes + timeout-minutes: 20 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] # All OSes pass except Windows because tests need Unix-only fcntl, grp, pwd, etc. + python-version: + # CPython <= 3.7 is EoL since 2023-06-27 + - "3.7" + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + # PyPy <= 3.8 is EoL since 2023-06-16 + - "pypy-3.9" + - "pypy-3.10" + steps: + - uses: actions/checkout@v4 + - name: Using Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: requirements_test.txt + check-latest: true + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox + - run: tox -e run-module + - run: tox -e run-entrypoint + - run: tox -e py diff --git a/.pylintrc b/.pylintrc index af99b3e1a..bc2046c0c 100644 --- a/.pylintrc +++ b/.pylintrc @@ -12,7 +12,6 @@ ignore= disable= attribute-defined-outside-init, - bad-continuation, bad-mcs-classmethod-argument, bare-except, broad-except, @@ -25,12 +24,10 @@ disable= import-self, inconsistent-return-statements, invalid-name, - misplaced-comparison-constant, missing-docstring, no-else-return, no-member, no-self-argument, - no-self-use, no-staticmethod-decorator, not-callable, protected-access, @@ -53,3 +50,6 @@ disable= useless-import-alias, comparison-with-callable, try-except-raise, + consider-using-with, + consider-using-f-string, + unspecified-encoding diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..0ff559627 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,22 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2dd038725..000000000 --- a/.travis.yml +++ /dev/null @@ -1,34 +0,0 @@ -language: python -matrix: - include: - - python: 3.8 - env: TOXENV=lint - - python: 3.8 - env: TOXENV=docs-lint - - python: 3.5 - env: TOXENV=py35 - - python: 3.6 - env: TOXENV=py36 - - python: 3.7 - env: TOXENV=py37 - - python: 3.8 - env: TOXENV=py38 - - python: 3.9 - env: TOXENV=py39 - - python: pypy3 - env: - - TOXENV=pypy3 - # Embedded c-ares takes a long time to build and - # as-of 2020-01-04 there are no PyPy3 manylinux - # wheels for gevent on PyPI. - - GEVENTSETUP_EMBED_CARES=no - dist: xenial -install: pip install -U tox coverage -# TODO: https://github.com/tox-dev/tox/issues/149 -script: tox --recreate -after_success: - - if [ -f .coverage ]; then coverage report ; fi -cache: - directories: - - .tox - - $HOME/.cache/pip diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7bd82abda..1231e715c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -139,7 +139,7 @@ The relevant maintainer for a pull request is assigned in 3 steps: * Step 1: Determine the subdirectory affected by the pull request. This might be src/registry, docs/source/api, or any other part of the repo. -* Step 2: Find the MAINTAINERS file which affects this directory. If the directory itself does not have a MAINTAINERS file, work your way up the the repo hierarchy until you find one. +* Step 2: Find the MAINTAINERS file which affects this directory. If the directory itself does not have a MAINTAINERS file, work your way up the repo hierarchy until you find one. * Step 3: The first maintainer listed is the primary maintainer who is assigned the Pull Request. The primary maintainer can reassign a Pull Request to other listed maintainers. diff --git a/LICENSE b/LICENSE index 65865a92a..b244a2eda 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -2009-2018 (c) Benoît Chesneau +2009-2023 (c) Benoît Chesneau 2009-2015 (c) Paul J. Davis Permission is hereby granted, free of charge, to any person diff --git a/NOTICE b/NOTICE index 8506656bd..56e8b56f8 100644 --- a/NOTICE +++ b/NOTICE @@ -1,6 +1,6 @@ Gunicorn -2009-2018 (c) Benoît Chesneau +2009-2023 (c) Benoît Chesneau 2009-2015 (c) Paul J. Davis Gunicorn is released under the MIT license. See the LICENSE @@ -86,4 +86,4 @@ OTHER DEALINGS IN THE SOFTWARE. util/unlink.py -------------- -backport frop python3 Lib/test/support.py +backport from python3 Lib/test/support.py diff --git a/README.rst b/README.rst index a6a27b850..4a4029dd3 100644 --- a/README.rst +++ b/README.rst @@ -9,16 +9,20 @@ Gunicorn :alt: Supported Python versions :target: https://pypi.python.org/pypi/gunicorn -.. image:: https://travis-ci.org/benoitc/gunicorn.svg?branch=master +.. image:: https://github.com/benoitc/gunicorn/actions/workflows/tox.yml/badge.svg :alt: Build Status - :target: https://travis-ci.org/benoitc/gunicorn + :target: https://github.com/benoitc/gunicorn/actions/workflows/tox.yml + +.. image:: https://github.com/benoitc/gunicorn/actions/workflows/lint.yml/badge.svg + :alt: Lint Status + :target: https://github.com/benoitc/gunicorn/actions/workflows/lint.yml Gunicorn 'Green Unicorn' is a Python WSGI HTTP Server for UNIX. It's a pre-fork worker model ported from Ruby's Unicorn_ project. The Gunicorn server is broadly compatible with various web frameworks, simply implemented, light on server resource usage, and fairly speedy. -Feel free to join us in `#gunicorn`_ on Freenode_. +Feel free to join us in `#gunicorn`_ on `Libera.chat`_. Documentation ------------- @@ -28,7 +32,7 @@ The documentation is hosted at https://docs.gunicorn.org. Installation ------------ -Gunicorn requires **Python 3.x >= 3.5**. +Gunicorn requires **Python 3.x >= 3.7**. Install from PyPI:: @@ -65,6 +69,6 @@ Gunicorn is released under the MIT License. See the LICENSE_ file for more details. .. _Unicorn: https://bogomips.org/unicorn/ -.. _`#gunicorn`: https://webchat.freenode.net/?channels=gunicorn -.. _Freenode: https://freenode.net/ +.. _`#gunicorn`: https://web.libera.chat/?channels=#gunicorn +.. _`Libera.chat`: https://libera.chat/ .. _LICENSE: https://github.com/benoitc/gunicorn/blob/master/LICENSE diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..852025c0c --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,22 @@ +# Security Policy + +## Reporting a Vulnerability + +**Please note that public Github issues are open for everyone to see!** + +If you believe you are found a problem in Gunicorn software, examples or documentation, we encourage you to send your report privately via [email](mailto:security@gunicorn.org?subject=Security%20issue%20in%20Gunicorn), or via Github using the *Report a vulnerability* button in the [Security](https://github.com/benoitc/gunicorn/security) section. + +## Supported Releases + +At this time, **only the latest release** receives any security attention whatsoever. + +| Version | Status | +| ------- | ------------------ | +| latest release | :white_check_mark: | +| 21.2.0 | :x: | +| 20.0.0 | :x: | +| < 20.0 | :x: | + +## Python Versions + +Gunicorn runs on Python 3.7+, we *highly recommend* the latest release of a [supported series](https://devguide.python.org/versions/) and will not prioritize issues exclusively affecting in EoL environments. diff --git a/THANKS b/THANKS index 2b226f35b..9f4c6b6b9 100644 --- a/THANKS +++ b/THANKS @@ -22,10 +22,12 @@ Andrew Svetlov Anil V Antoine Girard Anton Vlasenko +Artur Kruchinin Bartosz Oler Ben Cochran Ben Oswald Benjamin Gilbert +Benny Mei Benoit Chesneau Berker Peksag bninja @@ -39,6 +41,7 @@ Chris Adams Chris Forbes Chris Lamb Chris Streeter +Christian Clauss Christoph Heer Christos Stavrakakis CMGS @@ -103,12 +106,14 @@ Konstantin Kapustin kracekumar Kristian Glass Kristian Øllegaard +Krystian Krzysztof Urbaniak Kyle Kelley Kyle Mulka Lars Hansson Leonardo Santagada Levi Gross +licunlong Łukasz Kucharski Mahmoud Hashemi Malthe Borch @@ -153,6 +158,7 @@ Rik Ronan Amicel Ryan Peck Saeed Gharedaghi +Samuel Matos Sergey Rublev Shane Reustle shouse-cars @@ -163,8 +169,10 @@ Stephen DiCato Stephen Holsapple Steven Cummings Sébastien Fievet +Tal Einat <532281+taleinat@users.noreply.github.com> Talha Malik TedWantsMore +Teko012 <112829523+Teko012@users.noreply.github.com> Thomas Grainger Thomas Steinacher Travis Cline diff --git a/appveyor.yml b/appveyor.yml index 0bcf6c6ca..3cf11f0e9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,17 +2,34 @@ version: '{branch}.{build}' environment: matrix: - TOXENV: lint - PYTHON: "C:\\Python37-x64" - - TOXENV: py35 - PYTHON: "C:\\Python35-x64" - - TOXENV: py36 - PYTHON: "C:\\Python36-x64" - - TOXENV: py37 - PYTHON: "C:\\Python37-x64" - - TOXENV: py38 PYTHON: "C:\\Python38-x64" - - TOXENV: py39 - PYTHON: "C:\\Python39-x64" + - TOXENV: docs-lint + PYTHON: "C:\\Python38-x64" + - TOXENV: pycodestyle + PYTHON: "C:\\Python38-x64" + # Windows cannot even import the module when they unconditionally import, see below. + #- TOXENV: run-module + # PYTHON: "C:\\Python38-x64" + #- TOXENV: run-entrypoint + # PYTHON: "C:\\Python38-x64" + # Windows is not ready for testing!!! + # Python's fcntl, grp, pwd, os.geteuid(), and socket.AF_UNIX are all Unix-only. + #- TOXENV: py35 + # PYTHON: "C:\\Python35-x64" + #- TOXENV: py36 + # PYTHON: "C:\\Python36-x64" + #- TOXENV: py37 + # PYTHON: "C:\\Python37-x64" + #- TOXENV: py38 + # PYTHON: "C:\\Python38-x64" + #- TOXENV: py39 + # PYTHON: "C:\\Python39-x64" + #- TOXENV: py310 + # PYTHON: "C:\\Python310-x64" + #- TOXENV: py311 + # PYTHON: "C:\\Python311-x64" + #- TOXENV: py312 + # PYTHON: "C:\\Python312-x64" matrix: allow_failures: - TOXENV: py35 @@ -20,11 +37,13 @@ matrix: - TOXENV: py37 - TOXENV: py38 - TOXENV: py39 -init: SET "PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" +init: + - SET "PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" install: - pip install tox -build: off -test_script: tox +build: false +test_script: + - tox cache: # Not including the .tox directory since it takes longer to download/extract # the cache archive than for tox to clean install from the pip cache. diff --git a/docs/gunicorn_ext.py b/docs/gunicorn_ext.py index 5b4781479..4310162eb 100755 --- a/docs/gunicorn_ext.py +++ b/docs/gunicorn_ext.py @@ -48,7 +48,9 @@ def format_settings(app): def fmt_setting(s): - if callable(s.default): + if hasattr(s, "default_doc"): + val = s.default_doc + elif callable(s.default): val = inspect.getsource(s.default) val = "\n".join(" %s" % line for line in val.splitlines()) val = "\n\n.. code-block:: python\n\n" + val diff --git a/docs/site/index.html b/docs/site/index.html index 4fc6e170b..7e099a586 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -16,7 +16,7 @@
Latest version: 20.0.4 + href="https://docs.gunicorn.org/en/stable/">22.0.0
@@ -118,11 +118,11 @@

Project Management

  • Forum
  • Mailing list -

    Project maintenance guidelines are avaible on the wiki

    +

    Project maintenance guidelines are available on the wiki

    -

    Irc

    -

    The Gunicorn channel is on the Freenode IRC - network. You can chat with the community on the #gunicorn channel.

    +

    IRC

    +

    The Gunicorn channel is on the Libera Chat IRC + network. You can chat with the community on the #gunicorn channel.

    Issue Tracking

    Bug reports, enhancement requests and tasks generally go in the Github diff --git a/docs/source/2012-news.rst b/docs/source/2012-news.rst index 6ae37bfe6..ce4f7cc4d 100644 --- a/docs/source/2012-news.rst +++ b/docs/source/2012-news.rst @@ -75,7 +75,7 @@ Changelog - 2012 - fix tornado.wsgi.WSGIApplication calling error - **breaking change**: take the control on graceful reload back. - graceful can't be overrided anymore using the on_reload function. + graceful can't be overridden anymore using the on_reload function. 0.14.3 / 2012-05-15 ------------------- diff --git a/docs/source/2013-news.rst b/docs/source/2013-news.rst index 5f848e2cf..eb8cf556a 100644 --- a/docs/source/2013-news.rst +++ b/docs/source/2013-news.rst @@ -38,10 +38,10 @@ Changelog - 2013 - fix: give the initial global_conf to paster application - fix: fix 'Expect: 100-continue' support on Python 3 -New versionning: +New versioning: ++++++++++++++++ -With this release, the versionning of Gunicorn is changing. Gunicorn is +With this release, the versioning of Gunicorn is changing. Gunicorn is stable since a long time and there is no point to release a "1.0" now. It should have been done since a long time. 0.17 really meant it was the 17th stable version. From the beginning we have only 2 kind of @@ -49,7 +49,7 @@ releases: major release: releases with major changes or huge features added services releases: fixes and minor features added So from now we will -apply the following versionning ``.``. For example ``17.5`` is a +apply the following versioning ``.``. For example ``17.5`` is a service release. 0.17.4 / 2013-04-24 diff --git a/docs/source/2014-news.rst b/docs/source/2014-news.rst index 1a9af82bb..3eec18fcd 100644 --- a/docs/source/2014-news.rst +++ b/docs/source/2014-news.rst @@ -71,7 +71,7 @@ AioHttp worker Async worker ++++++++++++ -- fix :issue:`790`: StopIteration shouldn't be catched at this level. +- fix :issue:`790`: StopIteration shouldn't be caught at this level. Logging @@ -180,7 +180,7 @@ core - add: syslog logging can now be done to a unix socket - fix logging: don't try to redirect stdout/stderr to the logfile. - fix logging: don't propagate log -- improve logging: file option can be overriden by the gunicorn options +- improve logging: file option can be overridden by the gunicorn options `--error-logfile` and `--access-logfile` if they are given. - fix: don't override SERVER_* by the Host header - fix: handle_error diff --git a/docs/source/2019-news.rst b/docs/source/2019-news.rst index 771c28f0e..28b69216b 100644 --- a/docs/source/2019-news.rst +++ b/docs/source/2019-news.rst @@ -90,7 +90,7 @@ Changelog - 2019 - Simplify Paste Deployment documentation - Fix root logging: root and logger are same level. - Fixed typo in ssl_version documentation -- Documented systemd deployement unit examples +- Documented systemd deployment unit examples - Added systemd sd_notify support - Fixed typo in gthread.py - Added `tornado `_ 5 and 6 support diff --git a/docs/source/2020-news.rst b/docs/source/2020-news.rst index c83a67549..1d91ef7e5 100644 --- a/docs/source/2020-news.rst +++ b/docs/source/2020-news.rst @@ -5,50 +5,3 @@ Changelog - 2020 .. note:: Please see :doc:`news` for the latest changes - -Unreleased -========== - -- document WEB_CONCURRENCY is set by, at least, Heroku -- capture peername from accept: Avoid calls to getpeername by capturing the peer name returned by -accept -- log a warning when a worker was terminated due to a signal -- fix tornado usage with latest versions of Django -- add support for python -m gunicorn -- fix systemd socket activation example -- allows to set wsgi application in configg file using `wsgi_app` -- document `--timeout = 0` -- always close a connection when the number of requests exceeds the max requests -- Disable keepalive during graceful shutdown -- kill tasks in the gthread workers during upgrade -- fix latency in gevent worker when accepting new requests -- fix file watcher: handle errors when new worker reboot and ensure the list of files is kept -- document the default name and path of the configuration file -- document how variable impact configuration -- document the `$PORT` environment variable -- added milliseconds option to request_time in access_log -- added PIP requirements to be used for example -- remove version from the Server header -- fix sendfile: use `socket.sendfile` instead of `os.sendfile` -- reloader: use absolute path to prevent empty to prevent0 `InotifyError` when a file - is added to the working directory -- Add --print-config option to print the resolved settings at startup. -- remove the `--log-dict-config` CLI flag because it never had a working format - (the `logconfig_dict` setting in configuration files continues to work) - - -** Breaking changes ** - -- minimum version is Python 3.5 -- remove version from the Server header - -** Documentation ** - - - -** Others ** - -- miscellaneous changes in the code base to be a better citizen with Python 3 -- remove dead code -- fix documentation generation - diff --git a/docs/source/2021-news.rst b/docs/source/2021-news.rst index b9f27b0c2..3057600de 100644 --- a/docs/source/2021-news.rst +++ b/docs/source/2021-news.rst @@ -11,12 +11,12 @@ Changelog - 2021 - document WEB_CONCURRENCY is set by, at least, Heroku - capture peername from accept: Avoid calls to getpeername by capturing the peer name returned by -accept + accept - log a warning when a worker was terminated due to a signal - fix tornado usage with latest versions of Django - add support for python -m gunicorn - fix systemd socket activation example -- allows to set wsgi application in configg file using `wsgi_app` +- allows to set wsgi application in config file using `wsgi_app` - document `--timeout = 0` - always close a connection when the number of requests exceeds the max requests - Disable keepalive during graceful shutdown diff --git a/docs/source/2023-news.rst b/docs/source/2023-news.rst new file mode 100644 index 000000000..286f15852 --- /dev/null +++ b/docs/source/2023-news.rst @@ -0,0 +1,34 @@ +================ +Changelog - 2023 +================ + +21.2.0 - 2023-07-19 +=================== + +- fix thread worker: revert change considering connection as idle . + +*** NOTE *** + +This is fixing the bad file description error. + +21.0.1 - 2023-07-17 +=================== + +- fix documentation build + +21.0.0 - 2023-07-17 +=================== + +- support python 3.11 +- fix gevent and eventlet workers +- fix threads support (gththread): improve performance and unblock requests +- SSL: now use SSLContext object +- HTTP parser: miscellaneous fixes +- remove unnecessary setuid calls +- fix testing +- improve logging +- miscellaneous fixes to core engine + +*** RELEASE NOTE *** + +We made this release major to start our new release cycle. More info will be provided on our discussion forum. diff --git a/docs/source/community.rst b/docs/source/community.rst index 249d82c5b..e16767447 100644 --- a/docs/source/community.rst +++ b/docs/source/community.rst @@ -15,7 +15,7 @@ for 3 different purposes: * `Mailing list `_ : Discussion of Gunicorn development, new features and project management. -Project maintenance guidelines are avaible on the `wiki `_ +Project maintenance guidelines are available on the `wiki `_ . IRC diff --git a/docs/source/configure.rst b/docs/source/configure.rst index 0a8e7fe7e..dc9ba62da 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -65,6 +65,7 @@ usual:: There is also a ``--version`` flag available to the command line scripts that isn't mentioned in the list of :ref:`settings `. +.. _configuration_file: Configuration File ================== diff --git a/docs/source/deploy.rst b/docs/source/deploy.rst index 43bad9a80..6871421ab 100644 --- a/docs/source/deploy.rst +++ b/docs/source/deploy.rst @@ -38,6 +38,22 @@ To turn off buffering, you only need to add ``proxy_buffering off;`` to your } ... +If you want to ignore aborted requests like health check of Load Balancer, some +of which close the connection without waiting for a response, you need to turn +on `ignoring client abort`_. + +To ignore aborted requests, you only need to add +``proxy_ignore_client_abort on;`` to your ``location`` block:: + + ... + proxy_ignore_client_abort on; + ... + +.. note:: + The default value of ``proxy_ignore_client_abort`` is ``off``. Error code + 499 may appear in Nginx log and ``Ignoring EPIPE`` may appear in Gunicorn + log if loglevel is set to ``debug``. + It is recommended to pass protocol information to Gunicorn. Many web frameworks use this information to generate URLs. Without this information, the application may mistakenly generate 'http' URLs in @@ -216,7 +232,7 @@ A tool that is starting to be common on linux systems is Systemd_. It is a system services manager that allows for strict process management, resources and permissions control. -Below are configurations files and instructions for using systemd to create +Below are configuration files and instructions for using systemd to create a unix socket for incoming Gunicorn requests. Systemd will listen on this socket and start gunicorn automatically in response to traffic. Later in this section are instructions for configuring Nginx to forward web traffic @@ -357,3 +373,4 @@ utility:: .. _Virtualenv: https://pypi.python.org/pypi/virtualenv .. _Systemd: https://www.freedesktop.org/wiki/Software/systemd/ .. _Gaffer: https://gaffer.readthedocs.io/ +.. _`ignoring client abort`: http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_ignore_client_abort diff --git a/docs/source/design.rst b/docs/source/design.rst index 066fceafe..60e8a39aa 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -53,6 +53,15 @@ installed and `setup `_. Other applications might not be compatible at all as they, e.g., rely on the original unpatched behavior. +Gthread Workers +--------------- + +The worker `gthread` is a threaded worker. It accepts connections in the +main loop. Accepted connections are added to the thread pool as a +connection job. On keepalive connections are put back in the loop +waiting for an event. If no event happens after the keepalive timeout, +the connection is closed. + Tornado Workers --------------- @@ -68,12 +77,6 @@ AsyncIO Workers These workers are compatible with Python 3. -The worker `gthread` is a threaded worker. It accepts connections in the -main loop, accepted connections are added to the thread pool as a -connection job. On keepalive connections are put back in the loop -waiting for an event. If no event happen after the keep alive timeout, -the connection is closed. - You can port also your application to use aiohttp_'s ``web.Application`` API and use the ``aiohttp.worker.GunicornWebWorker`` worker. @@ -140,10 +143,6 @@ signal, as the application code will be shared among workers but loaded only in the worker processes (unlike when using the preload setting, which loads the code in the master process). -.. note:: - Under Python 2.x, you need to install the 'futures' package to use this - feature. - .. _Greenlets: https://github.com/python-greenlet/greenlet .. _Eventlet: http://eventlet.net/ .. _Gevent: http://www.gevent.org/ @@ -151,4 +150,4 @@ code in the master process). .. _aiohttp: https://docs.aiohttp.org/en/stable/deployment.html#nginx-gunicorn .. _`example`: https://github.com/benoitc/gunicorn/blob/master/examples/frameworks/flaskapp_aiohttp_wsgi.py .. _Psycopg: http://initd.org/psycopg/ -.. _psycogreen: https://bitbucket.org/dvarrazzo/psycogreen +.. _psycogreen: https://github.com/psycopg/psycogreen/ diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 71fb2b848..64b44c905 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -129,9 +129,13 @@ One of the first settings that usually needs to be bumped is the maximum number of open file descriptors for a given process. For the confused out there, remember that Unices treat sockets as files. -:: +.. warning:: ``sudo ulimit`` may not work + +Considering non-privileged users are not able to relax the limit, you should +firstly switch to root user, increase the limit, then run gunicorn. Using ``sudo +ulimit`` would not take effect. - $ sudo ulimit -n 2048 +Try systemd's service unit file, or an initscript which runs as root. How can I increase the maximum socket backlog? ---------------------------------------------- @@ -216,7 +220,7 @@ If you use a reverse proxy like NGINX you might see 502 returned to a client. In the gunicorn logs you might simply see ``[35] [INFO] Booting worker with pid: 35`` -It's completely normal for workers to be killed and startup, for example due to +It's completely normal for workers to be stop and start, for example due to max-requests setting. Ordinarily gunicorn will capture any signals and log something. This particular failure case is usually due to a SIGKILL being received, as it's diff --git a/docs/source/index.rst b/docs/source/index.rst index 50bb2abd8..3f89ce1eb 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,7 +7,7 @@ Gunicorn - WSGI server :Website: http://gunicorn.org :Source code: https://github.com/benoitc/gunicorn :Issue tracker: https://github.com/benoitc/gunicorn/issues -:IRC: ``#gunicorn`` on Freenode +:IRC: ``#gunicorn`` on Libera Chat :Usage questions: https://github.com/benoitc/gunicorn/issues Gunicorn 'Green Unicorn' is a Python WSGI HTTP Server for UNIX. It's a pre-fork @@ -23,7 +23,7 @@ Features * Simple Python configuration * Multiple worker configurations * Various server hooks for extensibility -* Compatible with Python 3.x >= 3.5 +* Compatible with Python 3.x >= 3.7 Contents diff --git a/docs/source/install.rst b/docs/source/install.rst index 66f2479e7..2367086df 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -4,7 +4,7 @@ Installation .. highlight:: bash -:Requirements: **Python 3.x >= 3.5** +:Requirements: **Python 3.x >= 3.7** To install the latest released version of Gunicorn:: diff --git a/docs/source/news.rst b/docs/source/news.rst index a8e379ebe..26c0fbb5f 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -2,46 +2,72 @@ Changelog ========= -20.1.0 - 2021-02-12 +22.0.0 - 2024-04-17 =================== -- document WEB_CONCURRENCY is set by, at least, Heroku -- capture peername from accept: Avoid calls to getpeername by capturing the peer name returned by accept -- log a warning when a worker was terminated due to a signal -- fix tornado usage with latest versions of Django -- add support for python -m gunicorn -- fix systemd socket activation example -- allows to set wsgi application in configg file using ``wsgi_app`` -- document ``--timeout = 0`` -- always close a connection when the number of requests exceeds the max requests -- Disable keepalive during graceful shutdown -- kill tasks in the gthread workers during upgrade -- fix latency in gevent worker when accepting new requests -- fix file watcher: handle errors when new worker reboot and ensure the list of files is kept -- document the default name and path of the configuration file -- document how variable impact configuration -- document the ``$PORT`` environment variable -- added milliseconds option to request_time in access_log -- added PIP requirements to be used for example -- remove version from the Server header -- fix sendfile: use ``socket.sendfile`` instead of ``os.sendfile`` -- reloader: use absolute path to prevent empty to prevent0 ``InotifyError`` when a file - is added to the working directory -- Add --print-config option to print the resolved settings at startup. -- remove the ``--log-dict-config`` CLI flag because it never had a working format - (the ``logconfig_dict`` setting in configuration files continues to work) +- use `utime` to notify workers liveness +- migrate setup to pyproject.toml +- fix numerous security vulnerabilities in HTTP parser (closing some request smuggling vectors) +- parsing additional requests is no longer attempted past unsupported request framing +- on HTTP versions < 1.1 support for chunked transfer is refused (only used in exploits) +- requests conflicting configured or passed SCRIPT_NAME now produce a verbose error +- Trailer fields are no longer inspected for headers indicating secure scheme +- support Python 3.12 ** Breaking changes ** -- minimum version is Python 3.5 -- remove version from the Server header +- minimum version is Python 3.7 +- the limitations on valid characters in the HTTP method have been bounded to Internet Standards +- requests specifying unsupported transfer coding (order) are refused by default (rare) +- HTTP methods are no longer casefolded by default (IANA method registry contains none affected) +- HTTP methods containing the number sign (#) are no longer accepted by default (rare) +- HTTP versions < 1.0 or >= 2.0 are no longer accepted by default (rare, only HTTP/1.1 is supported) +- HTTP versions consisting of multiple digits or containing a prefix/suffix are no longer accepted +- HTTP header field names Gunicorn cannot safely map to variables are silently dropped, as in other software +- HTTP headers with empty field name are refused by default (no legitimate use cases, used in exploits) +- requests with both Transfer-Encoding and Content-Length are refused by default (such a message might indicate an attempt to perform request smuggling) +- empty transfer codings are no longer permitted (reportedly seen with really old & broken proxies) -** Others ** -- miscellaneous changes in the code base to be a better citizen with Python 3 -- remove dead code -- fix documentation generation +** SECURITY ** +- fix CVE-2024-1135 + +21.2.0 - 2023-07-19 +=================== + +- fix thread worker: revert change considering connection as idle . + +*** NOTE *** + +This is fixing the bad file description error. + +21.1.0 - 2023-07-18 +=================== + +- fix thread worker: fix socket removal from the queue + +21.0.1 - 2023-07-17 +=================== + +- fix documentation build + +21.0.0 - 2023-07-17 +=================== + +- support python 3.11 +- fix gevent and eventlet workers +- fix threads support (gththread): improve performance and unblock requests +- SSL: noaw use SSLContext object +- HTTP parser: miscellaneous fixes +- remove unnecessary setuid calls +- fix testing +- improve logging +- miscellaneous fixes to core engine + +*** RELEASE NOTE *** + +We made this release major to start our new release cycle. More info will be provided on our discussion forum. History ======= @@ -49,6 +75,9 @@ History .. toctree:: :titlesonly: + 2024-news + 2023-news + 2021-news 2020-news 2019-news 2018-news diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 6e977d1a0..4e0c11877 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -32,7 +32,7 @@ Config File **Default:** ``'./gunicorn.conf.py'`` -The Gunicorn config file. +:ref:`The Gunicorn config file`. A string of the form ``PATH``, ``file:PATH``, or ``python:MODULE_NAME``. @@ -210,7 +210,7 @@ H protocol s status B response length b response length or ``'-'`` (CLF format) -f referer +f referrer a user agent T request time in seconds M request time in milliseconds @@ -310,19 +310,36 @@ file format. ``logconfig_dict`` ~~~~~~~~~~~~~~~~~~ -**Command line:** ``--log-config-dict`` - **Default:** ``{}`` The log config dictionary to use, using the standard Python logging module's dictionary configuration format. This option -takes precedence over the :ref:`logconfig` option, which uses the -older file configuration format. +takes precedence over the :ref:`logconfig` and :ref:`logConfigJson` options, +which uses the older file configuration format and JSON +respectively. Format: https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig +For more context you can look at the default configuration dictionary for logging, +which can be found at ``gunicorn.glogging.CONFIG_DEFAULTS``. + .. versionadded:: 19.8 +.. _logconfig-json: + +``logconfig_json`` +~~~~~~~~~~~~~~~~~~ + +**Command line:** ``--log-config-json FILE`` + +**Default:** ``None`` + +The log config to read config from a JSON file + +Format: https://docs.python.org/3/library/logging.config.html#logging.config.jsonConfig + +.. versionadded:: 20.0 + .. _syslog-addr: ``syslog_addr`` @@ -407,7 +424,12 @@ environment variable ``PYTHONUNBUFFERED`` . **Default:** ``None`` -``host:port`` of the statsd server to log to. +The address of the StatsD server to log to. + +Address is a string of the form: + +* ``unix://PATH`` : for a unix domain socket. +* ``HOST:PORT`` : for a network address .. versionadded:: 19.1 @@ -503,7 +525,10 @@ SSL certificate file **Default:** ``<_SSLMethod.PROTOCOL_TLS: 2>`` -SSL version to use. +SSL version to use (see stdlib ssl module's). + +.. deprecated:: 20.2 + The option is deprecated and it is currently ignored. Use :ref:`ssl-context` instead. ============= ============ --ssl-version Description @@ -526,6 +551,9 @@ TLS_SERVER Auto-negotiate the highest protocol version like TLS, .. versionchanged:: 20.0 This setting now accepts string names based on ``ssl.PROTOCOL_`` constants. +.. versionchanged:: 20.0.1 + The default value has been changed from ``ssl.PROTOCOL_SSLv23`` to + ``ssl.PROTOCOL_TLS`` when Python >= 3.6 . .. _cert-reqs: @@ -538,6 +566,14 @@ TLS_SERVER Auto-negotiate the highest protocol version like TLS, Whether client certificate is required (see stdlib ssl module's) +=========== =========================== +--cert-reqs Description +=========== =========================== +`0` no client verification +`1` ssl.CERT_OPTIONAL +`2` ssl.CERT_REQUIRED +=========== =========================== + .. _ca-certs: ``ca_certs`` @@ -818,7 +854,7 @@ The callable needs to accept a single instance variable for the Arbiter. .. code-block:: python def pre_request(worker, req): - worker.log.debug("%s %s" % (req.method, req.path)) + worker.log.debug("%s %s", req.method, req.path) Called just before a worker processes the request. @@ -914,6 +950,40 @@ Called just before exiting Gunicorn. The callable needs to accept a single instance variable for the Arbiter. +.. _ssl-context: + +``ssl_context`` +~~~~~~~~~~~~~~~ + +**Default:** + +.. code-block:: python + + def ssl_context(config, default_ssl_context_factory): + return default_ssl_context_factory() + +Called when SSLContext is needed. + +Allows customizing SSL context. + +The callable needs to accept an instance variable for the Config and +a factory function that returns default SSLContext which is initialized +with certificates, private key, cert_reqs, and ciphers according to +config and can be further customized by the callable. +The callable needs to return SSLContext object. + +Following example shows a configuration file that sets the minimum TLS version to 1.3: + +.. code-block:: python + + def ssl_context(conf, default_ssl_context_factory): + import ssl + context = default_ssl_context_factory() + context.minimum_version = ssl.TLSVersion.TLSv1_3 + return context + +.. versionadded:: 20.2 + Server Mechanics ---------------- @@ -974,7 +1044,7 @@ Set the ``SO_REUSEPORT`` flag on the listening socket. **Command line:** ``--chdir`` -**Default:** ``'/Users/chainz/Documents/Projects/gunicorn/docs'`` +**Default:** ``'.'`` Change directory to specified directory before loading apps. @@ -1058,7 +1128,7 @@ If not set, the default temporary directory will be used. **Command line:** ``-u USER`` or ``--user USER`` -**Default:** ``501`` +**Default:** ``os.geteuid()`` Switch worker processes to run as this user. @@ -1073,7 +1143,7 @@ change the worker process user. **Command line:** ``-g GROUP`` or ``--group GROUP`` -**Default:** ``20`` +**Default:** ``os.getegid()`` Switch worker process to run as this group. @@ -1137,10 +1207,16 @@ temporary directory. **Default:** ``{'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-SSL': 'on'}`` A dictionary containing headers and values that the front-end proxy -uses to indicate HTTPS requests. These tell Gunicorn to set +uses to indicate HTTPS requests. If the source IP is permitted by +``forwarded-allow-ips`` (below), *and* at least one request header matches +a key-value pair listed in this dictionary, then Gunicorn will set ``wsgi.url_scheme`` to ``https``, so your application can tell that the request is secure. +If the other headers listed in this dictionary are not present in the request, they will be ignored, +but if the other headers are present and do not match the provided values, then +the request will fail to parse. See the note below for more detailed examples of this behaviour. + The dictionary should map upper-case header names to exact string values. The value comparisons are case-sensitive, unlike the header names, so make sure they're exactly what your front-end proxy sends @@ -1168,6 +1244,69 @@ you still trust the environment). By default, the value of the ``FORWARDED_ALLOW_IPS`` environment variable. If it is not defined, the default is ``"127.0.0.1"``. +.. note:: + + The interplay between the request headers, the value of ``forwarded_allow_ips``, and the value of + ``secure_scheme_headers`` is complex. Various scenarios are documented below to further elaborate. + In each case, we have a request from the remote address 134.213.44.18, and the default value of + ``secure_scheme_headers``: + + .. code:: + + secure_scheme_headers = { + 'X-FORWARDED-PROTOCOL': 'ssl', + 'X-FORWARDED-PROTO': 'https', + 'X-FORWARDED-SSL': 'on' + } + + + .. list-table:: + :header-rows: 1 + :align: center + :widths: auto + + * - ``forwarded-allow-ips`` + - Secure Request Headers + - Result + - Explanation + * - .. code:: + + ["127.0.0.1"] + - .. code:: + + X-Forwarded-Proto: https + - .. code:: + + wsgi.url_scheme = "http" + - IP address was not allowed + * - .. code:: + + "*" + - + - .. code:: + + wsgi.url_scheme = "http" + - IP address allowed, but no secure headers provided + * - .. code:: + + "*" + - .. code:: + + X-Forwarded-Proto: https + - .. code:: + + wsgi.url_scheme = "https" + - IP address allowed, one request header matched + * - .. code:: + + ["134.213.44.18"] + - .. code:: + + X-Forwarded-Ssl: on + X-Forwarded-Proto: http + - ``InvalidSchemeHeaders()`` raised + - IP address allowed, but the two secure headers disagreed on if HTTPS was used + .. _pythonpath: ``pythonpath`` @@ -1340,8 +1479,9 @@ A positive integer generally in the ``2-4 x $(NUM_CORES)`` range. You'll want to vary this a bit to find the best for your particular application's work load. -By default, the value of the ``WEB_CONCURRENCY`` environment variable. -If it is not defined, the default is ``1``. +By default, the value of the ``WEB_CONCURRENCY`` environment variable, +which is set by some Platform-as-a-Service providers such as Heroku. If +it is not defined, the default is ``1``. .. _worker-class: @@ -1413,7 +1553,7 @@ This setting only affects the Gthread worker type. The maximum number of simultaneous clients. -This setting only affects the Eventlet and Gevent worker types. +This setting only affects the ``gthread``, ``eventlet`` and ``gevent`` worker types. .. _max-requests: diff --git a/examples/example_config.py b/examples/example_config.py index f8f3c1dfc..5a399a497 100644 --- a/examples/example_config.py +++ b/examples/example_config.py @@ -214,3 +214,27 @@ def worker_int(worker): def worker_abort(worker): worker.log.info("worker received SIGABRT signal") + +def ssl_context(conf, default_ssl_context_factory): + import ssl + + # The default SSLContext returned by the factory function is initialized + # with the TLS parameters from config, including TLS certificates and other + # parameters. + context = default_ssl_context_factory() + + # The SSLContext can be further customized, for example by enforcing + # minimum TLS version. + context.minimum_version = ssl.TLSVersion.TLSv1_3 + + # Server can also return different server certificate depending which + # hostname the client uses. Requires Python 3.7 or later. + def sni_callback(socket, server_hostname, context): + if server_hostname == "foo.127.0.0.1.nip.io": + new_context = default_ssl_context_factory() + new_context.load_cert_chain(certfile="foo.pem", keyfile="foo-key.pem") + socket.context = new_context + + context.sni_callback = sni_callback + + return context diff --git a/examples/frameworks/django/testing/testing/apps/someapp/tests.py b/examples/frameworks/django/testing/testing/apps/someapp/tests.py index 3b3114889..85c920bb8 100755 --- a/examples/frameworks/django/testing/testing/apps/someapp/tests.py +++ b/examples/frameworks/django/testing/testing/apps/someapp/tests.py @@ -12,7 +12,7 @@ def test_basic_addition(self): """ Tests that 1 + 1 always equals 2. """ - self.failUnlessEqual(1 + 1, 2) + self.assertEqual(1 + 1, 2) __test__ = {"doctest": """ Another way to test that 1 + 1 is equal to 2. diff --git a/gunicorn/__init__.py b/gunicorn/__init__.py index 29edada5f..70153f8e6 100644 --- a/gunicorn/__init__.py +++ b/gunicorn/__init__.py @@ -3,7 +3,7 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -version_info = (20, 1, 0) +version_info = (22, 0, 0) __version__ = ".".join([str(v) for v in version_info]) SERVER = "gunicorn" SERVER_SOFTWARE = "%s/%s" % (SERVER, __version__) diff --git a/gunicorn/__main__.py b/gunicorn/__main__.py index 49ba6960d..fda831570 100644 --- a/gunicorn/__main__.py +++ b/gunicorn/__main__.py @@ -4,4 +4,8 @@ # See the NOTICE for more information. from gunicorn.app.wsgiapp import run -run() + +if __name__ == "__main__": + # see config.py - argparse defaults to basename(argv[0]) == "__main__.py" + # todo: let runpy.run_module take care of argv[0] rewriting + run(prog="gunicorn") diff --git a/gunicorn/app/base.py b/gunicorn/app/base.py index df8c666f2..dbd05bc7f 100644 --- a/gunicorn/app/base.py +++ b/gunicorn/app/base.py @@ -218,6 +218,11 @@ def run(self): debug.spew() if self.cfg.daemon: + if os.environ.get('NOTIFY_SOCKET'): + msg = "Warning: you shouldn't specify `daemon = True`" \ + " when launching by systemd with `Type = notify`" + print(msg, file=sys.stderr, flush=True) + util.daemonize(self.cfg.enable_stdio_inheritance) # set python paths diff --git a/gunicorn/app/wsgiapp.py b/gunicorn/app/wsgiapp.py index 36cfba9d8..4e0031234 100644 --- a/gunicorn/app/wsgiapp.py +++ b/gunicorn/app/wsgiapp.py @@ -58,13 +58,13 @@ def load(self): return self.load_wsgiapp() -def run(): +def run(prog=None): """\ The ``gunicorn`` command line runner for launching Gunicorn with generic WSGI applications. """ from gunicorn.app.wsgiapp import WSGIApplication - WSGIApplication("%(prog)s [OPTIONS] [APP_MODULE]").run() + WSGIApplication("%(prog)s [OPTIONS] [APP_MODULE]", prog=prog).run() if __name__ == '__main__': diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 24ec38744..1cf436748 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -109,7 +109,7 @@ def setup(self, app): in sorted(self.cfg.settings.items(), key=lambda setting: setting[1])))) - # set enviroment' variables + # set environment' variables if self.cfg.env: for k, v in self.cfg.env.items(): os.environ[k] = v @@ -154,7 +154,7 @@ def start(self): self.LISTENERS = sock.create_sockets(self.cfg, self.log, fds) - listeners_str = ",".join([str(l) for l in self.LISTENERS]) + listeners_str = ",".join([str(lnr) for lnr in self.LISTENERS]) self.log.debug("Arbiter booted") self.log.info("Listening at: %s (%s)", listeners_str, self.pid) self.log.info("Using worker: %s", self.cfg.worker_class_str) @@ -230,8 +230,8 @@ def run(self): except SystemExit: raise except Exception: - self.log.info("Unhandled exception in main loop", - exc_info=True) + self.log.error("Unhandled exception in main loop", + exc_info=True) self.stop(False) if self.pidfile is not None: self.pidfile.unlink() @@ -340,9 +340,12 @@ def wakeup(self): def halt(self, reason=None, exit_status=0): """ halt arbiter """ self.stop() - self.log.info("Shutting down: %s", self.master_name) + + log_func = self.log.info if exit_status == 0 else self.log.error + log_func("Shutting down: %s", self.master_name) if reason is not None: - self.log.info("Reason: %s", reason) + log_func("Reason: %s", reason) + if self.pidfile is not None: self.pidfile.unlink() self.cfg.on_exit(self) @@ -421,7 +424,7 @@ def reexec(self): environ['LISTEN_FDS'] = str(len(self.LISTENERS)) else: environ['GUNICORN_FD'] = ','.join( - str(l.fileno()) for l in self.LISTENERS) + str(lnr.fileno()) for lnr in self.LISTENERS) os.chdir(self.START_CTX['cwd']) @@ -454,11 +457,11 @@ def reload(self): # do we need to change listener ? if old_address != self.cfg.address: # close all listeners - for l in self.LISTENERS: - l.close() + for lnr in self.LISTENERS: + lnr.close() # init new listeners self.LISTENERS = sock.create_sockets(self.cfg, self.log) - listeners_str = ",".join([str(l) for l in self.LISTENERS]) + listeners_str = ",".join([str(lnr) for lnr in self.LISTENERS]) self.log.info("Listening at: %s", listeners_str) # do some actions on reload @@ -492,7 +495,7 @@ def murder_workers(self): workers = list(self.WORKERS.items()) for (pid, worker) in workers: try: - if time.time() - worker.tmp.last_update() <= self.timeout: + if time.monotonic() - worker.tmp.last_update() <= self.timeout: continue except (OSError, ValueError): continue @@ -520,18 +523,35 @@ def reap_workers(self): # that it could not boot, we'll shut it down to avoid # infinite start/stop cycles. exitcode = status >> 8 + if exitcode != 0: + self.log.error('Worker (pid:%s) exited with code %s', wpid, exitcode) if exitcode == self.WORKER_BOOT_ERROR: reason = "Worker failed to boot." raise HaltServer(reason, self.WORKER_BOOT_ERROR) if exitcode == self.APP_LOAD_ERROR: reason = "App failed to load." raise HaltServer(reason, self.APP_LOAD_ERROR) - if os.WIFSIGNALED(status): - self.log.warning( - "Worker with pid %s was terminated due to signal %s", - wpid, - os.WTERMSIG(status) - ) + + if exitcode > 0: + # If the exit code of the worker is greater than 0, + # let the user know. + self.log.error("Worker (pid:%s) exited with code %s.", + wpid, exitcode) + elif status > 0: + # If the exit code of the worker is 0 and the status + # is greater than 0, then it was most likely killed + # via a signal. + try: + sig_name = signal.Signals(status).name + except ValueError: + sig_name = "code {}".format(status) + msg = "Worker (pid:{}) was sent {}!".format( + wpid, sig_name) + + # Additional hint for SIGKILL + if status == signal.SIGKILL: + msg += " Perhaps out of memory?" + self.log.error(msg) worker = self.WORKERS.pop(wpid, None) if not worker: diff --git a/gunicorn/config.py b/gunicorn/config.py index 8fd281bea..144acaecc 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -106,7 +106,7 @@ def worker_class_str(self): # are we using a threaded worker? is_sync = uri.endswith('SyncWorker') or uri == 'sync' if is_sync and self.threads > 1: - return "threads" + return "gthread" return uri @property @@ -364,25 +364,9 @@ def validate_pos_int(val): def validate_ssl_version(val): - ssl_versions = {} - for protocol in [p for p in dir(ssl) if p.startswith("PROTOCOL_")]: - ssl_versions[protocol[9:]] = getattr(ssl, protocol) - if val in ssl_versions: - # string matching PROTOCOL_... - return ssl_versions[val] - - try: - intval = validate_pos_int(val) - if intval in ssl_versions.values(): - # positive int matching a protocol int constant - return intval - except (ValueError, TypeError): - # negative integer or not an integer - # drop this in favour of the more descriptive ValueError below - pass - - raise ValueError("Invalid ssl_version: %s. Valid options: %s" - % (val, ', '.join(ssl_versions))) + if val != SSLVersion.default: + sys.stderr.write("Warning: option `ssl_version` is deprecated and it is ignored. Use ssl_context instead.\n") + return val def validate_string(val): @@ -448,7 +432,7 @@ def _validate_callable(val): raise TypeError(str(e)) except AttributeError: raise TypeError("Can not load '%s' from '%s'" - "" % (obj_name, mod_name)) + "" % (obj_name, mod_name)) if not callable(val): raise TypeError("Value is not callable: %s" % val) if arity != -1 and arity != util.get_arity(val): @@ -514,15 +498,25 @@ def validate_chdir(val): return path -def validate_hostport(val): +def validate_statsd_address(val): val = validate_string(val) if val is None: return None - elements = val.split(":") - if len(elements) == 2: - return (elements[0], int(elements[1])) - else: - raise TypeError("Value must consist of: hostname:port") + + # As of major release 20, util.parse_address would recognize unix:PORT + # as a UDS address, breaking backwards compatibility. We defend against + # that regression here (this is also unit-tested). + # Feel free to remove in the next major release. + unix_hostname_regression = re.match(r'^unix:(\d+)$', val) + if unix_hostname_regression: + return ('unix', int(unix_hostname_regression.group(1))) + + try: + address = util.parse_address(val, default_port='8125') + except RuntimeError: + raise TypeError("Value must be one of ('host:port', 'unix://PATH')") + + return address def validate_reload_engine(val): @@ -548,7 +542,7 @@ class ConfigFile(Setting): validator = validate_string default = "./gunicorn.conf.py" desc = """\ - The Gunicorn config file. + :ref:`The Gunicorn config file`. A string of the form ``PATH``, ``file:PATH``, or ``python:MODULE_NAME``. @@ -563,6 +557,7 @@ class ConfigFile(Setting): prefix. """ + class WSGIApp(Setting): name = "wsgi_app" section = "Config File" @@ -575,6 +570,7 @@ class WSGIApp(Setting): .. versionadded:: 20.1.0 """ + class Bind(Setting): name = "bind" action = "append" @@ -724,7 +720,7 @@ class WorkerConnections(Setting): desc = """\ The maximum number of simultaneous clients. - This setting only affects the Eventlet and Gevent worker types. + This setting only affects the ``gthread``, ``eventlet`` and ``gevent`` worker types. """ @@ -1054,6 +1050,7 @@ class Chdir(Setting): cli = ["--chdir"] validator = validate_chdir default = util.getcwd() + default_doc = "``'.'``" desc = """\ Change directory to specified directory before loading apps. """ @@ -1145,6 +1142,7 @@ class User(Setting): meta = "USER" validator = validate_user default = os.geteuid() + default_doc = "``os.geteuid()``" desc = """\ Switch worker processes to run as this user. @@ -1161,6 +1159,7 @@ class Group(Setting): meta = "GROUP" validator = validate_group default = os.getegid() + default_doc = "``os.getegid()``" desc = """\ Switch worker process to run as this group. @@ -1236,10 +1235,16 @@ class SecureSchemeHeader(Setting): desc = """\ A dictionary containing headers and values that the front-end proxy - uses to indicate HTTPS requests. These tell Gunicorn to set + uses to indicate HTTPS requests. If the source IP is permitted by + ``forwarded-allow-ips`` (below), *and* at least one request header matches + a key-value pair listed in this dictionary, then Gunicorn will set ``wsgi.url_scheme`` to ``https``, so your application can tell that the request is secure. + If the other headers listed in this dictionary are not present in the request, they will be ignored, + but if the other headers are present and do not match the provided values, then + the request will fail to parse. See the note below for more detailed examples of this behaviour. + The dictionary should map upper-case header names to exact string values. The value comparisons are case-sensitive, unlike the header names, so make sure they're exactly what your front-end proxy sends @@ -1267,6 +1272,71 @@ class ForwardedAllowIPS(Setting): By default, the value of the ``FORWARDED_ALLOW_IPS`` environment variable. If it is not defined, the default is ``"127.0.0.1"``. + + .. note:: + + The interplay between the request headers, the value of ``forwarded_allow_ips``, and the value of + ``secure_scheme_headers`` is complex. Various scenarios are documented below to further elaborate. + In each case, we have a request from the remote address 134.213.44.18, and the default value of + ``secure_scheme_headers``: + + .. code:: + + secure_scheme_headers = { + 'X-FORWARDED-PROTOCOL': 'ssl', + 'X-FORWARDED-PROTO': 'https', + 'X-FORWARDED-SSL': 'on' + } + + + .. list-table:: + :header-rows: 1 + :align: center + :widths: auto + + * - ``forwarded-allow-ips`` + - Secure Request Headers + - Result + - Explanation + * - .. code:: + + ["127.0.0.1"] + - .. code:: + + X-Forwarded-Proto: https + - .. code:: + + wsgi.url_scheme = "http" + - IP address was not allowed + * - .. code:: + + "*" + - + - .. code:: + + wsgi.url_scheme = "http" + - IP address allowed, but no secure headers provided + * - .. code:: + + "*" + - .. code:: + + X-Forwarded-Proto: https + - .. code:: + + wsgi.url_scheme = "https" + - IP address allowed, one request header matched + * - .. code:: + + ["134.213.44.18"] + - .. code:: + + X-Forwarded-Ssl: on + X-Forwarded-Proto: http + - ``InvalidSchemeHeaders()`` raised + - IP address allowed, but the two secure headers disagreed on if HTTPS was used + + """ @@ -1434,15 +1504,35 @@ class LogConfigDict(Setting): desc = """\ The log config dictionary to use, using the standard Python logging module's dictionary configuration format. This option - takes precedence over the :ref:`logconfig` option, which uses the - older file configuration format. + takes precedence over the :ref:`logconfig` and :ref:`logConfigJson` options, + which uses the older file configuration format and JSON + respectively. Format: https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig + For more context you can look at the default configuration dictionary for logging, + which can be found at ``gunicorn.glogging.CONFIG_DEFAULTS``. + .. versionadded:: 19.8 """ +class LogConfigJson(Setting): + name = "logconfig_json" + section = "Logging" + cli = ["--log-config-json"] + meta = "FILE" + validator = validate_string + default = None + desc = """\ + The log config to read config from a JSON file + + Format: https://docs.python.org/3/library/logging.config.html#logging.config.jsonConfig + + .. versionadded:: 20.0 + """ + + class SyslogTo(Setting): name = "syslog_addr" section = "Logging" @@ -1540,13 +1630,19 @@ class StatsdHost(Setting): cli = ["--statsd-host"] meta = "STATSD_ADDR" default = None - validator = validate_hostport + validator = validate_statsd_address desc = """\ - ``host:port`` of the statsd server to log to. + The address of the StatsD server to log to. + + Address is a string of the form: + + * ``unix://PATH`` : for a unix domain socket. + * ``HOST:PORT`` : for a network address .. versionadded:: 19.1 """ + # Datadog Statsd (dogstatsd) tags. https://docs.datadoghq.com/developers/dogstatsd/ class DogstatsdTags(Setting): name = "dogstatsd_tags" @@ -1562,6 +1658,7 @@ class DogstatsdTags(Setting): .. versionadded:: 20 """ + class StatsdPrefix(Setting): name = "statsd_prefix" section = "Logging" @@ -1799,7 +1896,7 @@ class PreRequest(Setting): type = callable def pre_request(worker, req): - worker.log.debug("%s %s" % (req.method, req.path)) + worker.log.debug("%s %s", req.method, req.path) default = staticmethod(pre_request) desc = """\ Called just before a worker processes the request. @@ -1898,6 +1995,41 @@ def on_exit(server): """ +class NewSSLContext(Setting): + name = "ssl_context" + section = "Server Hooks" + validator = validate_callable(2) + type = callable + + def ssl_context(config, default_ssl_context_factory): + return default_ssl_context_factory() + + default = staticmethod(ssl_context) + desc = """\ + Called when SSLContext is needed. + + Allows customizing SSL context. + + The callable needs to accept an instance variable for the Config and + a factory function that returns default SSLContext which is initialized + with certificates, private key, cert_reqs, and ciphers according to + config and can be further customized by the callable. + The callable needs to return SSLContext object. + + Following example shows a configuration file that sets the minimum TLS version to 1.3: + + .. code-block:: python + + def ssl_context(conf, default_ssl_context_factory): + import ssl + context = default_ssl_context_factory() + context.minimum_version = ssl.TLSVersion.TLSv1_3 + return context + + .. versionadded:: 21.0 + """ + + class ProxyProtocol(Setting): name = "proxy_protocol" section = "Server Mechanics" @@ -1974,17 +2106,12 @@ class SSLVersion(Setting): else: default = ssl.PROTOCOL_SSLv23 - desc = """\ - SSL version to use (see stdlib ssl module's) - - .. versionchanged:: 20.0.1 - The default value has been changed from ``ssl.PROTOCOL_SSLv23`` to - ``ssl.PROTOCOL_TLS`` when Python >= 3.6 . - - """ default = ssl.PROTOCOL_SSLv23 desc = """\ - SSL version to use. + SSL version to use (see stdlib ssl module's). + + .. deprecated:: 21.0 + The option is deprecated and it is currently ignored. Use :ref:`ssl-context` instead. ============= ============ --ssl-version Description @@ -2007,6 +2134,9 @@ class SSLVersion(Setting): .. versionchanged:: 20.0 This setting now accepts string names based on ``ssl.PROTOCOL_`` constants. + .. versionchanged:: 20.0.1 + The default value has been changed from ``ssl.PROTOCOL_SSLv23`` to + ``ssl.PROTOCOL_TLS`` when Python >= 3.6 . """ @@ -2018,6 +2148,14 @@ class CertReqs(Setting): default = ssl.CERT_NONE desc = """\ Whether client certificate is required (see stdlib ssl module's) + + =========== =========================== + --cert-reqs Description + =========== =========================== + `0` no client veirifcation + `1` ssl.CERT_OPTIONAL + `2` ssl.CERT_REQUIRED + =========== =========================== """ @@ -2095,7 +2233,7 @@ class PasteGlobalConf(Setting): The option can be specified multiple times. - The variables are passed to the the PasteDeploy entrypoint. Example:: + The variables are passed to the PasteDeploy entrypoint. Example:: $ gunicorn -b 127.0.0.1:8000 --paste development.ini --paste-global FOO=1 --paste-global BAR=2 @@ -2116,5 +2254,131 @@ class StripHeaderSpaces(Setting): This is known to induce vulnerabilities and is not compliant with the HTTP/1.1 standard. See https://portswigger.net/research/http-desync-attacks-request-smuggling-reborn. - Use with care and only if necessary. + Use with care and only if necessary. May be removed in a future version. + + .. versionadded:: 20.0.1 + """ + + +class PermitUnconventionalHTTPMethod(Setting): + name = "permit_unconventional_http_method" + section = "Server Mechanics" + cli = ["--permit-unconventional-http-method"] + validator = validate_bool + action = "store_true" + default = False + desc = """\ + Permit HTTP methods not matching conventions, such as IANA registration guidelines + + This permits request methods of length less than 3 or more than 20, + methods with lowercase characters or methods containing the # character. + HTTP methods are case sensitive by definition, and merely uppercase by convention. + + This option is provided to diagnose backwards-incompatible changes. + + Use with care and only if necessary. May be removed in a future version. + + .. versionadded:: 22.0.0 + """ + + +class PermitUnconventionalHTTPVersion(Setting): + name = "permit_unconventional_http_version" + section = "Server Mechanics" + cli = ["--permit-unconventional-http-version"] + validator = validate_bool + action = "store_true" + default = False + desc = """\ + Permit HTTP version not matching conventions of 2023 + + This disables the refusal of likely malformed request lines. + It is unusual to specify HTTP 1 versions other than 1.0 and 1.1. + + This option is provided to diagnose backwards-incompatible changes. + Use with care and only if necessary. May be removed in a future version. + + .. versionadded:: 22.0.0 + """ + + +class CasefoldHTTPMethod(Setting): + name = "casefold_http_method" + section = "Server Mechanics" + cli = ["--casefold-http-method"] + validator = validate_bool + action = "store_true" + default = False + desc = """\ + Transform received HTTP methods to uppercase + + HTTP methods are case sensitive by definition, and merely uppercase by convention. + + This option is provided because previous versions of gunicorn defaulted to this behaviour. + + Use with care and only if necessary. May be removed in a future version. + + .. versionadded:: 22.0.0 + """ + + +def validate_header_map_behaviour(val): + # FIXME: refactor all of this subclassing stdlib argparse + + if val is None: + return + + if not isinstance(val, str): + raise TypeError("Invalid type for casting: %s" % val) + if val.lower().strip() == "drop": + return "drop" + elif val.lower().strip() == "refuse": + return "refuse" + elif val.lower().strip() == "dangerous": + return "dangerous" + else: + raise ValueError("Invalid header map behaviour: %s" % val) + + +class HeaderMap(Setting): + name = "header_map" + section = "Server Mechanics" + cli = ["--header-map"] + validator = validate_header_map_behaviour + default = "drop" + desc = """\ + Configure how header field names are mapped into environ + + Headers containing underscores are permitted by RFC9110, + but gunicorn joining headers of different names into + the same environment variable will dangerously confuse applications as to which is which. + + The safe default ``drop`` is to silently drop headers that cannot be unambiguously mapped. + The value ``refuse`` will return an error if a request contains *any* such header. + The value ``dangerous`` matches the previous, not advisabble, behaviour of mapping different + header field names into the same environ name. + + Use with care and only if necessary and after considering if your problem could + instead be solved by specifically renaming or rewriting only the intended headers + on a proxy in front of Gunicorn. + + .. versionadded:: 22.0.0 + """ + + +class TolerateDangerousFraming(Setting): + name = "tolerate_dangerous_framing" + section = "Server Mechanics" + cli = ["--tolerate-dangerous-framing"] + validator = validate_bool + action = "store_true" + default = False + desc = """\ + Process requests with both Transfer-Encoding and Content-Length + + This is known to induce vulnerabilities, but not strictly forbidden by RFC9112. + + Use with care and only if necessary. May be removed in a future version. + + .. versionadded:: 22.0.0 """ diff --git a/gunicorn/debug.py b/gunicorn/debug.py index 996fe1b4b..a492df9e4 100644 --- a/gunicorn/debug.py +++ b/gunicorn/debug.py @@ -28,7 +28,7 @@ def __call__(self, frame, event, arg): if '__file__' in frame.f_globals: filename = frame.f_globals['__file__'] if (filename.endswith('.pyc') or - filename.endswith('.pyo')): + filename.endswith('.pyo')): filename = filename[:-1] name = frame.f_globals['__name__'] line = linecache.getline(filename, lineno) diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index 08bc121a1..b552e26a8 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -5,6 +5,7 @@ import base64 import binascii +import json import time import logging logging.Logger.manager.emittedNoHandlerWarning = 1 # noqa @@ -44,53 +45,51 @@ "local7": 23 } +CONFIG_DEFAULTS = { + "version": 1, + "disable_existing_loggers": False, + "root": {"level": "INFO", "handlers": ["console"]}, + "loggers": { + "gunicorn.error": { + "level": "INFO", + "handlers": ["error_console"], + "propagate": True, + "qualname": "gunicorn.error" + }, -CONFIG_DEFAULTS = dict( - version=1, - disable_existing_loggers=False, - - root={"level": "INFO", "handlers": ["console"]}, - loggers={ - "gunicorn.error": { - "level": "INFO", - "handlers": ["error_console"], - "propagate": True, - "qualname": "gunicorn.error" - }, - - "gunicorn.access": { - "level": "INFO", - "handlers": ["console"], - "propagate": True, - "qualname": "gunicorn.access" - } + "gunicorn.access": { + "level": "INFO", + "handlers": ["console"], + "propagate": True, + "qualname": "gunicorn.access" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "generic", + "stream": "ext://sys.stdout" }, - handlers={ - "console": { - "class": "logging.StreamHandler", - "formatter": "generic", - "stream": "ext://sys.stdout" - }, - "error_console": { - "class": "logging.StreamHandler", - "formatter": "generic", - "stream": "ext://sys.stderr" - }, + "error_console": { + "class": "logging.StreamHandler", + "formatter": "generic", + "stream": "ext://sys.stderr" }, - formatters={ - "generic": { - "format": "%(asctime)s [%(process)d] [%(levelname)s] %(message)s", - "datefmt": "[%Y-%m-%d %H:%M:%S %z]", - "class": "logging.Formatter" - } + }, + "formatters": { + "generic": { + "format": "%(asctime)s [%(process)d] [%(levelname)s] %(message)s", + "datefmt": "[%Y-%m-%d %H:%M:%S %z]", + "class": "logging.Formatter" } -) + } +} def loggers(): """ get list of all loggers """ root = logging.root - existing = root.manager.loggerDict.keys() + existing = list(root.manager.loggerDict.keys()) return [logging.getLogger(name) for name in existing] @@ -240,6 +239,21 @@ def setup(self, cfg): TypeError ) as exc: raise RuntimeError(str(exc)) + elif cfg.logconfig_json: + config = CONFIG_DEFAULTS.copy() + if os.path.exists(cfg.logconfig_json): + try: + config_json = json.load(open(cfg.logconfig_json)) + config.update(config_json) + dictConfig(config) + except ( + json.JSONDecodeError, + AttributeError, + ImportError, + ValueError, + TypeError + ) as exc: + raise RuntimeError(str(exc)) elif cfg.logconfig: if os.path.exists(cfg.logconfig): defaults = CONFIG_DEFAULTS.copy() @@ -275,7 +289,7 @@ def log(self, lvl, msg, *args, **kwargs): self.error_log.log(lvl, msg, *args, **kwargs) def atoms(self, resp, req, environ, request_time): - """ Gets atoms for log formating. + """ Gets atoms for log formatting. """ status = resp.status if isinstance(status, str): @@ -299,7 +313,7 @@ def atoms(self, resp, req, environ, request_time): 'a': environ.get('HTTP_USER_AGENT', '-'), 'T': request_time.seconds, 'D': (request_time.seconds * 1000000) + request_time.microseconds, - 'M': (request_time.seconds * 1000) + int(request_time.microseconds/1000), + 'M': (request_time.seconds * 1000) + int(request_time.microseconds / 1000), 'L': "%d.%06d" % (request_time.seconds, request_time.microseconds), 'p': "<%s>" % os.getpid() } @@ -334,7 +348,7 @@ def access(self, resp, req, environ, request_time): """ if not (self.cfg.accesslog or self.cfg.logconfig or - self.cfg.logconfig_dict or + self.cfg.logconfig_dict or self.cfg.logconfig_json or (self.cfg.syslog and not self.cfg.disable_redirect_access_to_syslog)): return @@ -403,7 +417,7 @@ def _set_handler(self, log, output, fmt, stream=None): if output == "-": h = logging.StreamHandler(stream) else: - util.check_is_writeable(output) + util.check_is_writable(output) h = logging.FileHandler(output) # make sure the user can reopen the file try: @@ -437,7 +451,7 @@ def _set_syslog_handler(self, log, cfg, fmt, name): # finally setup the syslog handler h = logging.handlers.SysLogHandler(address=addr, - facility=facility, socktype=socktype) + facility=facility, socktype=socktype) h.setFormatter(fmt) h._gunicorn = True @@ -454,11 +468,7 @@ def _get_user(self, environ): # so we need to convert it to a byte string auth = base64.b64decode(auth[1].strip().encode('utf-8')) # b64decode returns a byte string - auth = auth.decode('utf-8') - auth = auth.split(":", 1) + user = auth.split(b":", 1)[0].decode("UTF-8") except (TypeError, binascii.Error, UnicodeDecodeError) as exc: self.debug("Couldn't get username: %s", exc) - return user - if len(auth) == 2: - user = auth[0] return user diff --git a/gunicorn/http/body.py b/gunicorn/http/body.py index afde36854..78f03214a 100644 --- a/gunicorn/http/body.py +++ b/gunicorn/http/body.py @@ -18,7 +18,7 @@ def __init__(self, req, unreader): def read(self, size): if not isinstance(size, int): - raise TypeError("size must be an integral type") + raise TypeError("size must be an integer type") if size < 0: raise ValueError("Size must be positive.") if size == 0: @@ -51,7 +51,7 @@ def parse_trailers(self, unreader, data): if done: unreader.unread(buf.getvalue()[2:]) return b"" - self.req.trailers = self.req.parse_headers(buf.getvalue()[:idx]) + self.req.trailers = self.req.parse_headers(buf.getvalue()[:idx], from_trailer=True) unreader.unread(buf.getvalue()[idx + 4:]) def parse_chunked(self, unreader): @@ -85,11 +85,13 @@ def parse_chunk_size(self, unreader, data=None): data = buf.getvalue() line, rest_chunk = data[:idx], data[idx + 2:] - chunk_size = line.split(b";", 1)[0].strip() - try: - chunk_size = int(chunk_size, 16) - except ValueError: + # RFC9112 7.1.1: BWS before chunk-ext - but ONLY then + chunk_size, *chunk_ext = line.split(b";", 1) + if chunk_ext: + chunk_size = chunk_size.rstrip(b" \t") + if any(n not in b"0123456789abcdefABCDEF" for n in chunk_size): raise InvalidChunkSize(chunk_size) + chunk_size = int(chunk_size, 16) if chunk_size == 0: try: diff --git a/gunicorn/http/errors.py b/gunicorn/http/errors.py index 7839ef05a..340f0473c 100644 --- a/gunicorn/http/errors.py +++ b/gunicorn/http/errors.py @@ -22,6 +22,15 @@ def __str__(self): return "No more data after: %r" % self.buf +class ConfigurationProblem(ParseException): + def __init__(self, info): + self.info = info + self.code = 500 + + def __str__(self): + return "Configuration problem: %s" % self.info + + class InvalidRequestLine(ParseException): def __init__(self, req): self.req = req @@ -64,6 +73,15 @@ def __str__(self): return "Invalid HTTP header name: %r" % self.hdr +class UnsupportedTransferCoding(ParseException): + def __init__(self, hdr): + self.hdr = hdr + self.code = 501 + + def __str__(self): + return "Unsupported transfer coding: %r" % self.hdr + + class InvalidChunkSize(IOError): def __init__(self, data): self.data = data diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 17d22402b..88ffa5a25 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -12,6 +12,7 @@ InvalidHeader, InvalidHeaderName, NoMoreData, InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, LimitRequestLine, LimitRequestHeaders, + UnsupportedTransferCoding, ) from gunicorn.http.errors import InvalidProxyLine, ForbiddenProxyRequest from gunicorn.http.errors import InvalidSchemeHeaders @@ -21,9 +22,12 @@ MAX_HEADERS = 32768 DEFAULT_MAX_HEADERFIELD_SIZE = 8190 -HEADER_RE = re.compile(r"[\x00-\x1F\x7F()<>@,;:\[\]={} \t\\\"]") -METH_RE = re.compile(r"[A-Z0-9$-_.]{3,20}") -VERSION_RE = re.compile(r"HTTP/(\d+)\.(\d+)") +# verbosely on purpose, avoid backslash ambiguity +RFC9110_5_6_2_TOKEN_SPECIALS = r"!#$%&'*+-.^_`|~" +TOKEN_RE = re.compile(r"[%s0-9a-zA-Z]+" % (re.escape(RFC9110_5_6_2_TOKEN_SPECIALS))) +METHOD_BADCHAR_RE = re.compile("[a-z#]") +# usually 1.0 or 1.1 - RFC9112 permits restricting to single-digit versions +VERSION_RE = re.compile(r"HTTP/(\d)\.(\d)") class Message(object): @@ -31,16 +35,18 @@ def __init__(self, cfg, unreader, peer_addr): self.cfg = cfg self.unreader = unreader self.peer_addr = peer_addr + self.remote_addr = peer_addr self.version = None self.headers = [] self.trailers = [] self.body = None self.scheme = "https" if cfg.is_ssl else "http" + self.must_close = False # set headers limits self.limit_request_fields = cfg.limit_request_fields if (self.limit_request_fields <= 0 - or self.limit_request_fields > MAX_HEADERS): + or self.limit_request_fields > MAX_HEADERS): self.limit_request_fields = MAX_HEADERS self.limit_request_field_size = cfg.limit_request_field_size if self.limit_request_field_size < 0: @@ -55,22 +61,29 @@ def __init__(self, cfg, unreader, peer_addr): self.unreader.unread(unused) self.set_body_reader() + def force_close(self): + self.must_close = True + def parse(self, unreader): raise NotImplementedError() - def parse_headers(self, data): + def parse_headers(self, data, from_trailer=False): cfg = self.cfg headers = [] - # Split lines on \r\n keeping the \r\n on each line - lines = [bytes_to_str(line) + "\r\n" for line in data.split(b"\r\n")] + # Split lines on \r\n + lines = [bytes_to_str(line) for line in data.split(b"\r\n")] # handle scheme headers scheme_header = False secure_scheme_headers = {} - if ('*' in cfg.forwarded_allow_ips or - not isinstance(self.peer_addr, tuple) - or self.peer_addr[0] in cfg.forwarded_allow_ips): + if from_trailer: + # nonsense. either a request is https from the beginning + # .. or we are just behind a proxy who does not remove conflicting trailers + pass + elif ('*' in cfg.forwarded_allow_ips or + not isinstance(self.peer_addr, tuple) + or self.peer_addr[0] in cfg.forwarded_allow_ips): secure_scheme_headers = cfg.secure_scheme_headers # Parse headers into key/value pairs paying attention @@ -79,30 +92,34 @@ def parse_headers(self, data): if len(headers) >= self.limit_request_fields: raise LimitRequestHeaders("limit request headers fields") - # Parse initial header name : value pair. + # Parse initial header name: value pair. curr = lines.pop(0) - header_length = len(curr) - if curr.find(":") < 0: - raise InvalidHeader(curr.strip()) + header_length = len(curr) + len("\r\n") + if curr.find(":") <= 0: + raise InvalidHeader(curr) name, value = curr.split(":", 1) if self.cfg.strip_header_spaces: - name = name.rstrip(" \t").upper() - else: - name = name.upper() - if HEADER_RE.search(name): + name = name.rstrip(" \t") + if not TOKEN_RE.fullmatch(name): raise InvalidHeaderName(name) - name, value = name.strip(), [value.lstrip()] + # this is still a dangerous place to do this + # but it is more correct than doing it before the pattern match: + # after we entered Unicode wonderland, 8bits could case-shift into ASCII: + # b"\xDF".decode("latin-1").upper().encode("ascii") == b"SS" + name = name.upper() + + value = [value.lstrip(" \t")] # Consume value continuation lines while lines and lines[0].startswith((" ", "\t")): curr = lines.pop(0) - header_length += len(curr) + header_length += len(curr) + len("\r\n") if header_length > self.limit_request_field_size > 0: raise LimitRequestHeaders("limit request headers " "fields size") - value.append(curr) - value = ''.join(value).rstrip() + value.append(curr.strip("\t ")) + value = " ".join(value) if header_length > self.limit_request_field_size > 0: raise LimitRequestHeaders("limit request headers fields size") @@ -117,6 +134,23 @@ def parse_headers(self, data): scheme_header = True self.scheme = scheme + # ambiguous mapping allows fooling downstream, e.g. merging non-identical headers: + # X-Forwarded-For: 2001:db8::ha:cc:ed + # X_Forwarded_For: 127.0.0.1,::1 + # HTTP_X_FORWARDED_FOR = 2001:db8::ha:cc:ed,127.0.0.1,::1 + # Only modify after fixing *ALL* header transformations; network to wsgi env + if "_" in name: + if self.cfg.header_map == "dangerous": + # as if we did not know we cannot safely map this + pass + elif self.cfg.header_map == "drop": + # almost as if it never had been there + # but still counts against resource limits + continue + else: + # fail-safe fallthrough: refuse + raise InvalidHeaderName(name) + headers.append((name, value)) return headers @@ -132,13 +166,54 @@ def set_body_reader(self): content_length = value elif name == "TRANSFER-ENCODING": if value.lower() == "chunked": + # DANGER: transfer codings stack, and stacked chunking is never intended + if chunked: + raise InvalidHeader("TRANSFER-ENCODING", req=self) chunked = True + elif value.lower() == "identity": + # does not do much, could still plausibly desync from what the proxy does + # safe option: nuke it, its never needed + if chunked: + raise InvalidHeader("TRANSFER-ENCODING", req=self) + elif value.lower() == "": + # lacking security review on this case + # offer the option to restore previous behaviour, but refuse by default, for now + self.force_close() + if not self.cfg.tolerate_dangerous_framing: + raise UnsupportedTransferCoding(value) + # DANGER: do not change lightly; ref: request smuggling + # T-E is a list and we *could* support correctly parsing its elements + # .. but that is only safe after getting all the edge cases right + # .. for which no real-world need exists, so best to NOT open that can of worms + else: + self.force_close() + # even if parser is extended, retain this branch: + # the "chunked not last" case remains to be rejected! + raise UnsupportedTransferCoding(value) if chunked: + # two potentially dangerous cases: + # a) CL + TE (TE overrides CL.. only safe if the recipient sees it that way too) + # b) chunked HTTP/1.0 (always faulty) + if self.version < (1, 1): + # framing wonky, see RFC 9112 Section 6.1 + self.force_close() + if not self.cfg.tolerate_dangerous_framing: + raise InvalidHeader("TRANSFER-ENCODING", req=self) + if content_length is not None: + # we cannot be certain the message framing we understood matches proxy intent + # -> whatever happens next, remaining input must not be trusted + self.force_close() + # either processing or rejecting is permitted in RFC 9112 Section 6.1 + if not self.cfg.tolerate_dangerous_framing: + raise InvalidHeader("CONTENT-LENGTH", req=self) self.body = Body(ChunkedReader(self, self.unreader)) elif content_length is not None: try: - content_length = int(content_length) + if str(content_length).isnumeric(): + content_length = int(content_length) + else: + raise InvalidHeader("CONTENT-LENGTH", req=self) except ValueError: raise InvalidHeader("CONTENT-LENGTH", req=self) @@ -150,9 +225,11 @@ def set_body_reader(self): self.body = Body(EOFReader(self.unreader)) def should_close(self): + if self.must_close: + return True for (h, v) in self.headers: if h == "CONNECTION": - v = v.lower().strip() + v = v.lower().strip(" \t") if v == "close": return True elif v == "keep-alive": @@ -172,7 +249,7 @@ def __init__(self, cfg, unreader, peer_addr, req_number=1): # get max request line size self.limit_request_line = cfg.limit_request_line if (self.limit_request_line < 0 - or self.limit_request_line >= MAX_REQUEST_LINE): + or self.limit_request_line >= MAX_REQUEST_LINE): self.limit_request_line = MAX_REQUEST_LINE self.req_number = req_number @@ -226,7 +303,7 @@ def parse(self, unreader): self.unreader.unread(data[2:]) return b"" - self.headers = self.parse_headers(data[:idx]) + self.headers = self.parse_headers(data[:idx], from_trailer=False) ret = data[idx + 4:] buf = None @@ -275,11 +352,11 @@ def proxy_protocol_access_check(self): # check in allow list if ("*" not in self.cfg.proxy_allow_ips and isinstance(self.peer_addr, tuple) and - self.peer_addr[0] not in self.cfg.proxy_allow_ips): + self.peer_addr[0] not in self.cfg.proxy_allow_ips): raise ForbiddenProxyRequest(self.peer_addr[0]) def parse_proxy_protocol(self, line): - bits = line.split() + bits = line.split(" ") if len(bits) != 6: raise InvalidProxyLine(line) @@ -324,14 +401,27 @@ def parse_proxy_protocol(self, line): } def parse_request_line(self, line_bytes): - bits = [bytes_to_str(bit) for bit in line_bytes.split(None, 2)] + bits = [bytes_to_str(bit) for bit in line_bytes.split(b" ", 2)] if len(bits) != 3: raise InvalidRequestLine(bytes_to_str(line_bytes)) - # Method - if not METH_RE.match(bits[0]): - raise InvalidRequestMethod(bits[0]) - self.method = bits[0].upper() + # Method: RFC9110 Section 9 + self.method = bits[0] + + # nonstandard restriction, suitable for all IANA registered methods + # partially enforced in previous gunicorn versions + if not self.cfg.permit_unconventional_http_method: + if METHOD_BADCHAR_RE.search(self.method): + raise InvalidRequestMethod(self.method) + if not 3 <= len(bits[0]) <= 20: + raise InvalidRequestMethod(self.method) + # standard restriction: RFC9110 token + if not TOKEN_RE.fullmatch(self.method): + raise InvalidRequestMethod(self.method) + # nonstandard and dangerous + # methods are merely uppercase by convention, no case-insensitive treatment is intended + if self.cfg.casefold_http_method: + self.method = self.method.upper() # URI self.uri = bits[1] @@ -345,10 +435,14 @@ def parse_request_line(self, line_bytes): self.fragment = parts.fragment or "" # Version - match = VERSION_RE.match(bits[2]) + match = VERSION_RE.fullmatch(bits[2]) if match is None: raise InvalidHTTPVersion(bits[2]) self.version = (int(match.group(1)), int(match.group(2))) + if not (1, 0) <= self.version < (2, 0): + # if ever relaxing this, carefully review Content-Encoding processing + if not self.cfg.permit_unconventional_http_version: + raise InvalidHTTPVersion(self.version) def set_body_reader(self): super().set_body_reader() diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 478677f4b..6f3d9b68f 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -9,16 +9,18 @@ import re import sys -from gunicorn.http.message import HEADER_RE -from gunicorn.http.errors import InvalidHeader, InvalidHeaderName +from gunicorn.http.message import TOKEN_RE +from gunicorn.http.errors import ConfigurationProblem, InvalidHeader, InvalidHeaderName from gunicorn import SERVER_SOFTWARE, SERVER -import gunicorn.util as util +from gunicorn import util # Send files in at most 1GB blocks as some operating systems can have problems # with sending files in blocks over 2GB. BLKSIZE = 0x3FFFFFFF -HEADER_VALUE_RE = re.compile(r'[\x00-\x1F\x7F]') +# RFC9110 5.5: field-vchar = VCHAR / obs-text +# RFC4234 B.1: VCHAR = 0x21-x07E = printable ASCII +HEADER_VALUE_RE = re.compile(r'[ \t\x21-\x7e\x80-\xff]*') log = logging.getLogger(__name__) @@ -133,6 +135,8 @@ def create(req, sock, client, server, cfg): environ['CONTENT_LENGTH'] = hdr_value continue + # do not change lightly, this is a common source of security problems + # RFC9110 Section 17.10 discourages ambiguous or incomplete mappings key = 'HTTP_' + hdr_name.replace('-', '_') if key in environ: hdr_value = "%s,%s" % (environ[key], hdr_value) @@ -180,7 +184,11 @@ def create(req, sock, client, server, cfg): # set the path and script name path_info = req.path if script_name: - path_info = path_info.split(script_name, 1)[1] + if not path_info.startswith(script_name): + raise ConfigurationProblem( + "Request path %r does not start with SCRIPT_NAME %r" % + (path_info, script_name)) + path_info = path_info[len(script_name):] environ['PATH_INFO'] = util.unquote_to_wsgi_str(path_info) environ['SCRIPT_NAME'] = script_name @@ -249,31 +257,32 @@ def process_headers(self, headers): if not isinstance(name, str): raise TypeError('%r is not a string' % name) - if HEADER_RE.search(name): + if not TOKEN_RE.fullmatch(name): raise InvalidHeaderName('%r' % name) if not isinstance(value, str): raise TypeError('%r is not a string' % value) - if HEADER_VALUE_RE.search(value): + if not HEADER_VALUE_RE.fullmatch(value): raise InvalidHeader('%r' % value) - value = value.strip() - lname = name.lower().strip() + # RFC9110 5.5 + value = value.strip(" \t") + lname = name.lower() if lname == "content-length": self.response_length = int(value) elif util.is_hoppish(name): if lname == "connection": # handle websocket - if value.lower().strip() == "upgrade": + if value.lower() == "upgrade": self.upgrade = True elif lname == "upgrade": - if value.lower().strip() == "websocket": - self.headers.append((name.strip(), value)) + if value.lower() == "websocket": + self.headers.append((name, value)) # ignore hopbyhop headers continue - self.headers.append((name.strip(), value)) + self.headers.append((name, value)) def is_chunked(self): # Only use chunked responses when the client is @@ -371,8 +380,8 @@ def sendfile(self, respiter): if self.is_chunked(): chunk_size = "%X\r\n" % nbytes self.sock.sendall(chunk_size.encode('utf-8')) - - self.sock.sendfile(respiter.filelike, count=nbytes) + if nbytes > 0: + self.sock.sendfile(respiter.filelike, offset=offset, count=nbytes) if self.is_chunked(): self.sock.sendall(b"\r\n") diff --git a/gunicorn/instrument/statsd.py b/gunicorn/instrument/statsd.py index afbfd7b48..2c54b2e72 100644 --- a/gunicorn/instrument/statsd.py +++ b/gunicorn/instrument/statsd.py @@ -24,14 +24,17 @@ class Statsd(Logger): """statsD-based instrumentation, that passes as a logger """ def __init__(self, cfg): - """host, port: statsD server - """ Logger.__init__(self, cfg) self.prefix = sub(r"^(.+[^.]+)\.*$", "\\g<1>.", cfg.statsd_prefix) + + if isinstance(cfg.statsd_host, str): + address_family = socket.AF_UNIX + else: + address_family = socket.AF_INET + try: - host, port = cfg.statsd_host - self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - self.sock.connect((host, int(port))) + self.sock = socket.socket(address_family, socket.SOCK_DGRAM) + self.sock.connect(cfg.statsd_host) except Exception: self.sock = None diff --git a/gunicorn/reloader.py b/gunicorn/reloader.py index c1964785a..88b540bdb 100644 --- a/gunicorn/reloader.py +++ b/gunicorn/reloader.py @@ -17,7 +17,7 @@ class Reloader(threading.Thread): def __init__(self, extra_files=None, interval=1, callback=None): super().__init__() - self.setDaemon(True) + self.daemon = True self._extra_files = set(extra_files or ()) self._interval = interval self._callback = callback @@ -74,7 +74,7 @@ class InotifyReloader(threading.Thread): def __init__(self, extra_files=None, callback=None): super().__init__() - self.setDaemon(True) + self.daemon = True self._callback = callback self._dirs = set() self._watcher = Inotify() @@ -118,7 +118,7 @@ def run(self): else: class InotifyReloader(object): - def __init__(self, callback=None): + def __init__(self, extra_files=None, callback=None): raise ImportError('You must have the inotify module installed to ' 'use the inotify reloader') diff --git a/gunicorn/sock.py b/gunicorn/sock.py index d45867709..7700146a8 100644 --- a/gunicorn/sock.py +++ b/gunicorn/sock.py @@ -6,6 +6,7 @@ import errno import os import socket +import ssl import stat import sys import time @@ -39,7 +40,7 @@ def __getattr__(self, name): def set_options(self, sock, bound=False): sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if (self.conf.reuse_port - and hasattr(socket, 'SO_REUSEPORT')): # pragma: no cover + and hasattr(socket, 'SO_REUSEPORT')): # pragma: no cover try: sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) except socket.error as err: @@ -210,3 +211,22 @@ def close_sockets(listeners, unlink=True): sock.close() if unlink and _sock_type(sock_name) is UnixSocket: os.unlink(sock_name) + + +def ssl_context(conf): + def default_ssl_context_factory(): + context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=conf.ca_certs) + context.load_cert_chain(certfile=conf.certfile, keyfile=conf.keyfile) + context.verify_mode = conf.cert_reqs + if conf.ciphers: + context.set_ciphers(conf.ciphers) + return context + + return conf.ssl_context(conf, default_ssl_context_factory) + + +def ssl_wrap_socket(sock, conf): + return ssl_context(conf).wrap_socket(sock, + server_side=True, + suppress_ragged_eofs=conf.suppress_ragged_eofs, + do_handshake_on_connect=conf.do_handshake_on_connect) diff --git a/gunicorn/systemd.py b/gunicorn/systemd.py index cea482204..5bc1a7449 100644 --- a/gunicorn/systemd.py +++ b/gunicorn/systemd.py @@ -58,7 +58,6 @@ def sd_notify(state, logger, unset_environment=False): child processes. """ - addr = os.environ.get('NOTIFY_SOCKET') if addr is None: # not run in a service, just a noop @@ -69,7 +68,7 @@ def sd_notify(state, logger, unset_environment=False): addr = '\0' + addr[1:] sock.connect(addr) sock.sendall(state.encode('utf-8')) - except: + except Exception: logger.debug("Exception while invoking sd_notify()", exc_info=True) finally: if unset_environment: diff --git a/gunicorn/util.py b/gunicorn/util.py index a821e3572..751deea71 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -22,7 +22,10 @@ import traceback import warnings -import pkg_resources +try: + import importlib.metadata as importlib_metadata +except (ModuleNotFoundError, ImportError): + import importlib_metadata from gunicorn.errors import AppImportError from gunicorn.workers import SUPPORTED_WORKERS @@ -54,6 +57,15 @@ def _setproctitle(title): pass +def load_entry_point(distribution, group, name): + dist_obj = importlib_metadata.distribution(distribution) + eps = [ep for ep in dist_obj.entry_points + if ep.group == group and ep.name == name] + if not eps: + raise ImportError("Entry point %r not found" % ((group, name),)) + return eps[0].load() + + def load_class(uri, default="gunicorn.workers.sync.SyncWorker", section="gunicorn.workers"): if inspect.isclass(uri): @@ -68,7 +80,7 @@ def load_class(uri, default="gunicorn.workers.sync.SyncWorker", name = default try: - return pkg_resources.load_entry_point(dist, section, name) + return load_entry_point(dist, section, name) except Exception: exc = traceback.format_exc() msg = "class uri %r invalid or not found: \n\n[%s]" @@ -85,7 +97,7 @@ def load_class(uri, default="gunicorn.workers.sync.SyncWorker", break try: - return pkg_resources.load_entry_point( + return load_entry_point( "gunicorn", section, uri ) except Exception: @@ -97,7 +109,7 @@ def load_class(uri, default="gunicorn.workers.sync.SyncWorker", try: mod = importlib.import_module('.'.join(components)) - except: + except Exception: exc = traceback.format_exc() msg = "class uri %r invalid or not found: \n\n[%s]" raise RuntimeError(msg % (uri, exc)) @@ -145,7 +157,7 @@ def set_owner_process(uid, gid, initgroups=False): elif gid != os.getgid(): os.setgid(gid) - if uid: + if uid and uid != os.getuid(): os.setuid(uid) @@ -460,7 +472,7 @@ def is_hoppish(header): def daemonize(enable_stdio_inheritance=False): """\ Standard daemonization of a process. - http://www.svbug.com/documentation/comp.unix.programmer-FAQ/faq_2.html#SEC16 + http://www.faqs.org/faqs/unix-faq/programmer/faq/ section 1.7 """ if 'GUNICORN_FD' not in os.environ: if os.fork(): @@ -486,7 +498,10 @@ def daemonize(enable_stdio_inheritance=False): closerange(0, 3) fd_null = os.open(REDIRECT_TO, os.O_RDWR) + # PEP 446, make fd for /dev/null inheritable + os.set_inheritable(fd_null, True) + # expect fd_null to be always 0 here, but in-case not ... if fd_null != 0: os.dup2(fd_null, 0) @@ -547,12 +562,12 @@ def seed(): random.seed('%s.%s' % (time.time(), os.getpid())) -def check_is_writeable(path): +def check_is_writable(path): try: - f = open(path, 'a') + with open(path, 'a') as f: + f.close() except IOError as e: raise RuntimeError("Error: '%s' isn't writable [%r]" % (path, e)) - f.close() def to_bytestring(value, encoding="utf8"): diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index a6d84bd23..f97d923c7 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -230,7 +230,9 @@ def handle_error(self, req, client, addr, exc): elif isinstance(exc, LimitRequestLine): mesg = "%s" % str(exc) elif isinstance(exc, LimitRequestHeaders): + reason = "Request Header Fields Too Large" mesg = "Error parsing headers: '%s'" % str(exc) + status_int = 431 elif isinstance(exc, InvalidProxyLine): mesg = "'%s'" % str(exc) elif isinstance(exc, ForbiddenProxyRequest): @@ -245,10 +247,12 @@ def handle_error(self, req, client, addr, exc): status_int = 403 msg = "Invalid request from ip={ip}: {error}" - self.log.debug(msg.format(ip=addr[0], error=str(exc))) + self.log.warning(msg.format(ip=addr[0], error=str(exc))) else: if hasattr(req, "uri"): self.log.exception("Error handling request %s", req.uri) + else: + self.log.exception("Error handling request (no URI read)") status_int = 500 reason = "Internal Server Error" mesg = "" diff --git a/gunicorn/workers/base_async.py b/gunicorn/workers/base_async.py index 73c3f6c1b..6a79d7ed0 100644 --- a/gunicorn/workers/base_async.py +++ b/gunicorn/workers/base_async.py @@ -9,10 +9,10 @@ import ssl import sys -import gunicorn.http as http -import gunicorn.http.wsgi as wsgi -import gunicorn.util as util -import gunicorn.workers.base as base +from gunicorn import http +from gunicorn.http import wsgi +from gunicorn import util +from gunicorn.workers import base ALREADY_HANDLED = object() @@ -82,7 +82,7 @@ def handle(self, listener, client, addr): self.log.debug("Ignoring socket not connected") else: self.log.debug("Ignoring EPIPE") - except Exception as e: + except BaseException as e: self.handle_error(req, client, addr, e) finally: util.close(client) @@ -115,9 +115,9 @@ def handle_request(self, listener_name, req, sock, addr): for item in respiter: resp.write(item) resp.close() + finally: request_time = datetime.now() - request_start self.log.access(resp, req, environ, request_time) - finally: if hasattr(respiter, "close"): respiter.close() if resp.should_close(): diff --git a/gunicorn/workers/geventlet.py b/gunicorn/workers/geventlet.py index ffdb206c0..c42ed1186 100644 --- a/gunicorn/workers/geventlet.py +++ b/gunicorn/workers/geventlet.py @@ -11,16 +11,22 @@ except ImportError: raise RuntimeError("eventlet worker requires eventlet 0.24.1 or higher") else: - from pkg_resources import parse_version + from packaging.version import parse as parse_version if parse_version(eventlet.__version__) < parse_version('0.24.1'): raise RuntimeError("eventlet worker requires eventlet 0.24.1 or higher") from eventlet import hubs, greenthread from eventlet.greenio import GreenSocket -from eventlet.wsgi import ALREADY_HANDLED as EVENTLET_ALREADY_HANDLED +import eventlet.wsgi import greenlet from gunicorn.workers.base_async import AsyncWorker +from gunicorn.sock import ssl_wrap_socket + +# ALREADY_HANDLED is removed in 0.30.3+ now it's `WSGI_LOCAL.already_handled: bool` +# https://github.com/eventlet/eventlet/pull/544 +EVENTLET_WSGI_LOCAL = getattr(eventlet.wsgi, "WSGI_LOCAL", None) +EVENTLET_ALREADY_HANDLED = getattr(eventlet.wsgi, "ALREADY_HANDLED", None) def _eventlet_socket_sendfile(self, file, offset=0, count=None): @@ -61,7 +67,6 @@ def _eventlet_socket_sendfile(self, file, offset=0, count=None): file.seek(offset + total_sent) - def _eventlet_serve(sock, handle, concurrency): """ Serve requests forever. @@ -125,6 +130,10 @@ def patch(self): patch_sendfile() def is_already_handled(self, respiter): + # eventlet >= 0.30.3 + if getattr(EVENTLET_WSGI_LOCAL, "already_handled", None): + raise StopIteration() + # eventlet < 0.30.3 if respiter == EVENTLET_ALREADY_HANDLED: raise StopIteration() return super().is_already_handled(respiter) @@ -144,9 +153,7 @@ def timeout_ctx(self): def handle(self, listener, client, addr): if self.cfg.is_ssl: - client = eventlet.wrap_ssl(client, server_side=True, - **self.cfg.ssl_options) - + client = ssl_wrap_socket(client, self.cfg) super().handle(listener, client, addr) def run(self): @@ -166,6 +173,7 @@ def run(self): eventlet.sleep(1.0) self.notify() + t = None try: with eventlet.Timeout(self.cfg.graceful_timeout) as t: for a in acceptors: diff --git a/gunicorn/workers/ggevent.py b/gunicorn/workers/ggevent.py index 3941814f5..2125a32d0 100644 --- a/gunicorn/workers/ggevent.py +++ b/gunicorn/workers/ggevent.py @@ -14,7 +14,7 @@ except ImportError: raise RuntimeError("gevent worker requires gevent 1.4 or higher") else: - from pkg_resources import parse_version + from packaging.version import parse as parse_version if parse_version(gevent.__version__) < parse_version('1.4'): raise RuntimeError("gevent worker requires gevent 1.4 or higher") @@ -24,6 +24,7 @@ import gunicorn from gunicorn.http.wsgi import base_environ +from gunicorn.sock import ssl_context from gunicorn.workers.base_async import AsyncWorker VERSION = "gevent/%s gunicorn/%s" % (gevent.__version__, gunicorn.__version__) @@ -41,7 +42,7 @@ def patch(self): sockets = [] for s in self.sockets: sockets.append(socket.socket(s.FAMILY, socket.SOCK_STREAM, - fileno=s.sock.fileno())) + fileno=s.sock.fileno())) self.sockets = sockets def notify(self): @@ -58,7 +59,7 @@ def run(self): ssl_args = {} if self.cfg.is_ssl: - ssl_args = dict(server_side=True, **self.cfg.ssl_options) + ssl_args = {"ssl_context": ssl_context(self.cfg)} for s in self.sockets: s.setblocking(1) @@ -110,7 +111,7 @@ def run(self): gevent.sleep(1.0) # Force kill all active the handlers - self.log.warning("Worker graceful timeout (pid:%s)" % self.pid) + self.log.warning("Worker graceful timeout (pid:%s)", self.pid) for server in servers: server.stop(timeout=1) except Exception: diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index d53181154..c9c42345f 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -11,7 +11,7 @@ # closed. # pylint: disable=no-else-break -import concurrent.futures as futures +from concurrent import futures import errno import os import selectors @@ -27,6 +27,7 @@ from . import base from .. import http from .. import util +from .. import sock from ..http import wsgi @@ -40,17 +41,19 @@ def __init__(self, cfg, sock, client, server): self.timeout = None self.parser = None + self.initialized = False # set the socket to non blocking self.sock.setblocking(False) def init(self): + self.initialized = True self.sock.setblocking(True) + if self.parser is None: # wrap the socket if needed if self.cfg.is_ssl: - self.sock = ssl.wrap_socket(self.sock, server_side=True, - **self.cfg.ssl_options) + self.sock = sock.ssl_wrap_socket(self.sock, self.cfg) # initialize the parser self.parser = http.RequestParser(self.cfg, self.sock, self.client) @@ -119,24 +122,29 @@ def accept(self, server, listener): sock, client = listener.accept() # initialize the connection object conn = TConn(self.cfg, sock, client, server) + self.nr_conns += 1 - # enqueue the job - self.enqueue_req(conn) + # wait until socket is readable + with self._lock: + self.poller.register(conn.sock, selectors.EVENT_READ, + partial(self.on_client_socket_readable, conn)) except EnvironmentError as e: if e.errno not in (errno.EAGAIN, errno.ECONNABORTED, errno.EWOULDBLOCK): raise - def reuse_connection(self, conn, client): + def on_client_socket_readable(self, conn, client): with self._lock: # unregister the client from the poller self.poller.unregister(client) - # remove the connection from keepalive - try: - self._keep.remove(conn) - except ValueError: - # race condition - return + + if conn.initialized: + # remove the connection from keepalive + try: + self._keep.remove(conn) + except ValueError: + # race condition + return # submit the connection to a worker self.enqueue_req(conn) @@ -169,6 +177,9 @@ def murder_keepalived(self): except KeyError: # already removed by the system, continue pass + except ValueError: + # already removed by the system continue + pass # close the socket conn.close() @@ -249,7 +260,7 @@ def finish_request(self, fs): # add the socket to the event loop self.poller.register(conn.sock, selectors.EVENT_READ, - partial(self.reuse_connection, conn)) + partial(self.on_client_socket_readable, conn)) else: self.nr_conns -= 1 conn.close() @@ -329,9 +340,9 @@ def handle_request(self, req, conn): resp.write(item) resp.close() + finally: request_time = datetime.now() - request_start self.log.access(resp, req, environ, request_time) - finally: if hasattr(respiter, "close"): respiter.close() diff --git a/gunicorn/workers/gtornado.py b/gunicorn/workers/gtornado.py index 9dd3d7bc4..285061196 100644 --- a/gunicorn/workers/gtornado.py +++ b/gunicorn/workers/gtornado.py @@ -3,7 +3,6 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -import copy import os import sys @@ -17,6 +16,7 @@ from tornado.wsgi import WSGIContainer from gunicorn.workers.base import Worker from gunicorn import __version__ as gversion +from gunicorn.sock import ssl_context # Tornado 5.0 updated its IOLoop, and the `io_loop` arguments to many @@ -108,9 +108,10 @@ def run(self): if tornado.version_info[0] < 6: if not isinstance(app, tornado.web.Application) or \ - isinstance(app, tornado.wsgi.WSGIApplication): + isinstance(app, tornado.wsgi.WSGIApplication): app = WSGIContainer(app) - elif not isinstance(app, WSGIContainer): + elif not isinstance(app, WSGIContainer) and \ + not isinstance(app, tornado.web.Application): app = WSGIContainer(app) # Monkey-patching HTTPConnection.finish to count the @@ -139,16 +140,11 @@ def on_close(instance, server_conn): server_class = _HTTPServer if self.cfg.is_ssl: - _ssl_opt = copy.deepcopy(self.cfg.ssl_options) - # tornado refuses initialization if ssl_options contains following - # options - del _ssl_opt["do_handshake_on_connect"] - del _ssl_opt["suppress_ragged_eofs"] if TORNADO5: - server = server_class(app, ssl_options=_ssl_opt) + server = server_class(app, ssl_options=ssl_context(self.cfg)) else: server = server_class(app, io_loop=self.ioloop, - ssl_options=_ssl_opt) + ssl_options=ssl_context(self.cfg)) else: if TORNADO5: server = server_class(app) diff --git a/gunicorn/workers/sync.py b/gunicorn/workers/sync.py index eeb7f633f..ddcd77270 100644 --- a/gunicorn/workers/sync.py +++ b/gunicorn/workers/sync.py @@ -12,10 +12,11 @@ import ssl import sys -import gunicorn.http as http -import gunicorn.http.wsgi as wsgi -import gunicorn.util as util -import gunicorn.workers.base as base +from gunicorn import http +from gunicorn.http import wsgi +from gunicorn import sock +from gunicorn import util +from gunicorn.workers import base class StopWaiting(Exception): @@ -128,9 +129,7 @@ def handle(self, listener, client, addr): req = None try: if self.cfg.is_ssl: - client = ssl.wrap_socket(client, server_side=True, - **self.cfg.ssl_options) - + client = sock.ssl_wrap_socket(client, self.cfg) parser = http.RequestParser(self.cfg, client, addr) req = next(parser) self.handle_request(listener, req, client, addr) @@ -155,7 +154,7 @@ def handle(self, listener, client, addr): self.log.debug("Ignoring socket not connected") else: self.log.debug("Ignoring EPIPE") - except Exception as e: + except BaseException as e: self.handle_error(req, client, addr, e) finally: util.close(client) @@ -184,9 +183,9 @@ def handle_request(self, listener, req, client, addr): for item in respiter: resp.write(item) resp.close() + finally: request_time = datetime.now() - request_start self.log.access(resp, req, environ, request_time) - finally: if hasattr(respiter, "close"): respiter.close() except EnvironmentError: diff --git a/gunicorn/workers/workertmp.py b/gunicorn/workers/workertmp.py index 65bbe54fa..a9ae39de0 100644 --- a/gunicorn/workers/workertmp.py +++ b/gunicorn/workers/workertmp.py @@ -4,6 +4,7 @@ # See the NOTICE for more information. import os +import time import platform import tempfile @@ -28,7 +29,7 @@ def __init__(self, cfg): if cfg.uid != os.geteuid() or cfg.gid != os.getegid(): util.chown(name, cfg.uid, cfg.gid) - # unlink the file so we don't leak tempory files + # unlink the file so we don't leak temporary files try: if not IS_CYGWIN: util.unlink(name) @@ -39,14 +40,12 @@ def __init__(self, cfg): os.close(fd) raise - self.spinner = 0 - def notify(self): - self.spinner = (self.spinner + 1) % 2 - os.fchmod(self._tmp.fileno(), self.spinner) + new_time = time.monotonic() + os.utime(self._tmp.fileno(), (new_time, new_time)) def last_update(self): - return os.fstat(self._tmp.fileno()).st_ctime + return os.fstat(self._tmp.fileno()).st_mtime def fileno(self): return self._tmp.fileno() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..eaca1eac0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,89 @@ +[build-system] +requires = ["setuptools>=61.2"] +build-backend = "setuptools.build_meta" + +[project] +# see https://packaging.python.org/en/latest/specifications/pyproject-toml/ +name = "gunicorn" +authors = [{name = "Benoit Chesneau", email = "benoitc@gunicorn.org"}] +license = {text = "MIT"} +description = "WSGI HTTP Server for UNIX" +readme = "README.rst" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Other Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Internet", + "Topic :: Utilities", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Internet :: WWW/HTTP :: WSGI", + "Topic :: Internet :: WWW/HTTP :: WSGI :: Server", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content", +] +requires-python = ">=3.7" +dependencies = [ + 'importlib_metadata; python_version<"3.8"', + "packaging", +] +dynamic = ["version"] + +[project.urls] +Homepage = "https://gunicorn.org" +Documentation = "https://docs.gunicorn.org" +"Issue tracker" = "https://github.com/benoitc/gunicorn/issues" +"Source code" = "https://github.com/benoitc/gunicorn" +Changelog = "https://docs.gunicorn.org/en/stable/news.html" + +[project.optional-dependencies] +gevent = ["gevent>=1.4.0"] +eventlet = ["eventlet>=0.24.1,!=0.36.0"] +tornado = ["tornado>=0.2"] +gthread = [] +setproctitle = ["setproctitle"] +testing = [ + "gevent", + "eventlet", + "coverage", + "pytest", + "pytest-cov", +] + +[project.scripts] +# duplicates "python -m gunicorn" handling in __main__.py +gunicorn = "gunicorn.app.wsgiapp:run" + +# note the quotes around "paste.server_runner" to escape the dot +[project.entry-points."paste.server_runner"] +main = "gunicorn.app.pasterapp:serve" + +[tool.pytest.ini_options] +# # can override these: python -m pytest --override-ini="addopts=" +norecursedirs = ["examples", "lib", "local", "src"] +testpaths = ["tests/"] +addopts = "--assert=plain --cov=gunicorn --cov-report=xml" + +[tool.setuptools] +zip-safe = false +include-package-data = true +license-files = ["LICENSE"] + +[tool.setuptools.packages] +find = {namespaces = false} + +[tool.setuptools.dynamic] +version = {attr = "gunicorn.__version__"} diff --git a/requirements_dev.txt b/requirements_dev.txt index a36971d58..1d8c01291 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,9 @@ -r requirements_test.txt +# setuptools v68.0 fails hard on invalid pyproject.toml +# which a developer would want to know +# otherwise, oldest known-working version is 61.2 +setuptools>=68.0 + sphinx sphinx_rtd_theme diff --git a/requirements_test.txt b/requirements_test.txt index 03af49696..b618d1a73 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,5 @@ -aiohttp gevent eventlet coverage -pytest +pytest>=7.2.0 pytest-cov diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b880b5d9c..000000000 --- a/setup.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[tool:pytest] -norecursedirs = examples lib local src -testpaths = tests/ -addopts = --assert=plain --cov=gunicorn --cov-report=xml - -[metadata] -license_file = LICENSE diff --git a/setup.py b/setup.py deleted file mode 100644 index fb220d90e..000000000 --- a/setup.py +++ /dev/null @@ -1,122 +0,0 @@ -# -*- coding: utf-8 - -# -# This file is part of gunicorn released under the MIT license. -# See the NOTICE for more information. - -import os -import sys - -from setuptools import setup, find_packages -from setuptools.command.test import test as TestCommand - -from gunicorn import __version__ - - -CLASSIFIERS = [ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Other Environment', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: POSIX', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Topic :: Internet', - 'Topic :: Utilities', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Internet :: WWW/HTTP :: WSGI', - 'Topic :: Internet :: WWW/HTTP :: WSGI :: Server', - 'Topic :: Internet :: WWW/HTTP :: Dynamic Content'] - -# read long description -with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as f: - long_description = f.read() - -# read dev requirements -fname = os.path.join(os.path.dirname(__file__), 'requirements_test.txt') -with open(fname) as f: - tests_require = [l.strip() for l in f.readlines()] - -class PyTestCommand(TestCommand): - user_options = [ - ("cov", None, "measure coverage") - ] - - def initialize_options(self): - TestCommand.initialize_options(self) - self.cov = None - - def finalize_options(self): - TestCommand.finalize_options(self) - self.test_args = ['tests'] - if self.cov: - self.test_args += ['--cov', 'gunicorn'] - self.test_suite = True - - def run_tests(self): - import pytest - errno = pytest.main(self.test_args) - sys.exit(errno) - - -install_requires = [ - # We depend on functioning pkg_resources.working_set.add_entry() and - # pkg_resources.load_entry_point(). These both work as of 3.0 which - # is the first version to support Python 3.4 which we require as a - # floor. - 'setuptools>=3.0', -] - -extras_require = { - 'gevent': ['gevent>=1.4.0'], - 'eventlet': ['eventlet>=0.24.1'], - 'tornado': ['tornado>=0.2'], - 'gthread': [], - 'setproctitle': ['setproctitle'], -} - -setup( - name='gunicorn', - version=__version__, - - description='WSGI HTTP Server for UNIX', - long_description=long_description, - author='Benoit Chesneau', - author_email='benoitc@e-engura.com', - license='MIT', - url='https://gunicorn.org', - project_urls={ - 'Documentation': 'https://docs.gunicorn.org', - 'Homepage': 'https://gunicorn.org', - 'Issue tracker': 'https://github.com/benoitc/gunicorn/issues', - 'Source code': 'https://github.com/benoitc/gunicorn', - }, - - python_requires='>=3.5', - install_requires=install_requires, - classifiers=CLASSIFIERS, - zip_safe=False, - packages=find_packages(exclude=['examples', 'tests']), - include_package_data=True, - - tests_require=tests_require, - cmdclass={'test': PyTestCommand}, - - entry_points=""" - [console_scripts] - gunicorn=gunicorn.app.wsgiapp:run - - [paste.server_runner] - main=gunicorn.app.pasterapp:serve - """, - extras_require=extras_require, -) diff --git a/tests/requests/invalid/003.http b/tests/requests/invalid/003.http index cd1ab7fcb..5a9eaafcd 100644 --- a/tests/requests/invalid/003.http +++ b/tests/requests/invalid/003.http @@ -1,2 +1,2 @@ --blargh /foo HTTP/1.1\r\n -\r\n \ No newline at end of file +GET\n/\nHTTP/1.1\r\n +\r\n diff --git a/tests/requests/invalid/003.py b/tests/requests/invalid/003.py index 86a0774e5..5a4ca8961 100644 --- a/tests/requests/invalid/003.py +++ b/tests/requests/invalid/003.py @@ -1,2 +1,2 @@ -from gunicorn.http.errors import InvalidRequestMethod -request = InvalidRequestMethod \ No newline at end of file +from gunicorn.http.errors import InvalidRequestLine +request = InvalidRequestLine diff --git a/tests/requests/invalid/003b.http b/tests/requests/invalid/003b.http new file mode 100644 index 000000000..e05fd9897 --- /dev/null +++ b/tests/requests/invalid/003b.http @@ -0,0 +1,2 @@ +bla:rgh /foo HTTP/1.1\r\n +\r\n diff --git a/tests/requests/invalid/003b.py b/tests/requests/invalid/003b.py new file mode 100644 index 000000000..86a0774e5 --- /dev/null +++ b/tests/requests/invalid/003b.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidRequestMethod +request = InvalidRequestMethod \ No newline at end of file diff --git a/tests/requests/invalid/003c.http b/tests/requests/invalid/003c.http new file mode 100644 index 000000000..98bd20bf1 --- /dev/null +++ b/tests/requests/invalid/003c.http @@ -0,0 +1,2 @@ +-bl /foo HTTP/1.1\r\n +\r\n diff --git a/tests/requests/invalid/003c.py b/tests/requests/invalid/003c.py new file mode 100644 index 000000000..1dac27c04 --- /dev/null +++ b/tests/requests/invalid/003c.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidRequestMethod +request = InvalidRequestMethod diff --git a/tests/requests/invalid/022.http b/tests/requests/invalid/022.http new file mode 100644 index 000000000..521c7a06e --- /dev/null +++ b/tests/requests/invalid/022.http @@ -0,0 +1,3 @@ +GET /first HTTP/1.0\r\n +Content-Length: -0\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/invalid/022.py b/tests/requests/invalid/022.py new file mode 100644 index 000000000..95b0581ae --- /dev/null +++ b/tests/requests/invalid/022.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeader + +cfg = Config() +request = InvalidHeader diff --git a/tests/requests/invalid/023.http b/tests/requests/invalid/023.http new file mode 100644 index 000000000..c672f7896 --- /dev/null +++ b/tests/requests/invalid/023.http @@ -0,0 +1,3 @@ +GET /first HTTP/1.0\r\n +Content-Length: 0_1\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/invalid/023.py b/tests/requests/invalid/023.py new file mode 100644 index 000000000..95b0581ae --- /dev/null +++ b/tests/requests/invalid/023.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeader + +cfg = Config() +request = InvalidHeader diff --git a/tests/requests/invalid/024.http b/tests/requests/invalid/024.http new file mode 100644 index 000000000..31c062fa9 --- /dev/null +++ b/tests/requests/invalid/024.http @@ -0,0 +1,3 @@ +GET /first HTTP/1.0\r\n +Content-Length: +1\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/invalid/024.py b/tests/requests/invalid/024.py new file mode 100644 index 000000000..95b0581ae --- /dev/null +++ b/tests/requests/invalid/024.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeader + +cfg = Config() +request = InvalidHeader diff --git a/tests/requests/invalid/040.http b/tests/requests/invalid/040.http new file mode 100644 index 000000000..be03929d5 --- /dev/null +++ b/tests/requests/invalid/040.http @@ -0,0 +1,6 @@ +GET /keep/same/as?invalid/040 HTTP/1.0\r\n +Transfer_Encoding: tricked\r\n +Content-Length: 7\r\n +Content_Length: -1E23\r\n +\r\n +tricked\r\n diff --git a/tests/requests/invalid/040.py b/tests/requests/invalid/040.py new file mode 100644 index 000000000..643289fab --- /dev/null +++ b/tests/requests/invalid/040.py @@ -0,0 +1,7 @@ +from gunicorn.http.errors import InvalidHeaderName +from gunicorn.config import Config + +cfg = Config() +cfg.set("header_map", "refuse") + +request = InvalidHeaderName diff --git a/tests/requests/invalid/chunked_01.http b/tests/requests/invalid/chunked_01.http new file mode 100644 index 000000000..7a8e55d2e --- /dev/null +++ b/tests/requests/invalid/chunked_01.http @@ -0,0 +1,12 @@ +POST /chunked_w_underscore_chunk_size HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\r\n +hello\r\n +6_0\r\n + world\r\n +0\r\n +\r\n +POST /after HTTP/1.1\r\n +Transfer-Encoding: identity\r\n +\r\n diff --git a/tests/requests/invalid/chunked_01.py b/tests/requests/invalid/chunked_01.py new file mode 100644 index 000000000..0571e1183 --- /dev/null +++ b/tests/requests/invalid/chunked_01.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidChunkSize +request = InvalidChunkSize diff --git a/tests/requests/invalid/chunked_02.http b/tests/requests/invalid/chunked_02.http new file mode 100644 index 000000000..9ae49e525 --- /dev/null +++ b/tests/requests/invalid/chunked_02.http @@ -0,0 +1,9 @@ +POST /chunked_with_prefixed_value HTTP/1.1\r\n +Content-Length: 12\r\n +Transfer-Encoding: \tchunked\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +\r\n diff --git a/tests/requests/invalid/chunked_02.py b/tests/requests/invalid/chunked_02.py new file mode 100644 index 000000000..1541eb70b --- /dev/null +++ b/tests/requests/invalid/chunked_02.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidHeader +request = InvalidHeader diff --git a/tests/requests/invalid/chunked_03.http b/tests/requests/invalid/chunked_03.http new file mode 100644 index 000000000..0bbbfe6ef --- /dev/null +++ b/tests/requests/invalid/chunked_03.http @@ -0,0 +1,8 @@ +POST /double_chunked HTTP/1.1\r\n +Transfer-Encoding: identity, chunked, identity, chunked\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +\r\n diff --git a/tests/requests/invalid/chunked_03.py b/tests/requests/invalid/chunked_03.py new file mode 100644 index 000000000..58a34600c --- /dev/null +++ b/tests/requests/invalid/chunked_03.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import UnsupportedTransferCoding +request = UnsupportedTransferCoding diff --git a/tests/requests/invalid/chunked_04.http b/tests/requests/invalid/chunked_04.http new file mode 100644 index 000000000..d47109e31 --- /dev/null +++ b/tests/requests/invalid/chunked_04.http @@ -0,0 +1,11 @@ +POST /chunked_twice HTTP/1.1\r\n +Transfer-Encoding: identity\r\n +Transfer-Encoding: chunked\r\n +Transfer-Encoding: identity\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +\r\n diff --git a/tests/requests/invalid/chunked_04.py b/tests/requests/invalid/chunked_04.py new file mode 100644 index 000000000..1541eb70b --- /dev/null +++ b/tests/requests/invalid/chunked_04.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidHeader +request = InvalidHeader diff --git a/tests/requests/invalid/chunked_05.http b/tests/requests/invalid/chunked_05.http new file mode 100644 index 000000000..014e85ac0 --- /dev/null +++ b/tests/requests/invalid/chunked_05.http @@ -0,0 +1,11 @@ +POST /chunked_HTTP_1.0 HTTP/1.0\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +0\r\n +Vary: *\r\n +Content-Type: text/plain\r\n +\r\n diff --git a/tests/requests/invalid/chunked_05.py b/tests/requests/invalid/chunked_05.py new file mode 100644 index 000000000..1541eb70b --- /dev/null +++ b/tests/requests/invalid/chunked_05.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidHeader +request = InvalidHeader diff --git a/tests/requests/invalid/chunked_06.http b/tests/requests/invalid/chunked_06.http new file mode 100644 index 000000000..ef70faab9 --- /dev/null +++ b/tests/requests/invalid/chunked_06.http @@ -0,0 +1,9 @@ +POST /chunked_not_last HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +Transfer-Encoding: gzip\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +\r\n diff --git a/tests/requests/invalid/chunked_06.py b/tests/requests/invalid/chunked_06.py new file mode 100644 index 000000000..58a34600c --- /dev/null +++ b/tests/requests/invalid/chunked_06.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import UnsupportedTransferCoding +request = UnsupportedTransferCoding diff --git a/tests/requests/invalid/chunked_07.http b/tests/requests/invalid/chunked_07.http new file mode 100644 index 000000000..a78db48b7 --- /dev/null +++ b/tests/requests/invalid/chunked_07.http @@ -0,0 +1,10 @@ +POST /chunked_ambiguous_header_mapping HTTP/1.1\r\n +Transfer_Encoding: gzip\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +0\r\n +\r\n diff --git a/tests/requests/invalid/chunked_07.py b/tests/requests/invalid/chunked_07.py new file mode 100644 index 000000000..643289fab --- /dev/null +++ b/tests/requests/invalid/chunked_07.py @@ -0,0 +1,7 @@ +from gunicorn.http.errors import InvalidHeaderName +from gunicorn.config import Config + +cfg = Config() +cfg.set("header_map", "refuse") + +request = InvalidHeaderName diff --git a/tests/requests/invalid/chunked_08.http b/tests/requests/invalid/chunked_08.http new file mode 100644 index 000000000..8d4aaa6e9 --- /dev/null +++ b/tests/requests/invalid/chunked_08.http @@ -0,0 +1,9 @@ +POST /chunked_not_last HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +Transfer-Encoding: identity\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +\r\n diff --git a/tests/requests/invalid/chunked_08.py b/tests/requests/invalid/chunked_08.py new file mode 100644 index 000000000..1541eb70b --- /dev/null +++ b/tests/requests/invalid/chunked_08.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidHeader +request = InvalidHeader diff --git a/tests/requests/invalid/chunked_09.http b/tests/requests/invalid/chunked_09.http new file mode 100644 index 000000000..6207310cf --- /dev/null +++ b/tests/requests/invalid/chunked_09.http @@ -0,0 +1,7 @@ +POST /chunked_ows_without_ext HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\r\n +hello\r\n +0 \r\n +\r\n diff --git a/tests/requests/invalid/chunked_09.py b/tests/requests/invalid/chunked_09.py new file mode 100644 index 000000000..0571e1183 --- /dev/null +++ b/tests/requests/invalid/chunked_09.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidChunkSize +request = InvalidChunkSize diff --git a/tests/requests/invalid/chunked_10.http b/tests/requests/invalid/chunked_10.http new file mode 100644 index 000000000..db1e4c38d --- /dev/null +++ b/tests/requests/invalid/chunked_10.http @@ -0,0 +1,7 @@ +POST /chunked_ows_before HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\r\n +hello\r\n + 0\r\n +\r\n diff --git a/tests/requests/invalid/chunked_10.py b/tests/requests/invalid/chunked_10.py new file mode 100644 index 000000000..0571e1183 --- /dev/null +++ b/tests/requests/invalid/chunked_10.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidChunkSize +request = InvalidChunkSize diff --git a/tests/requests/invalid/chunked_11.http b/tests/requests/invalid/chunked_11.http new file mode 100644 index 000000000..da1f72e58 --- /dev/null +++ b/tests/requests/invalid/chunked_11.http @@ -0,0 +1,7 @@ +POST /chunked_ows_before HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\n;\r\n +hello\r\n +0\r\n +\r\n diff --git a/tests/requests/invalid/chunked_11.py b/tests/requests/invalid/chunked_11.py new file mode 100644 index 000000000..0571e1183 --- /dev/null +++ b/tests/requests/invalid/chunked_11.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidChunkSize +request = InvalidChunkSize diff --git a/tests/requests/invalid/nonascii_01.http b/tests/requests/invalid/nonascii_01.http new file mode 100644 index 000000000..30d18cd6f --- /dev/null +++ b/tests/requests/invalid/nonascii_01.http @@ -0,0 +1,4 @@ +GETß /germans.. HTTP/1.1\r\n +Content-Length: 3\r\n +\r\n +ÄÄÄ diff --git a/tests/requests/invalid/nonascii_01.py b/tests/requests/invalid/nonascii_01.py new file mode 100644 index 000000000..0da10f426 --- /dev/null +++ b/tests/requests/invalid/nonascii_01.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidRequestMethod + +cfg = Config() +request = InvalidRequestMethod diff --git a/tests/requests/invalid/nonascii_02.http b/tests/requests/invalid/nonascii_02.http new file mode 100644 index 000000000..36a617039 --- /dev/null +++ b/tests/requests/invalid/nonascii_02.http @@ -0,0 +1,4 @@ +GETÿ /french.. HTTP/1.1\r\n +Content-Length: 3\r\n +\r\n +ÄÄÄ diff --git a/tests/requests/invalid/nonascii_02.py b/tests/requests/invalid/nonascii_02.py new file mode 100644 index 000000000..0da10f426 --- /dev/null +++ b/tests/requests/invalid/nonascii_02.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidRequestMethod + +cfg = Config() +request = InvalidRequestMethod diff --git a/tests/requests/invalid/nonascii_03.http b/tests/requests/invalid/nonascii_03.http new file mode 100644 index 000000000..eda5d010f --- /dev/null +++ b/tests/requests/invalid/nonascii_03.http @@ -0,0 +1,5 @@ +GET /germans.. HTTP/1.1\r\n +Content-Lengthß: 3\r\n +Content-Length: 3\r\n +\r\n +ÄÄÄ diff --git a/tests/requests/invalid/nonascii_03.py b/tests/requests/invalid/nonascii_03.py new file mode 100644 index 000000000..d336fbc85 --- /dev/null +++ b/tests/requests/invalid/nonascii_03.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeaderName + +cfg = Config() +request = InvalidHeaderName diff --git a/tests/requests/invalid/nonascii_04.http b/tests/requests/invalid/nonascii_04.http new file mode 100644 index 000000000..be0b15661 --- /dev/null +++ b/tests/requests/invalid/nonascii_04.http @@ -0,0 +1,5 @@ +GET /french.. HTTP/1.1\r\n +Content-Lengthÿ: 3\r\n +Content-Length: 3\r\n +\r\n +ÄÄÄ diff --git a/tests/requests/invalid/nonascii_04.py b/tests/requests/invalid/nonascii_04.py new file mode 100644 index 000000000..d336fbc85 --- /dev/null +++ b/tests/requests/invalid/nonascii_04.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeaderName + +cfg = Config() +request = InvalidHeaderName diff --git a/tests/requests/invalid/prefix_01.http b/tests/requests/invalid/prefix_01.http new file mode 100644 index 000000000..f8bdeb352 --- /dev/null +++ b/tests/requests/invalid/prefix_01.http @@ -0,0 +1,2 @@ +GET\0PROXY /foo HTTP/1.1\r\n +\r\n diff --git a/tests/requests/invalid/prefix_01.py b/tests/requests/invalid/prefix_01.py new file mode 100644 index 000000000..86a0774e5 --- /dev/null +++ b/tests/requests/invalid/prefix_01.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidRequestMethod +request = InvalidRequestMethod \ No newline at end of file diff --git a/tests/requests/invalid/prefix_02.http b/tests/requests/invalid/prefix_02.http new file mode 100644 index 000000000..8a9b155c4 --- /dev/null +++ b/tests/requests/invalid/prefix_02.http @@ -0,0 +1,2 @@ +GET\0 /foo HTTP/1.1\r\n +\r\n diff --git a/tests/requests/invalid/prefix_02.py b/tests/requests/invalid/prefix_02.py new file mode 100644 index 000000000..86a0774e5 --- /dev/null +++ b/tests/requests/invalid/prefix_02.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidRequestMethod +request = InvalidRequestMethod \ No newline at end of file diff --git a/tests/requests/invalid/prefix_03.http b/tests/requests/invalid/prefix_03.http new file mode 100644 index 000000000..7803935cc --- /dev/null +++ b/tests/requests/invalid/prefix_03.http @@ -0,0 +1,4 @@ +GET /stuff/here?foo=bar HTTP/1.1\r\n +Content-Length: 0 1\r\n +\r\n +x diff --git a/tests/requests/invalid/prefix_03.py b/tests/requests/invalid/prefix_03.py new file mode 100644 index 000000000..95b0581ae --- /dev/null +++ b/tests/requests/invalid/prefix_03.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeader + +cfg = Config() +request = InvalidHeader diff --git a/tests/requests/invalid/prefix_04.http b/tests/requests/invalid/prefix_04.http new file mode 100644 index 000000000..712631c8f --- /dev/null +++ b/tests/requests/invalid/prefix_04.http @@ -0,0 +1,5 @@ +GET /stuff/here?foo=bar HTTP/1.1\r\n +Content-Length: 3 1\r\n +\r\n +xyz +abc123 diff --git a/tests/requests/invalid/prefix_04.py b/tests/requests/invalid/prefix_04.py new file mode 100644 index 000000000..95b0581ae --- /dev/null +++ b/tests/requests/invalid/prefix_04.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHeader + +cfg = Config() +request = InvalidHeader diff --git a/tests/requests/invalid/prefix_05.http b/tests/requests/invalid/prefix_05.http new file mode 100644 index 000000000..120b6577d --- /dev/null +++ b/tests/requests/invalid/prefix_05.http @@ -0,0 +1,4 @@ +GET: /stuff/here?foo=bar HTTP/1.1\r\n +Content-Length: 3\r\n +\r\n +xyz diff --git a/tests/requests/invalid/prefix_05.py b/tests/requests/invalid/prefix_05.py new file mode 100644 index 000000000..0da10f426 --- /dev/null +++ b/tests/requests/invalid/prefix_05.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidRequestMethod + +cfg = Config() +request = InvalidRequestMethod diff --git a/tests/requests/invalid/prefix_06.http b/tests/requests/invalid/prefix_06.http new file mode 100644 index 000000000..536c586a3 --- /dev/null +++ b/tests/requests/invalid/prefix_06.http @@ -0,0 +1,4 @@ +GET /the/future HTTP/1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111.1\r\n +Content-Length: 7\r\n +\r\n +Old Man diff --git a/tests/requests/invalid/prefix_06.py b/tests/requests/invalid/prefix_06.py new file mode 100644 index 000000000..b2286d428 --- /dev/null +++ b/tests/requests/invalid/prefix_06.py @@ -0,0 +1,5 @@ +from gunicorn.config import Config +from gunicorn.http.errors import InvalidHTTPVersion + +cfg = Config() +request = InvalidHTTPVersion diff --git a/tests/requests/invalid/version_01.http b/tests/requests/invalid/version_01.http new file mode 100644 index 000000000..0a222ce12 --- /dev/null +++ b/tests/requests/invalid/version_01.http @@ -0,0 +1,2 @@ +GET /foo HTTP/0.99\r\n +\r\n diff --git a/tests/requests/invalid/version_01.py b/tests/requests/invalid/version_01.py new file mode 100644 index 000000000..760840b69 --- /dev/null +++ b/tests/requests/invalid/version_01.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidHTTPVersion +request = InvalidHTTPVersion diff --git a/tests/requests/invalid/version_02.http b/tests/requests/invalid/version_02.http new file mode 100644 index 000000000..6d29ac5fe --- /dev/null +++ b/tests/requests/invalid/version_02.http @@ -0,0 +1,2 @@ +GET /foo HTTP/2.0\r\n +\r\n diff --git a/tests/requests/invalid/version_02.py b/tests/requests/invalid/version_02.py new file mode 100644 index 000000000..760840b69 --- /dev/null +++ b/tests/requests/invalid/version_02.py @@ -0,0 +1,2 @@ +from gunicorn.http.errors import InvalidHTTPVersion +request = InvalidHTTPVersion diff --git a/tests/requests/valid/016.py b/tests/requests/valid/016.py index 139b27009..4e5144f8c 100644 --- a/tests/requests/valid/016.py +++ b/tests/requests/valid/016.py @@ -1,35 +1,35 @@ -certificate = """-----BEGIN CERTIFICATE-----\r\n - MIIFbTCCBFWgAwIBAgICH4cwDQYJKoZIhvcNAQEFBQAwcDELMAkGA1UEBhMCVUsx\r\n - ETAPBgNVBAoTCGVTY2llbmNlMRIwEAYDVQQLEwlBdXRob3JpdHkxCzAJBgNVBAMT\r\n - AkNBMS0wKwYJKoZIhvcNAQkBFh5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMu\r\n - dWswHhcNMDYwNzI3MTQxMzI4WhcNMDcwNzI3MTQxMzI4WjBbMQswCQYDVQQGEwJV\r\n - SzERMA8GA1UEChMIZVNjaWVuY2UxEzARBgNVBAsTCk1hbmNoZXN0ZXIxCzAJBgNV\r\n - BAcTmrsogriqMWLAk1DMRcwFQYDVQQDEw5taWNoYWVsIHBhcmQYJKoZIhvcNAQEB\r\n - BQADggEPADCCAQoCggEBANPEQBgl1IaKdSS1TbhF3hEXSl72G9J+WC/1R64fAcEF\r\n - W51rEyFYiIeZGx/BVzwXbeBoNUK41OK65sxGuflMo5gLflbwJtHBRIEKAfVVp3YR\r\n - gW7cMA/s/XKgL1GEC7rQw8lIZT8RApukCGqOVHSi/F1SiFlPDxuDfmdiNzL31+sL\r\n - 0iwHDdNkGjy5pyBSB8Y79dsSJtCW/iaLB0/n8Sj7HgvvZJ7x0fr+RQjYOUUfrePP\r\n - u2MSpFyf+9BbC/aXgaZuiCvSR+8Snv3xApQY+fULK/xY8h8Ua51iXoQ5jrgu2SqR\r\n - wgA7BUi3G8LFzMBl8FRCDYGUDy7M6QaHXx1ZWIPWNKsCAwEAAaOCAiQwggIgMAwG\r\n - 1UdEwEB/wQCMAAwEQYJYIZIAYb4QgHTTPAQDAgWgMA4GA1UdDwEB/wQEAwID6DAs\r\n - BglghkgBhvhCAQ0EHxYdVUsgZS1TY2llbmNlIFVzZXIgQ2VydGlmaWNhdGUwHQYD\r\n - VR0OBBYEFDTt/sf9PeMaZDHkUIldrDYMNTBZMIGaBgNVHSMEgZIwgY+AFAI4qxGj\r\n - loCLDdMVKwiljjDastqooXSkcjBwMQswCQYDVQQGEwJVSzERMA8GA1UEChMIZVNj\r\n - aWVuY2UxEjAQBgNVBAsTCUF1dGhvcml0eTELMAkGA1UEAxMCQ0ExLTArBgkqhkiG\r\n - 9w0BCQEWHmNhLW9wZXJhdG9yQGdyaWQtc3VwcG9ydC5hYy51a4IBADApBgNVHRIE\r\n - IjAggR5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMudWswGQYDVR0gBBIwEDAO\r\n - BgwrBgEEAdkvAQEBAQYwPQYJYIZIAYb4QgEEBDAWLmh0dHA6Ly9jYS5ncmlkLXN1\r\n - cHBvcnQuYWMudmT4sopwqlBWsvcHViL2NybC9jYWNybC5jcmwwPQYJYIZIAYb4Qg\r\n - EDBDAWLmh0dHA6Ly9jYS5ncmlkLXN1cHBvcnQuYWMudWsvcHViL2NybC9jYWNybC\r\n - 5jcmwwPwYDVR0fBDgwNjA0oDKgMIYuaHR0cDovL2NhLmdyaWQt5hYy51ay9wdWIv\r\n - Y3JsL2NhY3JsLmNybDANBgkqhkiG9w0BAQUFAAOCAQEAS/U4iiooBENGW/Hwmmd3\r\n - XCy6Zrt08YjKCzGNjorT98g8uGsqYjSxv/hmi0qlnlHs+k/3Iobc3LjS5AMYr5L8\r\n - UO7OSkgFFlLHQyC9JzPfmLCAugvzEbyv4Olnsr8hbxF1MbKZoQxUZtMVu29wjfXk\r\n - hTeApBv7eaKCWpSp7MCbvgzm74izKhu3vlDk9w6qVrxePfGgpKPqfHiOoGhFnbTK\r\n - wTC6o2xq5y0qZ03JonF7OJspEd3I5zKY3E+ov7/ZhW6DqT8UFvsAdjvQbXyhV8Eu\r\n - Yhixw1aKEPzNjNowuIseVogKOLXxWI5vAi5HgXdS0/ES5gDGsABo4fqovUKlgop3\r\n - RA==\r\n - -----END CERTIFICATE-----""".replace("\n\n", "\n") +certificate = """-----BEGIN CERTIFICATE----- + MIIFbTCCBFWgAwIBAgICH4cwDQYJKoZIhvcNAQEFBQAwcDELMAkGA1UEBhMCVUsx + ETAPBgNVBAoTCGVTY2llbmNlMRIwEAYDVQQLEwlBdXRob3JpdHkxCzAJBgNVBAMT + AkNBMS0wKwYJKoZIhvcNAQkBFh5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMu + dWswHhcNMDYwNzI3MTQxMzI4WhcNMDcwNzI3MTQxMzI4WjBbMQswCQYDVQQGEwJV + SzERMA8GA1UEChMIZVNjaWVuY2UxEzARBgNVBAsTCk1hbmNoZXN0ZXIxCzAJBgNV + BAcTmrsogriqMWLAk1DMRcwFQYDVQQDEw5taWNoYWVsIHBhcmQYJKoZIhvcNAQEB + BQADggEPADCCAQoCggEBANPEQBgl1IaKdSS1TbhF3hEXSl72G9J+WC/1R64fAcEF + W51rEyFYiIeZGx/BVzwXbeBoNUK41OK65sxGuflMo5gLflbwJtHBRIEKAfVVp3YR + gW7cMA/s/XKgL1GEC7rQw8lIZT8RApukCGqOVHSi/F1SiFlPDxuDfmdiNzL31+sL + 0iwHDdNkGjy5pyBSB8Y79dsSJtCW/iaLB0/n8Sj7HgvvZJ7x0fr+RQjYOUUfrePP + u2MSpFyf+9BbC/aXgaZuiCvSR+8Snv3xApQY+fULK/xY8h8Ua51iXoQ5jrgu2SqR + wgA7BUi3G8LFzMBl8FRCDYGUDy7M6QaHXx1ZWIPWNKsCAwEAAaOCAiQwggIgMAwG + 1UdEwEB/wQCMAAwEQYJYIZIAYb4QgHTTPAQDAgWgMA4GA1UdDwEB/wQEAwID6DAs + BglghkgBhvhCAQ0EHxYdVUsgZS1TY2llbmNlIFVzZXIgQ2VydGlmaWNhdGUwHQYD + VR0OBBYEFDTt/sf9PeMaZDHkUIldrDYMNTBZMIGaBgNVHSMEgZIwgY+AFAI4qxGj + loCLDdMVKwiljjDastqooXSkcjBwMQswCQYDVQQGEwJVSzERMA8GA1UEChMIZVNj + aWVuY2UxEjAQBgNVBAsTCUF1dGhvcml0eTELMAkGA1UEAxMCQ0ExLTArBgkqhkiG + 9w0BCQEWHmNhLW9wZXJhdG9yQGdyaWQtc3VwcG9ydC5hYy51a4IBADApBgNVHRIE + IjAggR5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMudWswGQYDVR0gBBIwEDAO + BgwrBgEEAdkvAQEBAQYwPQYJYIZIAYb4QgEEBDAWLmh0dHA6Ly9jYS5ncmlkLXN1 + cHBvcnQuYWMudmT4sopwqlBWsvcHViL2NybC9jYWNybC5jcmwwPQYJYIZIAYb4Qg + EDBDAWLmh0dHA6Ly9jYS5ncmlkLXN1cHBvcnQuYWMudWsvcHViL2NybC9jYWNybC + 5jcmwwPwYDVR0fBDgwNjA0oDKgMIYuaHR0cDovL2NhLmdyaWQt5hYy51ay9wdWIv + Y3JsL2NhY3JsLmNybDANBgkqhkiG9w0BAQUFAAOCAQEAS/U4iiooBENGW/Hwmmd3 + XCy6Zrt08YjKCzGNjorT98g8uGsqYjSxv/hmi0qlnlHs+k/3Iobc3LjS5AMYr5L8 + UO7OSkgFFlLHQyC9JzPfmLCAugvzEbyv4Olnsr8hbxF1MbKZoQxUZtMVu29wjfXk + hTeApBv7eaKCWpSp7MCbvgzm74izKhu3vlDk9w6qVrxePfGgpKPqfHiOoGhFnbTK + wTC6o2xq5y0qZ03JonF7OJspEd3I5zKY3E+ov7/ZhW6DqT8UFvsAdjvQbXyhV8Eu + Yhixw1aKEPzNjNowuIseVogKOLXxWI5vAi5HgXdS0/ES5gDGsABo4fqovUKlgop3 + RA== + -----END CERTIFICATE-----""".replace("\n", "") request = { "method": "GET", diff --git a/tests/requests/valid/025.http b/tests/requests/valid/025.http index 62267add0..214f3094c 100644 --- a/tests/requests/valid/025.http +++ b/tests/requests/valid/025.http @@ -1,10 +1,9 @@ POST /chunked_cont_h_at_first HTTP/1.1\r\n -Content-Length: -1\r\n Transfer-Encoding: chunked\r\n \r\n 5; some; parameters=stuff\r\n hello\r\n -6; blahblah; blah\r\n +6 \t;\tblahblah; blah\r\n world\r\n 0\r\n \r\n @@ -16,4 +15,10 @@ Content-Length: -1\r\n hello\r\n 6; blahblah; blah\r\n world\r\n -0\r\n \ No newline at end of file +0\r\n +\r\n +PUT /ignored_after_dangerous_framing HTTP/1.1\r\n +Content-Length: 3\r\n +\r\n +foo\r\n +\r\n diff --git a/tests/requests/valid/025.py b/tests/requests/valid/025.py index 12ea9ab76..33f5845cb 100644 --- a/tests/requests/valid/025.py +++ b/tests/requests/valid/025.py @@ -1,9 +1,13 @@ +from gunicorn.config import Config + +cfg = Config() +cfg.set("tolerate_dangerous_framing", True) + req1 = { "method": "POST", "uri": uri("/chunked_cont_h_at_first"), "version": (1, 1), "headers": [ - ("CONTENT-LENGTH", "-1"), ("TRANSFER-ENCODING", "chunked") ], "body": b"hello world" diff --git a/tests/requests/valid/025compat.http b/tests/requests/valid/025compat.http new file mode 100644 index 000000000..828f6fb71 --- /dev/null +++ b/tests/requests/valid/025compat.http @@ -0,0 +1,18 @@ +POST /chunked_cont_h_at_first HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +\r\n +5; some; parameters=stuff\r\n +hello\r\n +6; blahblah; blah\r\n + world\r\n +0\r\n +\r\n +PUT /chunked_cont_h_at_last HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +Content-Length: -1\r\n +\r\n +5; some; parameters=stuff\r\n +hello\r\n +6; blahblah; blah\r\n + world\r\n +0\r\n diff --git a/tests/requests/valid/025compat.py b/tests/requests/valid/025compat.py new file mode 100644 index 000000000..33f5845cb --- /dev/null +++ b/tests/requests/valid/025compat.py @@ -0,0 +1,27 @@ +from gunicorn.config import Config + +cfg = Config() +cfg.set("tolerate_dangerous_framing", True) + +req1 = { + "method": "POST", + "uri": uri("/chunked_cont_h_at_first"), + "version": (1, 1), + "headers": [ + ("TRANSFER-ENCODING", "chunked") + ], + "body": b"hello world" +} + +req2 = { + "method": "PUT", + "uri": uri("/chunked_cont_h_at_last"), + "version": (1, 1), + "headers": [ + ("TRANSFER-ENCODING", "chunked"), + ("CONTENT-LENGTH", "-1"), + ], + "body": b"hello world" +} + +request = [req1, req2] diff --git a/tests/requests/valid/029.http b/tests/requests/valid/029.http index c8611dbd3..5d029dd91 100644 --- a/tests/requests/valid/029.http +++ b/tests/requests/valid/029.http @@ -1,6 +1,6 @@ GET /stuff/here?foo=bar HTTP/1.1\r\n -Transfer-Encoding: chunked\r\n Transfer-Encoding: identity\r\n +Transfer-Encoding: chunked\r\n \r\n 5\r\n hello\r\n diff --git a/tests/requests/valid/029.py b/tests/requests/valid/029.py index f25449d15..64d026604 100644 --- a/tests/requests/valid/029.py +++ b/tests/requests/valid/029.py @@ -7,8 +7,8 @@ "uri": uri("/stuff/here?foo=bar"), "version": (1, 1), "headers": [ + ('TRANSFER-ENCODING', 'identity'), ('TRANSFER-ENCODING', 'chunked'), - ('TRANSFER-ENCODING', 'identity') ], "body": b"hello" } diff --git a/tests/requests/valid/031.http b/tests/requests/valid/031.http new file mode 100644 index 000000000..ab3529da3 --- /dev/null +++ b/tests/requests/valid/031.http @@ -0,0 +1,2 @@ +-BLARGH /foo HTTP/1.1\r\n +\r\n diff --git a/tests/requests/valid/031.py b/tests/requests/valid/031.py new file mode 100644 index 000000000..9691a002b --- /dev/null +++ b/tests/requests/valid/031.py @@ -0,0 +1,7 @@ +request = { + "method": "-BLARGH", + "uri": uri("/foo"), + "version": (1, 1), + "headers": [], + "body": b"" +} diff --git a/tests/requests/valid/031compat.http b/tests/requests/valid/031compat.http new file mode 100644 index 000000000..cd1ab7fcb --- /dev/null +++ b/tests/requests/valid/031compat.http @@ -0,0 +1,2 @@ +-blargh /foo HTTP/1.1\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/valid/031compat.py b/tests/requests/valid/031compat.py new file mode 100644 index 000000000..424b7cb47 --- /dev/null +++ b/tests/requests/valid/031compat.py @@ -0,0 +1,13 @@ +from gunicorn.config import Config + +cfg = Config() +cfg.set("permit_unconventional_http_method", True) +cfg.set("casefold_http_method", True) + +request = { + "method": "-BLARGH", + "uri": uri("/foo"), + "version": (1, 1), + "headers": [], + "body": b"" +} diff --git a/tests/requests/valid/031compat2.http b/tests/requests/valid/031compat2.http new file mode 100644 index 000000000..dcbf4f134 --- /dev/null +++ b/tests/requests/valid/031compat2.http @@ -0,0 +1,2 @@ +-blargh /foo HTTP/1.1\r\n +\r\n diff --git a/tests/requests/valid/031compat2.py b/tests/requests/valid/031compat2.py new file mode 100644 index 000000000..594a8b6a8 --- /dev/null +++ b/tests/requests/valid/031compat2.py @@ -0,0 +1,12 @@ +from gunicorn.config import Config + +cfg = Config() +cfg.set("permit_unconventional_http_method", True) + +request = { + "method": "-blargh", + "uri": uri("/foo"), + "version": (1, 1), + "headers": [], + "body": b"" +} diff --git a/tests/requests/valid/040.http b/tests/requests/valid/040.http new file mode 100644 index 000000000..be03929d5 --- /dev/null +++ b/tests/requests/valid/040.http @@ -0,0 +1,6 @@ +GET /keep/same/as?invalid/040 HTTP/1.0\r\n +Transfer_Encoding: tricked\r\n +Content-Length: 7\r\n +Content_Length: -1E23\r\n +\r\n +tricked\r\n diff --git a/tests/requests/valid/040.py b/tests/requests/valid/040.py new file mode 100644 index 000000000..7c2243d9f --- /dev/null +++ b/tests/requests/valid/040.py @@ -0,0 +1,9 @@ +request = { + "method": "GET", + "uri": uri("/keep/same/as?invalid/040"), + "version": (1, 0), + "headers": [ + ("CONTENT-LENGTH", "7") + ], + "body": b'tricked' +} diff --git a/tests/requests/valid/040_compat.http b/tests/requests/valid/040_compat.http new file mode 100644 index 000000000..be03929d5 --- /dev/null +++ b/tests/requests/valid/040_compat.http @@ -0,0 +1,6 @@ +GET /keep/same/as?invalid/040 HTTP/1.0\r\n +Transfer_Encoding: tricked\r\n +Content-Length: 7\r\n +Content_Length: -1E23\r\n +\r\n +tricked\r\n diff --git a/tests/requests/valid/040_compat.py b/tests/requests/valid/040_compat.py new file mode 100644 index 000000000..5f13487c4 --- /dev/null +++ b/tests/requests/valid/040_compat.py @@ -0,0 +1,16 @@ +from gunicorn.config import Config + +cfg = Config() +cfg.set("header_map", "dangerous") + +request = { + "method": "GET", + "uri": uri("/keep/same/as?invalid/040"), + "version": (1, 0), + "headers": [ + ("TRANSFER_ENCODING", "tricked"), + ("CONTENT-LENGTH", "7"), + ("CONTENT_LENGTH", "-1E23"), + ], + "body": b'tricked' +} diff --git a/tests/test_arbiter.py b/tests/test_arbiter.py index dc059edb7..e856282c8 100644 --- a/tests/test_arbiter.py +++ b/tests/test_arbiter.py @@ -4,7 +4,7 @@ # See the NOTICE for more information. import os -import unittest.mock as mock +from unittest import mock import gunicorn.app.base import gunicorn.arbiter diff --git a/tests/test_config.py b/tests/test_config.py index 92cb73c37..c094f6a21 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -318,6 +318,30 @@ def nworkers_changed_3(server, new_value, old_value): assert c.nworkers_changed(1, 2, 3) == 3 +def test_statsd_host(): + c = config.Config() + assert c.statsd_host is None + c.set("statsd_host", "localhost") + assert c.statsd_host == ("localhost", 8125) + c.set("statsd_host", "statsd:7777") + assert c.statsd_host == ("statsd", 7777) + c.set("statsd_host", "unix:///path/to.sock") + assert c.statsd_host == "/path/to.sock" + pytest.raises(TypeError, c.set, "statsd_host", 666) + pytest.raises(TypeError, c.set, "statsd_host", "host:string") + + +def test_statsd_host_with_unix_as_hostname(): + # This is a regression test for major release 20. After this release + # we should consider modifying the behavior of util.parse_address to + # simplify gunicorn's code + c = config.Config() + c.set("statsd_host", "unix:7777") + assert c.statsd_host == ("unix", 7777) + c.set("statsd_host", "unix://some.socket") + assert c.statsd_host == "some.socket" + + def test_statsd_changes_logger(): c = config.Config() assert c.logger_class == glogging.Logger @@ -429,41 +453,6 @@ def test_umask_config(options, expected): assert app.cfg.umask == expected -@pytest.mark.parametrize("options, expected", [ - (["--ssl-version", "SSLv23"], 2), - (["--ssl-version", "TLSv1"], 3), - (["--ssl-version", "2"], 2), - (["--ssl-version", "3"], 3), -]) -def test_ssl_version_named_constants_python3(options, expected): - _test_ssl_version(options, expected) - - -@pytest.mark.skipif(sys.version_info < (3, 6), - reason="requires python3.6+") -@pytest.mark.parametrize("options, expected", [ - (["--ssl-version", "TLS"], 2), - (["--ssl-version", "TLSv1_1"], 4), - (["--ssl-version", "TLSv1_2"], 5), - (["--ssl-version", "TLS_SERVER"], 17), -]) -def test_ssl_version_named_constants_python36(options, expected): - _test_ssl_version(options, expected) - - -@pytest.mark.parametrize("ssl_version", [ - "FOO", - "-99", - "99991234" -]) -def test_ssl_version_bad(ssl_version): - c = config.Config() - with pytest.raises(ValueError) as exc: - c.set("ssl_version", ssl_version) - assert 'Valid options' in str(exc.value) - assert "TLSv" in str(exc.value) - - def _test_ssl_version(options, expected): cmdline = ["prog_name"] cmdline.extend(options) diff --git a/tests/test_http.py b/tests/test_http.py index 33481266d..0eb694601 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -3,13 +3,24 @@ import io import t import pytest -import unittest.mock as mock +from unittest import mock from gunicorn import util from gunicorn.http.body import Body, LengthReader, EOFReader from gunicorn.http.wsgi import Response from gunicorn.http.unreader import Unreader, IterUnreader, SocketUnreader from gunicorn.http.errors import InvalidHeader, InvalidHeaderName +from gunicorn.http.message import TOKEN_RE + + +def test_method_pattern(): + assert TOKEN_RE.fullmatch("GET") + assert TOKEN_RE.fullmatch("MKCALENDAR") + assert not TOKEN_RE.fullmatch("GET:") + assert not TOKEN_RE.fullmatch("GET;") + RFC9110_5_6_2_TOKEN_DELIM = r'"(),/:;<=>?@[\]{}' + for bad_char in RFC9110_5_6_2_TOKEN_DELIM: + assert not TOKEN_RE.match(bad_char) def assert_readline(payload, size, expected): diff --git a/tests/test_pidfile.py b/tests/test_pidfile.py index fdcd0a24f..ecbc052ff 100644 --- a/tests/test_pidfile.py +++ b/tests/test_pidfile.py @@ -4,7 +4,7 @@ # See the NOTICE for more information. import errno -import unittest.mock as mock +from unittest import mock import gunicorn.pidfile diff --git a/tests/test_sock.py b/tests/test_sock.py index 36205d840..adc348c6f 100644 --- a/tests/test_sock.py +++ b/tests/test_sock.py @@ -3,7 +3,7 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -import unittest.mock as mock +from unittest import mock from gunicorn import sock diff --git a/tests/test_ssl.py b/tests/test_ssl.py index 97e05d86b..a31c1fe0f 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -8,7 +8,7 @@ import pytest from gunicorn.config import ( - KeyFile, CertFile, SSLVersion, CACerts, SuppressRaggedEOFs, + KeyFile, CertFile, CACerts, SuppressRaggedEOFs, DoHandshakeOnConnect, Setting, Ciphers, ) @@ -32,14 +32,6 @@ def test_certfile(): assert CertFile.default is None -def test_ssl_version(): - assert issubclass(SSLVersion, Setting) - assert SSLVersion.name == 'ssl_version' - assert SSLVersion.section == 'SSL' - assert SSLVersion.cli == ['--ssl-version'] - assert SSLVersion.default == ssl.PROTOCOL_SSLv23 - - def test_cacerts(): assert issubclass(CACerts, Setting) assert CACerts.name == 'ca_certs' diff --git a/tests/test_statsd.py b/tests/test_statsd.py index 06c1d9646..6f7bf426b 100644 --- a/tests/test_statsd.py +++ b/tests/test_statsd.py @@ -59,6 +59,18 @@ def test_statsd_fail(): logger.exception("No impact on logging") +def test_statsd_host_initialization(): + c = Config() + c.set('statsd_host', 'unix:test.sock') + logger = Statsd(c) + logger.info("Can be initialized and used with a UDS socket") + + # Can be initialized and used with a UDP address + c.set('statsd_host', 'host:8080') + logger = Statsd(c) + logger.info("Can be initialized and used with a UDP socket") + + def test_dogstatsd_tags(): c = Config() tags = 'yucatan,libertine:rhubarb' diff --git a/tests/test_systemd.py b/tests/test_systemd.py index d2c78aa2c..ff8b959a0 100644 --- a/tests/test_systemd.py +++ b/tests/test_systemd.py @@ -5,7 +5,7 @@ from contextlib import contextmanager import os -import unittest.mock as mock +from unittest import mock import pytest diff --git a/tests/treq.py b/tests/treq.py index ffe0691fd..aeaae151f 100644 --- a/tests/treq.py +++ b/tests/treq.py @@ -51,7 +51,9 @@ def __init__(self, fname, expect): with open(self.fname, 'rb') as handle: self.data = handle.read() self.data = self.data.replace(b"\n", b"").replace(b"\\r\\n", b"\r\n") - self.data = self.data.replace(b"\\0", b"\000") + self.data = self.data.replace(b"\\0", b"\000").replace(b"\\n", b"\n").replace(b"\\t", b"\t") + if b"\\" in self.data: + raise AssertionError("Unexpected backslash in test data - only handling HTAB, NUL and CRLF") # Functions for sending data to the parser. # These functions mock out reading from a @@ -246,8 +248,10 @@ def test_req(sn, sz, mt): def check(self, cfg, sender, sizer, matcher): cases = self.expect[:] p = RequestParser(cfg, sender(), None) - for req in p: + parsed_request_idx = -1 + for parsed_request_idx, req in enumerate(p): self.same(req, sizer, matcher, cases.pop(0)) + assert len(self.expect) == parsed_request_idx + 1 assert not cases def same(self, req, sizer, matcher, exp): @@ -262,7 +266,8 @@ def same(self, req, sizer, matcher, exp): assert req.trailers == exp.get("trailers", []) -class badrequest(object): +class badrequest: + # FIXME: no good reason why this cannot match what the more extensive mechanism above def __init__(self, fname): self.fname = fname self.name = os.path.basename(fname) @@ -270,7 +275,9 @@ def __init__(self, fname): with open(self.fname) as handle: self.data = handle.read() self.data = self.data.replace("\n", "").replace("\\r\\n", "\r\n") - self.data = self.data.replace("\\0", "\000") + self.data = self.data.replace("\\0", "\000").replace("\\n", "\n").replace("\\t", "\t") + if "\\" in self.data: + raise AssertionError("Unexpected backslash in test data - only handling HTAB, NUL and CRLF") self.data = self.data.encode('latin1') def send(self): @@ -283,4 +290,6 @@ def send(self): def check(self, cfg): p = RequestParser(cfg, self.send(), None) - next(p) + # must fully consume iterator, otherwise EOF errors could go unnoticed + for _ in p: + pass diff --git a/tests/workers/test_geventlet.py b/tests/workers/test_geventlet.py index 815dcec34..06c7b5305 100644 --- a/tests/workers/test_geventlet.py +++ b/tests/workers/test_geventlet.py @@ -3,5 +3,15 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. +import pytest +import sys + def test_import(): + + try: + import eventlet + except AttributeError: + if (3,13) > sys.version_info >= (3, 12): + pytest.skip("Ignoring eventlet failures on Python 3.12") + raise __import__('gunicorn.workers.geventlet') diff --git a/tox.ini b/tox.ini index 6000eb617..c1c2fd053 100644 --- a/tox.ini +++ b/tox.ini @@ -1,16 +1,32 @@ [tox] -envlist = py35, py36, py37, py38, py39, pypy3, lint -skipsdist = True +envlist = + py{37,38,39,310,311,312,py3}, + lint, + docs-lint, + pycodestyle, + run-entrypoint, + run-module, +skipsdist = false +; Can't set skipsdist and use_develop in tox v4 to true due to https://github.com/tox-dev/tox/issues/2730 [testenv] -usedevelop = True -commands = py.test --cov=gunicorn {posargs} +use_develop = true +commands = pytest --cov=gunicorn {posargs} deps = -rrequirements_test.txt +[testenv:run-entrypoint] +# entry point: console script (provided by setuptools from pyproject.toml) +commands = python -c 'import subprocess; cmd_out = subprocess.check_output(["gunicorn", "--version"])[:79].decode("utf-8", errors="replace"); print(cmd_out); assert cmd_out.startswith("gunicorn ")' + +[testenv:run-module] +# runpy (provided by module.__main__) +commands = python -c 'import sys,subprocess; cmd_out = subprocess.check_output([sys.executable, "-m", "gunicorn", "--version"])[:79].decode("utf-8", errors="replace"); print(cmd_out); assert cmd_out.startswith("gunicorn ")' + [testenv:lint] commands = pylint -j0 \ + --max-line-length=120 \ gunicorn \ tests/test_arbiter.py \ tests/test_config.py \ @@ -25,10 +41,10 @@ commands = tests/test_util.py \ tests/test_valid_requests.py deps = - pylint + pylint==2.17.4 [testenv:docs-lint] -whitelist_externals = +allowlist_externals = rst-lint bash grep