Better checksum calculation (aio-libs#5183) (aio-libs#5185)

Co-authored-by: Andrew Svetlov <[email protected]>

.github/workflows/ci.yml

@@ -115,27 +115,16 @@ jobs:
  path: ${{ steps.pip-cache.outputs.dir }}
  restore-keys: |
  pip-ci-${{ runner.os }}-${{ matrix.pyver }}-{{ matrix.no-extensions }}-
- - name: Install cython
- if: ${{ matrix.no-extensions == '' }}
- uses: py-actions/py-dependency-install@v2
- with:
- path: requirements/cython.txt
  - name: Cythonize
  if: ${{ matrix.no-extensions == '' }}
  run: |
  make cythonize
- - name: Install dependencies
- uses: py-actions/py-dependency-install@v2
- with:
- path: requirements/test.txt
- env:
- AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
  - name: Run unittests
  env:
  COLOR: 'yes'
  AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
  run: |
- python -m pytest tests -vv
+ make vvtest
  python -m coverage xml
  - name: Upload coverage
  uses: codecov/codecov-action@v1
@@ -168,10 +157,6 @@ jobs:
  uses: actions/setup-python@v2
  with:
  python-version: 3.8
- - name: Install cython
- uses: py-actions/py-dependency-install@v2
- with:
- path: requirements/cython.txt
  - name: Cythonize
  run: |
  make cythonize
@@ -210,11 +195,6 @@ jobs:
  uses: actions/setup-python@v2
  with:
  python-version: 3.8
- - name: Install cython
- if: ${{ matrix.no-extensions == '' }}
- uses: py-actions/py-dependency-install@v2
- with:
- path: requirements/cython.txt
  - name: Cythonize
  if: ${{ matrix.no-extensions == '' }}
  run: |
@@ -255,11 +235,6 @@ jobs:
  uses: actions/setup-python@v2
  with:
  python-version: ${{ matrix.pyver }}
- - name: Install cython
- if: ${{ matrix.no-extensions == '' }}
- uses: py-actions/py-dependency-install@v2
- with:
- path: requirements/cython.txt
  - name: Cythonize
  if: ${{ matrix.no-extensions == '' }}
  run: |

.gitignore

@@ -20,6 +20,7 @@
 .envrc
 .flake
 .gitconfig
+.hash
 .idea
 .install-cython
 .install-deps

Makefile

@@ -1,47 +1,67 @@
 # Some simple testing tasks (sorry, UNIX only).
 
-to-md5 = $1 $(addsuffix .md5,$1)
+to-hash-one = $(dir $1).hash/$(addsuffix .hash,$(notdir $1))
+to-hash = $(foreach fname,$1,$(call to-hash-one,$(fname)))
 
-CYS = $(wildcard aiohttp/*.pyx) $(wildcard aiohttp/*.pyi) $(wildcard aiohttp/*.pxd)
-PYXS = $(wildcard aiohttp/*.pyx)
-CS = $(wildcard aiohttp/*.c)
-PYS = $(wildcard aiohttp/*.py)
-REQS = $(wildcard requirements/*.txt)
-SRC = aiohttp examples tests setup.py
+CYS := $(wildcard aiohttp/*.pyx) $(wildcard aiohttp/*.pyi) $(wildcard aiohttp/*.pxd)
+PYXS := $(wildcard aiohttp/*.pyx)
+CS := $(wildcard aiohttp/*.c)
+PYS := $(wildcard aiohttp/*.py)
+REQS := $(wildcard requirements/*.txt)
+ALLS := $(sort $(CYS) $(CS) $(PYS) $(REQS))
 
 .PHONY: all
 all: test
 
-# Recipe from https://www.cmcrossroads.com/article/rebuilding-when-files-checksum-changes
-%.md5: FORCE
- @$(if $(filter-out $(shell cat $@ 2>/dev/null),$(shell md5sum $*)),md5sum $* > $@)
+tst:
+ @echo $(call to-hash,requirements/cython.txt)
+ @echo $(call to-hash,aiohttp/%.pyx)
+
 
+# Recipe from https://www.cmcrossroads.com/article/rebuilding-when-files-checksum-changes
 FORCE:
 
+# check_sum.py works perfectly fine but slow when called for every file from $(ALLS)
+# (perhaps even several times for each file).
+# That is why much less readable but faster solution exists
+ifneq (, $(shell which sha256sum))
+%.hash: FORCE
+ $(eval $@_ABS := $(abspath $@))
+ $(eval $@_NAME := $($@_ABS))
+ $(eval $@_HASHDIR := $(dir $($@_ABS)))
+ $(eval $@_TMP := $($@_HASHDIR)../$(notdir $($@_ABS)))
+ $(eval $@_ORIG := $(subst /.hash/../,/,$(basename $($@_TMP))))
+ @#echo ==== $($@_ABS) $($@_HASHDIR) $($@_NAME) $($@_TMP) $($@_ORIG)
+ @if ! (sha256sum --check $($@_ABS) 1>/dev/null 2>/dev/null); then \
+ mkdir -p $($@_HASHDIR); \
+ echo re-hash $($@_ORIG); \
+ sha256sum $($@_ORIG) > $($@_ABS); \
+ fi
+else
+%.hash: FORCE
+ @./tools/check_sum.py $@ # --debug
+endif
+
 # Enumerate intermediate files to don't remove them automatically.
-# The target must exist, no need to execute it.
-.PHONY: _keep-intermediate-files
-_keep-intermediate-files: $(addsuffix .md5,$(CYS))\
- $(addsuffix .md5,$(CS))\
- $(addsuffix .md5,$(PYS))\
- $(addsuffix .md5,$(REQS))
-
-.install-cython: $(call to-md5,requirements/cython.txt)
+.SECONDARY: $(call to-hash,$(ALLS))
+
+
+.install-cython: $(call to-hash,requirements/cython.txt)
  pip install -r requirements/cython.txt
  @touch .install-cython
 
-aiohttp/_find_header.c: $(call to-md5,aiohttp/hdrs.py)
+aiohttp/_find_header.c: $(call to-hash,aiohttp/hdrs.py ./tools/gen.py)
  ./tools/gen.py
 
 # _find_headers generator creates _headers.pyi as well
-aiohttp/%.c: $(call to-md5,aiohttp/%.pyx) aiohttp/_find_header.c
+aiohttp/%.c: aiohttp/%.pyx $(call to-hash,$(CYS)) aiohttp/_find_header.c
  cython -3 -o $@ $< -I aiohttp
 
 
 .PHONY: cythonize
 cythonize: .install-cython $(PYXS:.pyx=.c)
 
-.install-deps: .install-cython $(PYXS:.pyx=.c) $(call to-md5,$(CYS) $(REQS))
+.install-deps: .install-cython $(PYXS:.pyx=.c) $(call to-hash,$(CYS) $(REQS))
  pip install -r requirements/dev.txt
  @touch .install-deps
 
@@ -56,7 +76,7 @@ fmt format:
 mypy:
  mypy aiohttp
 
-.develop: .install-deps $(call to-md5,$(PYS) $(CYS) $(CS))
+.develop: .install-deps $(call to-hash,$(PYS) $(CYS) $(CS))
  pip install -e .
  @touch .develop
 
@@ -68,16 +88,23 @@ test: .develop
 vtest: .develop
  @pytest -s -v
 
+.PHONY: vvtest
+vvtest: .develop
+ @pytest -vv
+
 .PHONY: clean
 clean:
  @rm -rf `find . -name __pycache__`
+ @rm -rf `find . -name .hash`
+ @rm -rf `find . -name .md5` # old styling
  @rm -f `find . -type f -name '*.py[co]' `
  @rm -f `find . -type f -name '*~' `
  @rm -f `find . -type f -name '.*~' `
  @rm -f `find . -type f -name '@*' `
  @rm -f `find . -type f -name '#*#' `
  @rm -f `find . -type f -name '*.orig' `
  @rm -f `find . -type f -name '*.rej' `
+ @rm -f `find . -type f -name '*.md5' ` # old styling
  @rm -f .coverage
  @rm -rf htmlcov
  @rm -rf build

requirements/lint.txt

@@ -1,4 +1,4 @@
-black==20.8b1; python_version >= "3.6"
+black==20.8b1; implementation_name=="cpython"
 flake8==3.8.4
 flake8-pyi==20.10.0
 isort==5.6.4

tools/check_sum.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+import argparse
+import hashlib
+import pathlib
+import sys
+
+PARSER = argparse.ArgumentParser(
+ description="Helper for check file hashes in Makefile instead of bare timestamps"
+)
+PARSER.add_argument("dst", metavar="DST", type=pathlib.Path)
+PARSER.add_argument("-d", "--debug", action="store_true", default=False)
+
+
+def main(argv):
+ args = PARSER.parse_args(argv)
+ dst = args.dst
+ assert dst.suffix == ".hash"
+ dirname = dst.parent
+ if dirname.name != ".hash":
+ if args.debug:
+ print(f"Invalid name {dst} -> dirname {dirname}", file=sys.stderr)
+ return 0
+ dirname.mkdir(exist_ok=True)
+ src_dir = dirname.parent
+ src_name = dst.stem # drop .hash
+ full_src = src_dir / src_name
+ hasher = hashlib.sha256()
+ try:
+ hasher.update(full_src.read_bytes())
+ except OSError:
+ if args.debug:
+ print(f"Cannot open {full_src}", file=sys.stderr)
+ return 0
+ src_hash = hasher.hexdigest()
+ if dst.exists():
+ dst_hash = dst.read_text()
+ else:
+ dst_hash = ""
+ if src_hash != dst_hash:
+ dst.write_text(src_hash)
+ print(f"re-hash {src_hash}")
+ else:
+ if args.debug:
+ print(f"Skip {src_hash} checksum, up-to-date")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))