Add typesafety checks (#212)

* tests: add typesafety tests

* nox: add typesafety command

* upath: typing fixes for generated flavours

* upath: update _flavour_sources.py

* upath: more typing fixes

* typesafety: add more tests

* upath: typing fixes

* ci: add typesafety checks

* nox -s lint

* upath: exclude fsspec==2024.3.1 for now

* _strip_protocol signature changes break local fs tests
* Windows URI parsing has issues

* nox -s lint fixes

* typesafety: add pathlib interface tests

* upath: various typing fixes

* typesafety: fix mypy output issue on python3.8

* typesafety: check .link_to and .walk

.github/workflows/tests.yml

@@ -49,6 +49,24 @@ jobs:
  - name: Run tests
  run: nox -s tests-${{ matrix.fsspec || matrix.pyv }} -- --cov-report=xml
 
+ typesafety:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Check out the repository
+ uses: actions/checkout@v4
+
+ - name: Set up Python ${{ matrix.pyv }}
+ uses: actions/setup-python@v4
+ with:
+ python-version: '3.8'
+
+ - name: Install nox
+ run: python -m pip install --upgrade nox
+
+ - name: Run typesafety checks
+ run: nox -s typesafety
+
  lint:
  runs-on: ubuntu-latest
 

dev/generate_flavours.py

@@ -43,6 +43,7 @@
 import logging
 import re
 from typing import Any
+from typing import Literal
 from typing import cast
 from urllib.parse import parse_qs
 from urllib.parse import urlsplit
@@ -67,6 +68,22 @@
 class FileSystemFlavourBase:
  """base class for the fsspec flavours"""
 
+ protocol: str | tuple[str, ...]
+ root_marker: Literal["/", ""]
+ sep: Literal["/"]
+
+ @classmethod
+ def _strip_protocol(cls, path):
+ raise NotImplementedError
+
+ @staticmethod
+ def _get_kwargs_from_urls(path):
+ raise NotImplementedError
+
+ @classmethod
+ def _parent(cls, path):
+ raise NotImplementedError
+
  def __init_subclass__(cls: Any, **kwargs):
  if isinstance(cls.protocol, str):
  protocols = (cls.protocol,)
@@ -99,12 +116,27 @@ def __init_subclass__(cls: Any, **kwargs):
 }
 
 
+def _fix_abstract_file_system(x: str) -> str:
+ x = re.sub(
+ "protocol = 'abstract'", "protocol: str | tuple[str, ...] = 'abstract'", x
+ )
+ x = re.sub("root_marker = ''", "root_marker: Literal['', '/'] = ''", x)
+ x = re.sub("sep = '/'", "sep: Literal['/'] = '/'", x)
+ return x
+
+
 def _fix_azure_blob_file_system(x: str) -> str:
- return re.sub(
- r"host = ops.get\(\"host\", None\)",
- 'host: str | None = ops.get("host", None)',
+ x = re.sub(
+ r"if isinstance\(path, list\):",
+ "if isinstance(path, list): # type: ignore[unreachable]",
  x,
  )
+ x = re.sub(
+ r"(return \[.*\])",
+ r"\1 # type: ignore[unreachable]",
+ x,
+ )
+ return x
 
 
 def _fix_memfs_file_system(x: str) -> str:
@@ -115,6 +147,15 @@ def _fix_memfs_file_system(x: str) -> str:
  )
 
 
+def _fix_oss_file_system(x: str) -> str:
+ x = re.sub(
+ r"path_string: str = stringify_path\(path\)",
+ "path_string = stringify_path(path)",
+ x,
+ )
+ return x
+
+
 def _fix_xrootd_file_system(x: str) -> str:
  x = re.sub(
  r"client.URL",
@@ -129,8 +170,10 @@ def _fix_xrootd_file_system(x: str) -> str:
 
 
 FIX_SOURCE = {
+ "AbstractFileSystem": _fix_abstract_file_system,
  "AzureBlobFileSystem": _fix_azure_blob_file_system,
  "MemFS": _fix_memfs_file_system,
+ "OSSFileSystem": _fix_oss_file_system,
  "XRootDFileSystem": _fix_xrootd_file_system,
 }
 
@@ -303,7 +346,7 @@ def create_source() -> str:
  AbstractFileSystem,
  ["_strip_protocol", "_get_kwargs_from_urls", "_parent"],
  {},
- ["protocol", "root_marker"],
+ ["protocol", "root_marker", "sep"],
  cls_suffix=BASE_CLASS_NAME_SUFFIX,
  base_cls="FileSystemFlavourBase",
  )

dev/requirements.txt

@@ -0,0 +1,18 @@
+fsspec[git,hdfs,dask,http,sftp,smb]==2024.2.0
+
+# these dependencies define their own filesystems
+adlfs==2024.2.0
+boxfs==0.2.1
+dropboxdrivefs==1.3.1
+gcsfs==2024.2.0
+s3fs==2024.2.0
+ocifs==1.3.1
+webdav4[fsspec]==0.9.8
+# gfrivefs @ git+https://github.com/fsspec/gdrivefs@master broken ...
+morefs[asynclocalfs]==0.2.0
+dvc==3.47.0
+huggingface_hub==0.20.3
+lakefs-spec==0.7.0
+ossfs==2023.12.0
+fsspec-xrootd==0.2.4
+wandbfs==0.0.2

noxfile.py

@@ -91,6 +91,22 @@ def type_checking(session):
  session.run("python", "-m", "mypy")
 
 
+@nox.session
+def typesafety(session):
+ session.install("-e", ".[tests]")
+ session.run(
+ "python",
+ "-m",
+ "pytest",
+ "-v",
+ "-p",
+ "pytest-mypy-plugins",
+ "--mypy-pyproject-toml-file",
+ "pyproject.toml",
+ "typesafety",
+ )
+
+
 @nox.session()
 def smoke(session):
  print("please tun `nox -s tests` instead")

pyproject.toml

@@ -37,7 +37,7 @@ force_single_line = true
 line_length = 88
 
 [tool.pytest.ini_options]
-addopts = "-ra -m 'not hdfs'"
+addopts = "-ra -m 'not hdfs' -p no:pytest-mypy-plugins"
 markers = [
  "hdfs: mark test as hdfs",
  "pathlib: mark cpython pathlib tests",
@@ -61,7 +61,7 @@ exclude_lines = [
 
 [tool.mypy]
 # Error output
-show_column_numbers = true
+show_column_numbers = false
 show_error_codes = true
 show_error_context = true
 show_traceback = true

setup.cfg

@@ -24,7 +24,7 @@ python_requires = >=3.8
 zip_safe = False
 packages = find:
 install_requires=
- fsspec>=2022.1.0
+ fsspec >=2022.1.0,!=2024.3.1
 
 [options.extras_require]
 tests =
@@ -33,7 +33,8 @@ tests =
  pytest-cov==4.1.0
  pytest-mock==3.12.0
  pylint==2.17.4
- mypy==1.8.0
+ mypy==1.10.0
+ pytest-mypy-plugins==3.1.2
  packaging
 dev =
  %(tests)s