add py35 support ( without concat_win) (#11)

* Add release (#1) * add release.yaml to help release * change the action name * delete trigger branch limit * add branch limit * Add release (#2) * add release.yaml to help release * change the action name * delete trigger branch limit * add branch limit * add needs * fix some bugs * add wheel to requirements-dev.txt * fix wrong command * move publish to a single job * delete wrong command * change upload mistake * fix wrong macos platform tag * Add win compatibility (#3) * juicefs can format and init on Windows * Wrap os.path.join() with Path().as_posix() to support Windows. jfs.concat() remains problematic on Windows * format the files * Win build (#4) * add win build and test action * fix syntax error * enable download artifact on windows * add win build and test action * fix syntax error * enable download artifact on windows * add windows build and test to on-push.yaml * change the build win binary if statements * remove win_build_test.yaml. It's merged into on-push.yaml * add windows build and publish to release.yaml * update requirements-dev.txt to support python3.5 * fix requirements-dev.txt syntax error * delete the f-string because python3.5 doesn't support it * only test python3.5 in test * make variable annotation compatible with py35 * make variable annotation compatible with py35 * make variable annotation compatible with py35 * format the file * add py36, py37, py38, py39 to the ci build and test
megvii-research · Aug 30, 2021 · 3545e3e · 3545e3e
1 parent 10d7fb7
commit 3545e3e
Show file tree

Hide file tree

Showing 5 changed files with 72 additions and 27 deletions.
diff --git a/.github/workflows/on-push.yaml b/.github/workflows/on-push.yaml
@@ -87,7 +87,8 @@ jobs:
  strategy:
  matrix:
  os: [macos-latest, ubuntu-18.04, windows-latest]
- python-version: [3.6, 3.7, 3.8, 3.9]
+ python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
+
  include:
  - os: ubuntu-18.04
  os-key: linux

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -28,11 +28,35 @@ jobs:
  ./juicefs/lib/juicefs
  key: ${{ matrix.os }}-jfs-binary-${{ steps.jfs-version.outputs.JUICEFS_VERSION }}
 
+ - name: Cache juicefs and libjfs binary windows
+ if: ${{ matrix.os == 'ubuntu-18.04' }}
+ uses: actions/cache@v2
+ id: jfs-cache-win
+ with:
+ path: |
+ ./juicefs/lib/libjfs.dll
+ ./juicefs/lib/juicefs.exe
+ key: ${{ matrix.os }}-jfs-binary-win-${{ steps.jfs-version.outputs.JUICEFS_VERSION }}
+
+
  - name: Run build-libjfs linux/mac
  if: ${{ steps.jfs-cache.outputs.cache-hit != 'true' }}
  run: |
  make build_libjfs_so
 
+
+ - name: Install MinGW GCC
+ if: ${{ steps.jfs-cache-win.outputs.cache-hit != 'true' && matrix.os == 'ubuntu-18.04'}}
+ run: |
+ sudo apt-get update
+ sudo apt-get install gcc-mingw-w64-x86-64
+
+ - name: Run build-libjfs windows
+ if: ${{ steps.jfs-cache-win.outputs.cache-hit != 'true' && matrix.os == 'ubuntu-18.04'}}
+ run: |
+ make build_libjfs_dll
+
+
  - name: Upload linux jfs binary
  if: matrix.os == 'ubuntu-18.04'
  uses: actions/upload-artifact@v2
@@ -51,13 +75,22 @@ jobs:
  ./juicefs/lib/libjfs.so
  ./juicefs/lib/juicefs
  retention-days: 1
+ - name: Upload windows jfs binary
+ if: matrix.os == 'ubuntu-18.04'
+ uses: actions/upload-artifact@v2
+ with:
+ name: jfs-binary-win-${{ steps.jfs-version.outputs.JUICEFS_VERSION }}
+ path: |
+ ./juicefs/lib/libjfs.dll
+ ./juicefs/lib/juicefs.exe
+ retention-days: 1
 
  build_release:
  needs: build-juicefs-libjfs
  runs-on: ${{ matrix.os }}
  strategy:
  matrix:
- os: [ubuntu-18.04, macos-latest]
+ os: [macos-latest, ubuntu-18.04, windows-latest]
  python-version: [3.6]
  include:
  - os: ubuntu-18.04
@@ -69,6 +102,12 @@ jobs:
  pip-path: /Users/runner/hostedtoolcache/Python
  plat-name: macosx_10_15_x86_64
 
+ - os: windows-latest
+ os-key: win
+ pip-path: C:\hostedtoolcache\windows\Python
+ plat-name: win_amd64
+
+
  steps:
  - name: Checkout Github Repository
  uses: actions/checkout@v2
@@ -91,8 +130,8 @@ jobs:
  - name: Get JuiceFS Version
  id: jfs-version
  run: echo "::set-output name=JUICEFS_VERSION::$(make -s print_libjfs_version)"
- - name: Download jfs binary artifact Linux/Mac
-  if: ${{ matrix.os != 'windows-latest' }}
+
+ - name: Download jfs binary artifact Linux/Mac/Windows
  uses: actions/download-artifact@v2
  with:
  name: jfs-binary-${{ matrix.os-key }}-${{ steps.jfs-version.outputs.JUICEFS_VERSION }}
@@ -102,6 +141,14 @@ jobs:
  run: |
  chmod 755 ./juicefs/lib/juicefs
  ls -l ./juicefs/lib/juicefs
+
+
+ - name: Chmod to make juicefs executable windows
+ if: ${{ matrix.os == 'windows-latest' }}
+ run: |
+ chmod 755 ./juicefs/lib/juicefs.exe
+ ls -l ./juicefs/lib/juicefs.exe
+
  - name: Build wheel
  run: |
  python setup.py bdist_wheel --plat-name ${{ matrix.plat-name }}
@@ -136,6 +183,13 @@ jobs:
  with:
  name: jfs-wheel-mac-${{ steps.jfs-version.outputs.JUICEFS_VERSION }}
  path: ./dist/
+
+ - name: Download jfs wheel artifact Windows
+ uses: actions/download-artifact@v2
+ with:
+ name: jfs-wheel-win-${{ steps.jfs-version.outputs.JUICEFS_VERSION }}
+ path: ./dist/
+
  - name: Publish package to PyPI
  uses: pypa/gh-action-pypi-publish@release/v1
  with:

diff --git a/juicefs/io.py b/juicefs/io.py
@@ -49,7 +49,7 @@ def __del__(self):
  self.close()
 
  def __getstate__(self):
- raise TypeError(f"cannot pickle {self.__class__.__name__!r} object")
+ raise TypeError("cannot pickle '{}' object".format(self.__class__.__name__))
 
  def __repr__(self) -> str:
  class_name = "%s.%s" % (self.__class__.__module__, self.__class__.__qualname__)

diff --git a/juicefs/libjfs.py b/juicefs/libjfs.py
@@ -107,15 +107,10 @@ def create_summary(data: bytes):
 
 
 class DirEntry:
-
- _stat: os.stat_result
- root: str
- name: str
-
  def __init__(self, name: str, root: str, stat: os.stat_result):
- self.name = name
- self.root = root
- self._stat = stat
+ self.name = name # type: str
+ self.root = root # type: str
+ self._stat = stat # type: os.stat_result
 
  def __repr__(self):
  return "<DirEntry %r>" % self.name
@@ -141,15 +136,10 @@ def stat(self):
 
 
 class DirSummary:
-
- size: int
- files: int
- dirs: int
-
  def __init__(self, size: int, files: int, dirs: int):
- self.size = size
- self.files = files
- self.dirs = dirs
+ self.size = size # type: int
+ self.files = files # type: int
+ self.dirs = dirs # type: int
 
  def __repr__(self):
  return "<DirSummary size=%d files=%d dirs=%d>" % (

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -6,11 +6,11 @@ pytest-cov
 pytest-mock
 pytest-socket
 
-pytype; sys_platform != 'win32'
+pytype; sys_platform != 'win32' and python_version == '3.6'
 
-isort ~= 5.0; sys_platform != 'win32'
-black ~= 21.5b0; sys_platform != 'win32'
+isort ~= 5.0; sys_platform != 'win32' and python_version == '3.6'
+black ~= 21.5b0; sys_platform != 'win32' and python_version == '3.6'
 
-Sphinx >= 4.0.0; sys_platform != 'win32'
-sphinx-rtd-theme; sys_platform != 'win32'
-m2r2; sys_platform != 'win32'
+Sphinx >= 4.0.0; sys_platform != 'win32' and python_version == '3.6'
+sphinx-rtd-theme; sys_platform != 'win32' and python_version == '3.6'
+m2r2; sys_platform != 'win32' and python_version == '3.6'