Add Windows compatibility (#7)

* Add release (#1)

* add release.yaml to help release

* change the action name

* delete trigger branch limit

* add branch limit

* Add release (#2)

* add release.yaml to help release

* change the action name

* delete trigger branch limit

* add branch limit

* add needs

* fix some bugs

* add wheel to requirements-dev.txt

* fix wrong command

* move publish to a single job

* delete wrong command

* change upload mistake

* fix wrong macos platform tag

* juicefs can format and init on Windows

* Wrap os.path.join() with Path().as_posix() to support Windows. jfs.concat() remains problematic on Windows

* format the files

.github/workflows/on-push.yaml

@@ -20,7 +20,6 @@ jobs:
  run: echo "::set-output name=JUICEFS_VERSION::$(make -s print_libjfs_version)"
 
  - name: Cache juicefs and libjfs binary linux/mac
- if: matrix.os == 'ubuntu-18.04'
  uses: actions/cache@v2
  id: jfs-cache
  with:

.github/workflows/release.yaml

@@ -0,0 +1,142 @@
+name: release wheel
+on:
+ workflow_dispatch:
+ branches: [main]
+
+jobs:
+ build-juicefs-libjfs:
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [ubuntu-18.04, macos-latest]
+
+ steps:
+ - name: Checkout Github Repository
+ uses: actions/checkout@v2
+ - name: Set up Golang
+ uses: actions/setup-go@v2
+ - name: Get JuiceFS Version
+ id: jfs-version
+ run: echo "::set-output name=JUICEFS_VERSION::$(make -s print_libjfs_version)"
+
+ - name: Cache juicefs and libjfs binary linux/mac
+ uses: actions/cache@v2
+ id: jfs-cache
+ with:
+ path: |
+ ./juicefs/lib/libjfs.so
+ ./juicefs/lib/juicefs
+ key: ${{ matrix.os }}-jfs-binary-${{ steps.jfs-version.outputs.JUICEFS_VERSION }}
+
+ - name: Run build-libjfs linux/mac
+ if: ${{ steps.jfs-cache.outputs.cache-hit != 'true' }}
+ run: |
+ make build_libjfs_so
+
+ - name: Upload linux jfs binary
+ if: matrix.os == 'ubuntu-18.04'
+ uses: actions/upload-artifact@v2
+ with:
+ name: jfs-binary-linux-${{ steps.jfs-version.outputs.JUICEFS_VERSION }}
+ path: |
+ ./juicefs/lib/libjfs.so
+ ./juicefs/lib/juicefs
+ retention-days: 1
+ - name: Upload macos jfs binary
+ if: matrix.os == 'macos-latest'
+ uses: actions/upload-artifact@v2
+ with:
+ name: jfs-binary-mac-${{ steps.jfs-version.outputs.JUICEFS_VERSION }}
+ path: |
+ ./juicefs/lib/libjfs.so
+ ./juicefs/lib/juicefs
+ retention-days: 1
+
+ build_release:
+ needs: build-juicefs-libjfs
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [ubuntu-18.04, macos-latest]
+ python-version: [3.6]
+ include:
+ - os: ubuntu-18.04
+ os-key: linux
+ pip-path: /opt/hostedtoolcache/Python
+ plat-name: manylinux2014_x86_64
+ - os: macos-latest
+ os-key: mac
+ pip-path: /Users/runner/hostedtoolcache/Python
+ plat-name: macosx_10_15_x86_64
+
+ steps:
+ - name: Checkout Github Repository
+ uses: actions/checkout@v2
+ - name: Set up Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Cache Pip
+ id: pip-cache
+ uses: actions/cache@v2
+ with:
+ path: ${{ matrix.pip-path }}
+ key: ${{ matrix.os }}-python${{ matrix.python-version }}-pip-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }}
+ - name: Install Package Dependencies
+ if: steps.pip-cache.outputs.cache-hit != 'true'
+ run: |
+ python -m pip install --upgrade pip
+ pip install -r requirements-dev.txt
+ pip install -r requirements.txt
+ - name: Get JuiceFS Version
+ id: jfs-version
+ run: echo "::set-output name=JUICEFS_VERSION::$(make -s print_libjfs_version)"
+ - name: Download jfs binary artifact Linux/Mac
+ if: ${{ matrix.os != 'windows-latest' }}
+ uses: actions/download-artifact@v2
+ with:
+ name: jfs-binary-${{ matrix.os-key }}-${{ steps.jfs-version.outputs.JUICEFS_VERSION }}
+ path: ./juicefs/lib/
+ - name: Chmod to make juicefs executable linux/mac
+ if: ${{ matrix.os != 'windows-latest' }}
+ run: |
+ chmod 755 ./juicefs/lib/juicefs
+ ls -l ./juicefs/lib/juicefs
+ - name: Build wheel
+ run: |
+ python setup.py bdist_wheel --plat-name ${{ matrix.plat-name }}
+ - name: Upload Wheel
+ uses: actions/upload-artifact@v2
+ with:
+ name: jfs-wheel-${{ matrix.os-key }}-${{ steps.jfs-version.outputs.JUICEFS_VERSION }}
+ path: |
+ ./dist/*
+ retention-days: 1
+
+ publish_release:
+ needs: build_release
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [ubuntu-18.04]
+
+ steps:
+ - name: Checkout Github Repository
+ uses: actions/checkout@v2
+ - name: Get JuiceFS Version
+ id: jfs-version
+ run: echo "::set-output name=JUICEFS_VERSION::$(make -s print_libjfs_version)"
+ - name: Download jfs wheel artifact Linux
+ uses: actions/download-artifact@v2
+ with:
+ name: jfs-wheel-linux-${{ steps.jfs-version.outputs.JUICEFS_VERSION }}
+ path: ./dist/
+ - name: Download jfs wheel artifact Mac
+ uses: actions/download-artifact@v2
+ with:
+ name: jfs-wheel-mac-${{ steps.jfs-version.outputs.JUICEFS_VERSION }}
+ path: ./dist/
+ - name: Publish package to PyPI
+ uses: pypa/gh-action-pypi-publish@release/v1
+ with:
+ password: ${{ secrets.JUICEFS_PYPI_TOKEN }}

juicefs/juicefs.py

@@ -7,6 +7,7 @@
 import time
 from ctypes import create_string_buffer
 from io import BytesIO
+from pathlib import Path
 from typing import Dict, Iterator, List, Optional, Tuple, Union
 
 from juicefs.io import FileIO
@@ -149,7 +150,7 @@ def makedirs(
  if code == -errno.EEXIST and exist_ok:
  return 0
  if code == -errno.ENOENT:
- self.makedirs(os.path.dirname(path), mode, exist_ok)
+ self.makedirs(Path(os.path.dirname(path)).as_posix(), mode, exist_ok)
  code = self._lib.mkdir(path, mode)
  check_juicefs_error(code)
 
@@ -547,7 +548,7 @@ def walk(self, top, topdown: bool = True):
  yield top, dirs, files
  # Recurse into sub-directories
  for dirname in dirs:
- new_path = os.path.join(top, dirname)
+ new_path = Path(os.path.join(top, dirname)).as_posix()
  if not self.path.islink(new_path):
  yield from self.walk(new_path, topdown)
  else:

juicefs/libjfs.py

@@ -5,6 +5,7 @@
 import struct
 from ctypes import CDLL
 from io import BytesIO
+from pathlib import Path
 from threading import current_thread
 from typing import Callable, List, Optional
 
@@ -121,7 +122,7 @@ def __repr__(self):
 
  @property
  def path(self):
- return os.path.join(self.root, self.name)
+ return Path(os.path.join(self.root, self.name)).as_posix()
 
  def inode(self):
  return self._stat.st_ino

requirements-dev.txt

@@ -1,4 +1,5 @@
 setuptools
+wheel
 
 pytest
 pytest-cov

tests/conftest.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import sys
 
 import pytest
 
@@ -11,6 +12,11 @@
 META = "/tmp/test-jfs.db"
 META_URL = "sqlite3:https:///tmp/test-jfs.db"
 
+if sys.platform == "win32":
+ BUCKET = os.path.join(__file__, "..", "tmp")
+ META = os.path.join(__file__, "..", "test-jfs.db")
+ META_URL = r"sqlite3:https:///{}".format(os.path.join(__file__, "..", "test-jfs.db"))
+
 
 @pytest.fixture(scope="session")
 def jfs():

tests/test_io.py

@@ -1,5 +1,6 @@
 import io
 import os
+import sys
 
 import pytest
 
@@ -41,6 +42,8 @@ def remove_local(path):
 @pytest.fixture()
 def local_filename():
  path = "/tmp/test.file"
+ if sys.platform == "win32":
+ path = os.path.join(__file__, "..", "tmp", "test.file")
  remove_local(path)
  yield path
  remove_local(path)

tests/test_juicefs.py

@@ -1,7 +1,9 @@
 import os
 import random
 import shutil
+import sys
 import time
+from pathlib import Path
 
 import pytest
 
@@ -196,7 +198,7 @@ def test_mkdir(jfs, dirname):
 
  jfs.chmod(dirname, 0o444) # readonly
  with pytest.raises(PermissionError):
- jfs.mkdir(os.path.join(dirname, "test"))
+ jfs.mkdir(Path(os.path.join(dirname, "test")).as_posix())
 
 
 def test_rmdir(jfs, filename, dirname):
@@ -340,17 +342,20 @@ def test_listdir(jfs, filename, filename2, dirname):
 
 def test_walk(jfs, dirname):
  jfs.mkdir(dirname)
- jfs.create(os.path.join(dirname, "file"))
- jfs.mkdir(os.path.join(dirname, "dir"))
- jfs.symlink(os.path.join(dirname, "file"), os.path.join(dirname, "dir", "link"))
+ jfs.create(Path(os.path.join(dirname, "file")).as_posix())
+ jfs.mkdir(Path(os.path.join(dirname, "dir")).as_posix())
+ jfs.symlink(
+ Path(os.path.join(dirname, "file")).as_posix(),
+ Path(os.path.join(dirname, "dir", "link")).as_posix(),
+ )
 
  assert list(jfs.walk(dirname)) == [
  (dirname, ["dir"], ["file"]),
- (os.path.join(dirname, "dir"), [], ["link"]),
+ (Path(os.path.join(dirname, "dir")).as_posix(), [], ["link"]),
  ]
 
  assert list(jfs.walk(dirname, topdown=False)) == [
- (os.path.join(dirname, "dir"), [], ["link"]),
+ (Path(os.path.join(dirname, "dir")).as_posix(), [], ["link"]),
  (dirname, ["dir"], ["file"]),
  ]
 
@@ -508,6 +513,10 @@ def test_ftruncate(jfs, filename):
  assert jfs.path.getsize(filename) == length2
 
 
+@pytest.mark.skipif(
+ sys.platform == "win32",
+ reason="Windows can't use os.chmod() to change any file mode we want",
+)
 def test_stat_file(jfs, filename, os_filename):
  jfs.create(filename, 0o644)
 
@@ -522,6 +531,10 @@ def test_stat_file(jfs, filename, os_filename):
  assert os.stat(os_filename).st_mode == jfs.stat(filename).st_mode
 
 
+@pytest.mark.skipif(
+ sys.platform == "win32",
+ reason="Windows can't use os.chmod() to change any file mode we want",
+)
 def test_stat_dir(jfs, dirname, os_dirname):
  jfs.mkdir(dirname)
  os.mkdir(os_dirname)
@@ -559,6 +572,10 @@ def test_lstat_link(jfs, filename, filename2):
  assert int(oct(jfs.lstat(filename2).st_mode)[-3:], 8) == 0o644
 
 
+@pytest.mark.skipif(
+ sys.platform == "win32",
+ reason="Windows can't use os.chmod() to change any file mode we want",
+)
 def test_lstat_file(jfs, filename, os_filename):
  jfs.create(filename, 0o644)
 
@@ -573,6 +590,10 @@ def test_lstat_file(jfs, filename, os_filename):
  assert os.lstat(os_filename).st_mode == jfs.lstat(filename).st_mode
 
 
+@pytest.mark.skipif(
+ sys.platform == "win32",
+ reason="Windows can't use os.chmod() to change any file mode we want",
+)
 def test_lstat_dir(jfs, dirname, os_dirname):
  jfs.mkdir(dirname)
  os.mkdir(os_dirname)
@@ -665,6 +686,10 @@ def test_flush(jfs, filename):
  jfs.close(fdw)
 
 
+@pytest.mark.skipif(
+ sys.platform == "win32",
+ reason="jfs.concat() will raise FileNotFoundError on Windows",
+)
 def test_concat(jfs, filename, filename2, filename3):
  jfs.create(filename)
  jfs.create(filename2)
@@ -694,15 +719,15 @@ def test_summary(jfs, dirname):
  assert summary.files == 0
  assert summary.dirs == 1
 
- jfs.mkdir(os.path.join(dirname, "dir"))
- jfs.create(os.path.join(dirname, "dir", "file"))
- jfs.create(os.path.join(dirname, "file"))
+ jfs.mkdir(Path(os.path.join(dirname, "dir")).as_posix())
+ jfs.create(Path(os.path.join(dirname, "dir", "file")).as_posix())
+ jfs.create(Path(os.path.join(dirname, "file")).as_posix())
 
- fdw = jfs.open(os.path.join(dirname, "file"), os.O_WRONLY)
+ fdw = jfs.open(Path(os.path.join(dirname, "file")).as_posix(), os.O_WRONLY)
  jfs.write(fdw, CONTENT)
  jfs.close(fdw)
 
- fdw = jfs.open(os.path.join(dirname, "dir", "file"), os.O_WRONLY)
+ fdw = jfs.open(Path(os.path.join(dirname, "dir", "file")).as_posix(), os.O_WRONLY)
  jfs.write(fdw, CONTENT)
  jfs.close(fdw)