filestore: Allow customising the storage path (#1083)

* filestore: Allow customising the destination path

* Compatible with uploads from previous version

* Add todo about documentation

* More comments

* Documentation

docs/_advanced-topics/hooks.md

@@ -82,7 +82,6 @@ Below you can find an annotated, JSON-ish encoded example of a hook request:
  "PartialUploads": null,
  // Storage contains information about where the upload is stored. The exact values
  // depend on the storage that is used and are not available in the pre-create hook.
- // This example belongs to the file store. 
  "Storage": {
  // For example, the filestore supplies the absolute file path:
  "Type": "filestore",
@@ -190,6 +189,28 @@ Below you can find an annotated, JSON-ish encoded example of a hook response:
  // in the Upload-Metadata header in HEAD responses.
  "MetaData": {
  "my-custom-field": "..."
+ },
+ // Storage can be used to customize the location where the uploaded file (aka the binary
+ // file is saved). The exact behavior depends on the storage that is used. Please note
+ // that this only influences the location of the binary file. tusd will still create an
+ // info file whose location is derived from the upload ID and can not be customized using
+ // this ChangeFileInfo.Storage property, but only using ChangeFileInfo.ID.
+ //
+ // The location can contain forward slashes (/) to store uploads in a hierarchical structure,
+ // such as nested directories.
+ //
+ // Similar to ChangeFileInfo.ID, tusd will not check whether a file is already saved under
+ // this location and might overwrite it. It is the hooks responsibility to ensure that
+ // the location is save to use. A good approach is to embed a random part (e.g. a UUID) in
+ // the location.
+ "Storage": {
+ // When the filestore is used, the Path property defines where the uploaded file is saved.
+ // The path may be absolute or relative, and point towards a location outside of the directory
+ // defined using the `-dir` flag. If it's relative, the path will be resolved relative to `-dir`.
+ "Path": "./upload-e7a036dc-33f4-451f-9520-49032b87e952/presentation.pdf"
+
+ // Other storages, such as S3Store, GCSStore, and AzureStore, do not support the Storage
+ // property yet.
  }
  },
 

docs/_storage-backends/local-disk.md

@@ -17,6 +17,25 @@ When a new upload is created, for example with the upload ID `abcdef123`, tusd c
 - `./uploads/abcdef123` to hold the file content that is uploaded
 - `./uploads/abcdef123.info` to hold [meta information about the upload]({{ site.baseurl }}/storage-backends/overview/#storage-format)
 
+## Custom storage location
+
+The locations of the two files mentioned above can be fully customized using the [pre-create hook](({ site.baseurl }}/advanced-topics/hooks/). The location of the `.info` file is derived from the upload ID, which can be customized by the pre-create hook using the [`ChangeFileInfo.ID` setting]({ site.baseurl }}/advanced-topics/hooks/#hook-requests-and-responses). Similarly, the location where the file content is saved is by default derived from the upload ID, but can be fully customized using the [`ChangeFileInfo.Storage.Path` setting]({ site.baseurl }}/advanced-topics/hooks/#hook-requests-and-responses).
+
+For example, if the pre-create hook returns the following hook response, an upload with ID `project-123/abc` is created, the info file is saved at `./uploads/project-123/abc.info`, and the file content is saved at `./uploads/project-123/abc/presentation.pdf`:
+
+```js
+{
+ "ChangeFileInfo": {
+ "ID": "project-123/abc",
+ "Storage": {
+ "Path": "project-123/abc/presentation.pdf"
+ }
+ },
+}
+```
+
+If the defined path is relative, it will be resolved from the directory defined using `-dir`.
+
 ## Issues with NFS and shared folders
 
 Tusd maintains [upload locks]({{ site.baseurl }}/advanced-topics/locks/) on disk to ensure exclusive access to uploads and prevent data corruption. These disk-based locks utilize hard links, which might not be supported by older NFS versions or when a folder is shared in a VM using VirtualBox or Vagrant. In these cases, you might get errors like this:

pkg/filestore/filestore.go

@@ -56,7 +56,17 @@ func (store FileStore) NewUpload(ctx context.Context, info handler.FileInfo) (ha
  if info.ID == "" {
  info.ID = uid.Uid()
  }
- binPath := store.binPath(info.ID)
+
+ // The .info file's location can directly be deduced from the upload ID
+ infoPath := store.infoPath(info.ID)
+ // The binary file's location might be modified by the pre-create hook.
+ var binPath string
+ if info.Storage != nil && info.Storage["Path"] != "" {
+ binPath = filepath.Join(store.Path, info.Storage["Path"])
+ } else {
+ binPath = store.defaultBinPath(info.ID)
+ }
+
  info.Storage = map[string]string{
  "Type": "filestore",
  "Path": binPath,
@@ -69,7 +79,7 @@ func (store FileStore) NewUpload(ctx context.Context, info handler.FileInfo) (ha
 
  upload := &fileUpload{
  info: info,
- infoPath: store.infoPath(info.ID),
+ infoPath: infoPath,
  binPath: binPath,
  }
 
@@ -82,21 +92,32 @@ func (store FileStore) NewUpload(ctx context.Context, info handler.FileInfo) (ha
 }
 
 func (store FileStore) GetUpload(ctx context.Context, id string) (handler.Upload, error) {
- info := handler.FileInfo{}
- data, err := os.ReadFile(store.infoPath(id))
+ infoPath := store.infoPath(id)
+ data, err := os.ReadFile(infoPath)
  if err != nil {
  if os.IsNotExist(err) {
  // Interpret os.ErrNotExist as 404 Not Found
  err = handler.ErrNotFound
  }
  return nil, err
  }
+ var info handler.FileInfo
  if err := json.Unmarshal(data, &info); err != nil {
  return nil, err
  }
 
- binPath := store.binPath(id)
- infoPath := store.infoPath(id)
+ // If the info file contains a custom path to the binary file, we use that. If not, we
+ // fall back to the default value (although the Path property should always be set in recent
+ // tusd versions).
+ var binPath string
+ if info.Storage != nil && info.Storage["Path"] != "" {
+ // No filepath.Join here because the joining already happened in NewUpload. Duplicate joining
+ // with relative paths lead to incorrect paths
+ binPath = info.Storage["Path"]
+ } else {
+ binPath = store.defaultBinPath(info.ID)
+ }
+
  stat, err := os.Stat(binPath)
  if err != nil {
  if os.IsNotExist(err) {
@@ -127,8 +148,9 @@ func (store FileStore) AsConcatableUpload(upload handler.Upload) handler.Concata
  return upload.(*fileUpload)
 }
 
-// binPath returns the path to the file storing the binary data.
-func (store FileStore) binPath(id string) string {
+// defaultBinPath returns the path to the file storing the binary data, if it is
+// not customized using the pre-create hook.
+func (store FileStore) defaultBinPath(id string) string {
  return filepath.Join(store.Path, id)
 }
 

pkg/filestore/filestore_test.go

@@ -245,3 +245,71 @@ func TestDeclareLength(t *testing.T) {
  a.EqualValues(100, updatedInfo.Size)
  a.Equal(false, updatedInfo.SizeIsDeferred)
 }
+
+// TestCustomPath tests whether the upload's destination can be customized.
+func TestCustomPath(t *testing.T) {
+ a := assert.New(t)
+
+ tmp, err := os.MkdirTemp("", "tusd-filestore-")
+ a.NoError(err)
+
+ store := FileStore{tmp}
+ ctx := context.Background()
+
+ // Create new upload
+ upload, err := store.NewUpload(ctx, handler.FileInfo{
+ ID: "folder1/info",
+ Size: 42,
+ Storage: map[string]string{
+ "Path": "./folder2/bin",
+ },
+ })
+ a.NoError(err)
+ a.NotEqual(nil, upload)
+
+ // Check info without writing
+ info, err := upload.GetInfo(ctx)
+ a.NoError(err)
+ a.EqualValues(42, info.Size)
+ a.EqualValues(0, info.Offset)
+ a.Equal(2, len(info.Storage))
+ a.Equal("filestore", info.Storage["Type"])
+ a.Equal(filepath.Join(tmp, "./folder2/bin"), info.Storage["Path"])
+
+ // Write data to upload
+ bytesWritten, err := upload.WriteChunk(ctx, 0, strings.NewReader("hello world"))
+ a.NoError(err)
+ a.EqualValues(len("hello world"), bytesWritten)
+
+ // Check new offset
+ info, err = upload.GetInfo(ctx)
+ a.NoError(err)
+ a.EqualValues(42, info.Size)
+ a.EqualValues(11, info.Offset)
+
+ // Read content
+ reader, err := upload.GetReader(ctx)
+ a.NoError(err)
+
+ content, err := io.ReadAll(reader)
+ a.NoError(err)
+ a.Equal("hello world", string(content))
+ reader.(io.Closer).Close()
+
+ // Check that the output file and info file exist on disk
+ statInfo, err := os.Stat(filepath.Join(tmp, "folder2/bin"))
+ a.NoError(err)
+ a.True(statInfo.Mode().IsRegular())
+ a.EqualValues(11, statInfo.Size())
+ statInfo, err = os.Stat(filepath.Join(tmp, "folder1/info.info"))
+ a.NoError(err)
+ a.True(statInfo.Mode().IsRegular())
+
+ // Terminate upload
+ a.NoError(store.AsTerminatableUpload(upload).Terminate(ctx))
+
+ // Test if upload is deleted
+ upload, err = store.GetUpload(ctx, info.ID)
+ a.Equal(nil, upload)
+ a.Equal(handler.ErrNotFound, err)
+}