From 8a8b5388819c5ecf2c98ed6a856ff1fba61409bc Mon Sep 17 00:00:00 2001 From: Marius Date: Fri, 21 Jun 2024 09:51:01 +0200 Subject: [PATCH] filestore: Allow customising the storage path (#1083) * filestore: Allow customising the destination path * Compatible with uploads from previous version * Add todo about documentation * More comments * Documentation --- docs/_advanced-topics/hooks.md | 23 +++++++++- docs/_storage-backends/local-disk.md | 19 ++++++++ pkg/filestore/filestore.go | 38 ++++++++++++---- pkg/filestore/filestore_test.go | 68 ++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 9 deletions(-) diff --git a/docs/_advanced-topics/hooks.md b/docs/_advanced-topics/hooks.md index 09e44bbe..968b0e5d 100644 --- a/docs/_advanced-topics/hooks.md +++ b/docs/_advanced-topics/hooks.md @@ -82,7 +82,6 @@ Below you can find an annotated, JSON-ish encoded example of a hook request: "PartialUploads": null, // Storage contains information about where the upload is stored. The exact values // depend on the storage that is used and are not available in the pre-create hook. - // This example belongs to the file store. "Storage": { // For example, the filestore supplies the absolute file path: "Type": "filestore", @@ -190,6 +189,28 @@ Below you can find an annotated, JSON-ish encoded example of a hook response: // in the Upload-Metadata header in HEAD responses. "MetaData": { "my-custom-field": "..." + }, + // Storage can be used to customize the location where the uploaded file (aka the binary + // file is saved). The exact behavior depends on the storage that is used. Please note + // that this only influences the location of the binary file. tusd will still create an + // info file whose location is derived from the upload ID and can not be customized using + // this ChangeFileInfo.Storage property, but only using ChangeFileInfo.ID. + // + // The location can contain forward slashes (/) to store uploads in a hierarchical structure, + // such as nested directories. + // + // Similar to ChangeFileInfo.ID, tusd will not check whether a file is already saved under + // this location and might overwrite it. It is the hooks responsibility to ensure that + // the location is save to use. A good approach is to embed a random part (e.g. a UUID) in + // the location. + "Storage": { + // When the filestore is used, the Path property defines where the uploaded file is saved. + // The path may be absolute or relative, and point towards a location outside of the directory + // defined using the `-dir` flag. If it's relative, the path will be resolved relative to `-dir`. + "Path": "./upload-e7a036dc-33f4-451f-9520-49032b87e952/presentation.pdf" + + // Other storages, such as S3Store, GCSStore, and AzureStore, do not support the Storage + // property yet. } }, diff --git a/docs/_storage-backends/local-disk.md b/docs/_storage-backends/local-disk.md index fb3fdf68..ef796a0c 100644 --- a/docs/_storage-backends/local-disk.md +++ b/docs/_storage-backends/local-disk.md @@ -17,6 +17,25 @@ When a new upload is created, for example with the upload ID `abcdef123`, tusd c - `./uploads/abcdef123` to hold the file content that is uploaded - `./uploads/abcdef123.info` to hold [meta information about the upload]({{ site.baseurl }}/storage-backends/overview/#storage-format) +## Custom storage location + +The locations of the two files mentioned above can be fully customized using the [pre-create hook](({ site.baseurl }}/advanced-topics/hooks/). The location of the `.info` file is derived from the upload ID, which can be customized by the pre-create hook using the [`ChangeFileInfo.ID` setting]({ site.baseurl }}/advanced-topics/hooks/#hook-requests-and-responses). Similarly, the location where the file content is saved is by default derived from the upload ID, but can be fully customized using the [`ChangeFileInfo.Storage.Path` setting]({ site.baseurl }}/advanced-topics/hooks/#hook-requests-and-responses). + +For example, if the pre-create hook returns the following hook response, an upload with ID `project-123/abc` is created, the info file is saved at `./uploads/project-123/abc.info`, and the file content is saved at `./uploads/project-123/abc/presentation.pdf`: + +```js +{ + "ChangeFileInfo": { + "ID": "project-123/abc", + "Storage": { + "Path": "project-123/abc/presentation.pdf" + } + }, +} +``` + +If the defined path is relative, it will be resolved from the directory defined using `-dir`. + ## Issues with NFS and shared folders Tusd maintains [upload locks]({{ site.baseurl }}/advanced-topics/locks/) on disk to ensure exclusive access to uploads and prevent data corruption. These disk-based locks utilize hard links, which might not be supported by older NFS versions or when a folder is shared in a VM using VirtualBox or Vagrant. In these cases, you might get errors like this: diff --git a/pkg/filestore/filestore.go b/pkg/filestore/filestore.go index 5c03f483..6f9ab522 100644 --- a/pkg/filestore/filestore.go +++ b/pkg/filestore/filestore.go @@ -56,7 +56,17 @@ func (store FileStore) NewUpload(ctx context.Context, info handler.FileInfo) (ha if info.ID == "" { info.ID = uid.Uid() } - binPath := store.binPath(info.ID) + + // The .info file's location can directly be deduced from the upload ID + infoPath := store.infoPath(info.ID) + // The binary file's location might be modified by the pre-create hook. + var binPath string + if info.Storage != nil && info.Storage["Path"] != "" { + binPath = filepath.Join(store.Path, info.Storage["Path"]) + } else { + binPath = store.defaultBinPath(info.ID) + } + info.Storage = map[string]string{ "Type": "filestore", "Path": binPath, @@ -69,7 +79,7 @@ func (store FileStore) NewUpload(ctx context.Context, info handler.FileInfo) (ha upload := &fileUpload{ info: info, - infoPath: store.infoPath(info.ID), + infoPath: infoPath, binPath: binPath, } @@ -82,8 +92,8 @@ func (store FileStore) NewUpload(ctx context.Context, info handler.FileInfo) (ha } func (store FileStore) GetUpload(ctx context.Context, id string) (handler.Upload, error) { - info := handler.FileInfo{} - data, err := os.ReadFile(store.infoPath(id)) + infoPath := store.infoPath(id) + data, err := os.ReadFile(infoPath) if err != nil { if os.IsNotExist(err) { // Interpret os.ErrNotExist as 404 Not Found @@ -91,12 +101,23 @@ func (store FileStore) GetUpload(ctx context.Context, id string) (handler.Upload } return nil, err } + var info handler.FileInfo if err := json.Unmarshal(data, &info); err != nil { return nil, err } - binPath := store.binPath(id) - infoPath := store.infoPath(id) + // If the info file contains a custom path to the binary file, we use that. If not, we + // fall back to the default value (although the Path property should always be set in recent + // tusd versions). + var binPath string + if info.Storage != nil && info.Storage["Path"] != "" { + // No filepath.Join here because the joining already happened in NewUpload. Duplicate joining + // with relative paths lead to incorrect paths + binPath = info.Storage["Path"] + } else { + binPath = store.defaultBinPath(info.ID) + } + stat, err := os.Stat(binPath) if err != nil { if os.IsNotExist(err) { @@ -127,8 +148,9 @@ func (store FileStore) AsConcatableUpload(upload handler.Upload) handler.Concata return upload.(*fileUpload) } -// binPath returns the path to the file storing the binary data. -func (store FileStore) binPath(id string) string { +// defaultBinPath returns the path to the file storing the binary data, if it is +// not customized using the pre-create hook. +func (store FileStore) defaultBinPath(id string) string { return filepath.Join(store.Path, id) } diff --git a/pkg/filestore/filestore_test.go b/pkg/filestore/filestore_test.go index 9e0fc2ae..d25ed3a3 100644 --- a/pkg/filestore/filestore_test.go +++ b/pkg/filestore/filestore_test.go @@ -245,3 +245,71 @@ func TestDeclareLength(t *testing.T) { a.EqualValues(100, updatedInfo.Size) a.Equal(false, updatedInfo.SizeIsDeferred) } + +// TestCustomPath tests whether the upload's destination can be customized. +func TestCustomPath(t *testing.T) { + a := assert.New(t) + + tmp, err := os.MkdirTemp("", "tusd-filestore-") + a.NoError(err) + + store := FileStore{tmp} + ctx := context.Background() + + // Create new upload + upload, err := store.NewUpload(ctx, handler.FileInfo{ + ID: "folder1/info", + Size: 42, + Storage: map[string]string{ + "Path": "./folder2/bin", + }, + }) + a.NoError(err) + a.NotEqual(nil, upload) + + // Check info without writing + info, err := upload.GetInfo(ctx) + a.NoError(err) + a.EqualValues(42, info.Size) + a.EqualValues(0, info.Offset) + a.Equal(2, len(info.Storage)) + a.Equal("filestore", info.Storage["Type"]) + a.Equal(filepath.Join(tmp, "./folder2/bin"), info.Storage["Path"]) + + // Write data to upload + bytesWritten, err := upload.WriteChunk(ctx, 0, strings.NewReader("hello world")) + a.NoError(err) + a.EqualValues(len("hello world"), bytesWritten) + + // Check new offset + info, err = upload.GetInfo(ctx) + a.NoError(err) + a.EqualValues(42, info.Size) + a.EqualValues(11, info.Offset) + + // Read content + reader, err := upload.GetReader(ctx) + a.NoError(err) + + content, err := io.ReadAll(reader) + a.NoError(err) + a.Equal("hello world", string(content)) + reader.(io.Closer).Close() + + // Check that the output file and info file exist on disk + statInfo, err := os.Stat(filepath.Join(tmp, "folder2/bin")) + a.NoError(err) + a.True(statInfo.Mode().IsRegular()) + a.EqualValues(11, statInfo.Size()) + statInfo, err = os.Stat(filepath.Join(tmp, "folder1/info.info")) + a.NoError(err) + a.True(statInfo.Mode().IsRegular()) + + // Terminate upload + a.NoError(store.AsTerminatableUpload(upload).Terminate(ctx)) + + // Test if upload is deleted + upload, err = store.GetUpload(ctx, info.ID) + a.Equal(nil, upload) + a.Equal(handler.ErrNotFound, err) +}