From f99d19c37c35261166aaf096453f54dc482cacd9 Mon Sep 17 00:00:00 2001 From: Haymaker Date: Thu, 16 Mar 2023 15:32:53 -0400 Subject: [PATCH 1/6] S3 class modifications to support path-style S3Compatible --- src/Storage/Device/S3.php | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index a8447b98..fd00fb69 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -6,6 +6,8 @@ use Utopia\Storage\Device; use Utopia\Storage\Storage; +use function ltrim; + class S3 extends Device { const METHOD_GET = 'GET'; @@ -136,6 +138,11 @@ class S3 extends Device */ protected array $amzHeaders; + /** + * @var bool + */ + protected bool $vhost; + /** * S3 Constructor * @@ -155,6 +162,7 @@ public function __construct(string $root, string $accessKey, string $secretKey, $this->root = $root; $this->acl = $acl; $this->amzHeaders = []; + $this->vhost = true; $host = match ($region) { self::CN_NORTH_1, self::CN_NORTH_4, self::CN_NORTHWEST_1 => $bucket.'.s3.'.$region.'.amazonaws.cn', @@ -196,6 +204,14 @@ public function getRoot(): string return $this->root; } + /** + * @return string + */ + public function getBucket(): string + { + return $this->bucket; + } + /** * @param string $filename * @param string|null $prefix @@ -438,6 +454,17 @@ public function delete(string $path, bool $recursive = false): bool return true; } + private function makeUri(): string + { + $base = '/'; + + if(!$this->vhost) { + return $base . $this->getBucket(); + } else { + return $base; + } + } + /** * Get list of objects in the given path. * @@ -448,16 +475,19 @@ public function delete(string $path, bool $recursive = false): bool */ private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '') { - $uri = '/'; - $prefix = ltrim($prefix, '/'); /** S3 specific requirement that prefix should never contain a leading slash */ + $uri = $this->makeUri(); $this->headers['content-type'] = 'text/plain'; $this->headers['content-md5'] = \base64_encode(md5('', true)); $parameters = [ 'list-type' => 2, - 'prefix' => $prefix, 'max-keys' => $maxKeys, ]; + + if($this->vhost) { + $parameters['prefix'] = ltrim($prefix, '/'); /** S3 specific requirement that prefix should never contain a leading slash */ + } + if (! empty($continuationToken)) { $parameters['continuation-token'] = $continuationToken; } @@ -477,8 +507,8 @@ private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = public function deletePath(string $path): bool { $path = $this->getRoot().'/'.$path; + $uri = $this->makeUri(); - $uri = '/'; $continuationToken = ''; do { $objects = $this->listObjects($path, continuationToken: $continuationToken); From e69f84d96b8b271e1bd82dd0f0848fbbd36de280 Mon Sep 17 00:00:00 2001 From: Haymaker Date: Thu, 16 Mar 2023 15:40:32 -0400 Subject: [PATCH 2/6] S3 changes post format/lint --- src/Storage/Device/S3.php | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index fd00fb69..87b6a1d8 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -3,11 +3,10 @@ namespace Utopia\Storage\Device; use Exception; +use function ltrim; use Utopia\Storage\Device; use Utopia\Storage\Storage; -use function ltrim; - class S3 extends Device { const METHOD_GET = 'GET'; @@ -458,8 +457,8 @@ private function makeUri(): string { $base = '/'; - if(!$this->vhost) { - return $base . $this->getBucket(); + if (! $this->vhost) { + return $base.$this->getBucket(); } else { return $base; } @@ -484,7 +483,7 @@ private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = 'max-keys' => $maxKeys, ]; - if($this->vhost) { + if ($this->vhost) { $parameters['prefix'] = ltrim($prefix, '/'); /** S3 specific requirement that prefix should never contain a leading slash */ } From 42f78d4311a72ffe3dd13bb6b7c9ab2f4441039a Mon Sep 17 00:00:00 2001 From: Haymaker Date: Thu, 16 Mar 2023 19:33:42 -0400 Subject: [PATCH 3/6] S3 compatible storage --- src/Storage/Device/S3Compatible.php | 84 +++++++++++++++++++++++++++++ src/Storage/Storage.php | 2 + 2 files changed, 86 insertions(+) create mode 100644 src/Storage/Device/S3Compatible.php diff --git a/src/Storage/Device/S3Compatible.php b/src/Storage/Device/S3Compatible.php new file mode 100644 index 00000000..9bc69973 --- /dev/null +++ b/src/Storage/Device/S3Compatible.php @@ -0,0 +1,84 @@ +endpoint = $endpoint; + $this->vhost = $vhost; + + /** + * Workaround to prevent having to do a refactor of + * the S3 class along with adapter addition. Class only + * supports https for the time being. + * + * There are multiple endpoint styles dependent upon the provider + * used. Examples are: + * :/// + * :/// + * :// + */ + $endpoint = preg_replace('/^https?:\/\//i', '', $endpoint); + $this->headers['host'] = $endpoint; + } + + /** + * @return string + */ + public function getName(): string + { + return 'S3-Compatible Storage'; + } + + /** + * @return string + */ + public function getType(): string + { + return Storage::DEVICE_S3COMPATIBLE; + } + + /** + * @return string + */ + public function getDescription(): string + { + return 'Generic Connector For S3-Compatible Storage Providers'; + } +} diff --git a/src/Storage/Storage.php b/src/Storage/Storage.php index cabd18c8..785591fe 100644 --- a/src/Storage/Storage.php +++ b/src/Storage/Storage.php @@ -21,6 +21,8 @@ class Storage const DEVICE_LINODE = 'linode'; + const DEVICE_S3COMPATIBLE = 's3compatible'; + /** * Devices. * From 4cc792173fc8f1ad31d2a7b8cdf8338afdf07715 Mon Sep 17 00:00:00 2001 From: Haymaker Date: Thu, 16 Mar 2023 19:38:47 -0400 Subject: [PATCH 4/6] Test for S3Compatible plus changes to accomodate vhost or path-style --- tests/Storage/Device/S3CompatibleTest.php | 55 +++++++++++++++++++++++ tests/Storage/S3Base.php | 15 ++++++- 2 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 tests/Storage/Device/S3CompatibleTest.php diff --git a/tests/Storage/Device/S3CompatibleTest.php b/tests/Storage/Device/S3CompatibleTest.php new file mode 100644 index 00000000..675000f4 --- /dev/null +++ b/tests/Storage/Device/S3CompatibleTest.php @@ -0,0 +1,55 @@ +vhost = $vhost; + $key = $_SERVER['S3COMPATIBLE_ACCESS_KEY'] ?? ''; + $secret = $_SERVER['S3COMPATIBLE_SECRET'] ?? ''; + $bucket = $_SERVER['S3COMPATIBLE_BUCKET'] ?? 'appwrite-test-bucket'; + $region = $_SERVER['S3COMPATIBLE_REGION'] ?? ''; + + if ($vhost) { + $this->root = $root; + } else { + $this->root = $bucket.$root; + } + + $this->object = new S3Compatible($endpoint, $root, $key, $secret, $bucket, $vhost, $region); + } + + /** + * @return string + */ + public function getAdapterName(): string + { + return $this->object->getName(); + } + + /** + * @return string + */ + public function getAdapterType(): string + { + return $this->object->getType(); + } + + /** + * @return string + */ + public function getAdapterDescription(): string + { + return $this->object->getDescription(); + } +} diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index cf17d915..4544d6ad 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -29,6 +29,11 @@ abstract protected function getAdapterDescription(): string; */ protected $root = '/root'; + /** + * @var bool + */ + protected $vhost = true; + public function setUp(): void { $this->init(); @@ -138,12 +143,18 @@ public function testXMLUpload() public function testDeletePath() { + if ($this->vhost) { + $dpParam = 'bucket'; + } else { + $dpParam = ''; + } + // Test Single Object $path = $this->object->getPath('text-for-delete-path.txt'); $path = str_ireplace($this->object->getRoot(), $this->object->getRoot().DIRECTORY_SEPARATOR.'bucket', $path); $this->assertEquals(true, $this->object->write($path, 'Hello World', 'text/plain')); $this->assertEquals(true, $this->object->exists($path)); - $this->assertEquals(true, $this->object->deletePath('bucket')); + $this->assertEquals(true, $this->object->deletePath($dpParam)); $this->assertEquals(false, $this->object->exists($path)); // Test Multiple Objects @@ -157,7 +168,7 @@ public function testDeletePath() $this->assertEquals(true, $this->object->write($path2, 'Hello World', 'text/plain')); $this->assertEquals(true, $this->object->exists($path2)); - $this->assertEquals(true, $this->object->deletePath('bucket')); + $this->assertEquals(true, $this->object->deletePath($dpParam)); $this->assertEquals(false, $this->object->exists($path)); $this->assertEquals(false, $this->object->exists($path2)); } From 5ea37192be329300ed76536b5e9ee0f5f4c5cadd Mon Sep 17 00:00:00 2001 From: Haymaker Date: Thu, 16 Mar 2023 19:39:15 -0400 Subject: [PATCH 5/6] follow non-inversion best practice --- src/Storage/Device/S3.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 87b6a1d8..75e057ba 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -457,10 +457,10 @@ private function makeUri(): string { $base = '/'; - if (! $this->vhost) { - return $base.$this->getBucket(); - } else { + if ($this->vhost) { return $base; + } else { + return $base.$this->getBucket(); } } From 890b8c0be888d3ff8bf5215304098d52c2028669 Mon Sep 17 00:00:00 2001 From: Haymaker Date: Mon, 5 Jun 2023 13:46:40 -0400 Subject: [PATCH 6/6] remove use function statements, remove strict_types statements --- src/Storage/Device/S3.php | 3 +-- src/Storage/Device/S3Compatible.php | 5 +---- tests/Storage/Device/S3CompatibleTest.php | 2 -- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 75e057ba..19cdd99c 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -3,7 +3,6 @@ namespace Utopia\Storage\Device; use Exception; -use function ltrim; use Utopia\Storage\Device; use Utopia\Storage\Storage; @@ -484,7 +483,7 @@ private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = ]; if ($this->vhost) { - $parameters['prefix'] = ltrim($prefix, '/'); /** S3 specific requirement that prefix should never contain a leading slash */ + $parameters['prefix'] = \ltrim($prefix, '/'); /** S3 specific requirement that prefix should never contain a leading slash */ } if (! empty($continuationToken)) { diff --git a/src/Storage/Device/S3Compatible.php b/src/Storage/Device/S3Compatible.php index 9bc69973..5c3f0478 100644 --- a/src/Storage/Device/S3Compatible.php +++ b/src/Storage/Device/S3Compatible.php @@ -1,10 +1,7 @@ :/// * :// */ - $endpoint = preg_replace('/^https?:\/\//i', '', $endpoint); + $endpoint = \preg_replace('/^https?:\/\//i', '', $endpoint); $this->headers['host'] = $endpoint; } diff --git a/tests/Storage/Device/S3CompatibleTest.php b/tests/Storage/Device/S3CompatibleTest.php index 675000f4..1006f37a 100644 --- a/tests/Storage/Device/S3CompatibleTest.php +++ b/tests/Storage/Device/S3CompatibleTest.php @@ -1,7 +1,5 @@