diff --git a/Dockerfile b/Dockerfile index 1d82930c1d4..ed02a6b7de7 100755 --- a/Dockerfile +++ b/Dockerfile @@ -64,13 +64,15 @@ RUN mkdir -p /storage/uploads && \ mkdir -p /storage/config && \ mkdir -p /storage/certificates && \ mkdir -p /storage/functions && \ + mkdir -p /storage/syncs && \ mkdir -p /storage/debug && \ chown -Rf www-data.www-data /storage/uploads && chmod -Rf 0755 /storage/uploads && \ chown -Rf www-data.www-data /storage/cache && chmod -Rf 0755 /storage/cache && \ chown -Rf www-data.www-data /storage/config && chmod -Rf 0755 /storage/config && \ chown -Rf www-data.www-data /storage/certificates && chmod -Rf 0755 /storage/certificates && \ chown -Rf www-data.www-data /storage/functions && chmod -Rf 0755 /storage/functions && \ - chown -Rf www-data.www-data /storage/debug && chmod -Rf 0755 /storage/debug + chown -Rf www-data.www-data /storage/debug && chmod -Rf 0755 /storage/debug && \ + chown -Rf www-data.www-data /storage/syncs && chmod -Rf 0755 /storage/syncs # Executables RUN chmod +x /usr/local/bin/doctor && \ @@ -78,6 +80,7 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/maintenance && \ chmod +x /usr/local/bin/migrate && \ chmod +x /usr/local/bin/realtime && \ + chmod +x /usr/local/bin/region-sync && \ chmod +x /usr/local/bin/schedule-functions && \ chmod +x /usr/local/bin/schedule-messages && \ chmod +x /usr/local/bin/sdks && \ @@ -101,7 +104,10 @@ RUN chmod +x /usr/local/bin/doctor && \ chmod +x /usr/local/bin/worker-migrations && \ chmod +x /usr/local/bin/worker-webhooks && \ chmod +x /usr/local/bin/worker-usage && \ - chmod +x /usr/local/bin/worker-usage-dump + chmod +x /usr/local/bin/worker-usage-dump && \ + chmod +x /usr/local/bin/worker-sync-out-aggregation && \ + chmod +x /usr/local/bin/worker-sync-out-delivery && \ + chmod +x /usr/local/bin/worker-sync-in # Letsencrypt Permissions RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/ diff --git a/app/cli.php b/app/cli.php index 87bb711daa4..0bd53c1d84f 100644 --- a/app/cli.php +++ b/app/cli.php @@ -19,6 +19,7 @@ use Utopia\Logger\Log; use Utopia\Platform\Service; use Utopia\Pools\Group; +use Utopia\Queue\Client; use Utopia\Queue\Connection; use Utopia\Registry\Registry; use Utopia\System\System; @@ -165,6 +166,12 @@ CLI::setResource('queueForCertificates', function (Connection $queue) { return new Certificate($queue); }, ['queue']); +CLI::setResource('queueForSyncOutAggregation', function (Connection $queue) { + return new Client('v1-sync-out-aggregation', $queue); +}, ['queue']); +CLI::setResource('queueForSyncOutDelivery', function (Connection $queue) { + return new Client('v1-sync-out-delivery', $queue); +}, ['queue']); CLI::setResource('logError', function (Registry $register) { return function (Throwable $error, string $namespace, string $action) use ($register) { $logger = $register->get('logger'); diff --git a/app/config/collections.php b/app/config/collections.php index 72d126e343b..e6ab10000af 100644 --- a/app/config/collections.php +++ b/app/config/collections.php @@ -4141,6 +4141,91 @@ ], $commonCollections); $consoleCollections = array_merge([ + 'syncs' => [ + '$collection' => ID::custom(Database::METADATA), + '$id' => ID::custom('syncs'), + 'name' => 'Syncs', + 'attributes' => [ + [ + '$id' => ID::custom('sourceRegion'), + 'type' => Database::VAR_STRING, + 'size' => 50, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('destRegion'), + 'type' => Database::VAR_STRING, + 'size' => 50, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('filename'), + 'type' => Database::VAR_STRING, + 'size' => 150, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('logCreatedAt'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => ID::custom('logSentAt'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ], + [ + '$id' => ID::custom('status'), + 'type' => Database::VAR_INTEGER, + 'size' => 256, + 'required' => false, + 'signed' => true, + 'default' => 0, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => ID::custom('attempts'), + 'type' => Database::VAR_INTEGER, + 'size' => 256, + 'required' => false, + 'signed' => true, + 'default' => 0, + 'array' => false, + 'filters' => [], + ], + ], + 'indexes' => [ + [ + '$id' => ID::custom('_key_sourceRegion'), + 'type' => Database::INDEX_KEY, + 'attributes' => ['sourceRegion'], + 'lengths' => [], + 'orders' => [], + ], + ], + ], 'projects' => [ '$collection' => ID::custom(Database::METADATA), '$id' => ID::custom('projects'), diff --git a/app/config/regions.php b/app/config/regions.php index b40667ab5e0..f9a03b404d9 100644 --- a/app/config/regions.php +++ b/app/config/regions.php @@ -7,6 +7,8 @@ 'disabled' => false, 'flag' => 'de', 'default' => true, + 'domain' => '172.29.0.1' + ], 'fra' => [ '$id' => 'fra', @@ -14,6 +16,7 @@ 'disabled' => false, 'flag' => 'de', 'default' => true, + 'domain' => '172.29.0.1' ], 'nyc' => [ '$id' => 'nyc', @@ -21,6 +24,7 @@ 'disabled' => true, 'flag' => 'us', 'default' => true, + 'domain' => '172.29.0.1' ], 'sfo' => [ '$id' => 'sfo', @@ -28,6 +32,7 @@ 'disabled' => true, 'flag' => 'us', 'default' => true, + 'domain' => '172.29.0.1' ], 'blr' => [ '$id' => 'blr', @@ -35,6 +40,7 @@ 'disabled' => true, 'flag' => 'in', 'default' => true, + 'domain' => '172.29.0.1' ], 'lon' => [ '$id' => 'lon', @@ -42,6 +48,7 @@ 'disabled' => true, 'flag' => 'gb', 'default' => true, + 'domain' => '172.29.0.1' ], 'ams' => [ '$id' => 'ams', @@ -49,6 +56,7 @@ 'disabled' => true, 'flag' => 'nl', 'default' => true, + 'domain' => '172.29.0.1' ], 'sgp' => [ '$id' => 'sgp', @@ -56,6 +64,7 @@ 'disabled' => true, 'flag' => 'sg', 'default' => true, + 'domain' => '172.29.0.1' ], 'tor' => [ '$id' => 'tor', @@ -63,6 +72,7 @@ 'disabled' => true, 'flag' => 'ca', 'default' => true, + 'domain' => '172.29.0.1' ], 'syd' => [ '$id' => 'syd', @@ -70,5 +80,6 @@ 'disabled' => true, 'flag' => 'au', 'default' => true, + 'domain' => '172.29.0.1' ], ]; diff --git a/app/config/services.php b/app/config/services.php index c4fb98c59a5..59f304a2c85 100644 --- a/app/config/services.php +++ b/app/config/services.php @@ -263,5 +263,18 @@ 'tests' => true, 'optional' => true, 'icon' => '/images/services/messaging.png', - ] + ], + 'syncs' => [ + 'key' => 'region', + 'name' => 'region', + 'subtitle' => 'Appwrite\'s region resources sync Endpoint', + 'description' => 'region resources sync Endpoint', + 'controller' => 'api/region.php', + 'sdk' => false, + 'docs' => false, + 'docsUrl' => '', + 'tests' => true, + 'optional' => false, + 'icon' => '', + ], ]; diff --git a/app/controllers/api/region.php b/app/controllers/api/region.php new file mode 100644 index 00000000000..28c43780234 --- /dev/null +++ b/app/controllers/api/region.php @@ -0,0 +1,63 @@ +groups(['region']) + ->inject('request') + ->action(function (Request $request) { + + $token = $request->getHeader('authorization'); + $token = str_replace(["Bearer"," "], "", $token); + $jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 600, 10); + try { + $payload = $jwt->decode($token); + } catch (JWTException $error) { + throw new Exception(Exception::USER_JWT_INVALID, 'Failed to verify JWT. ' . $error->getMessage()); + } + }); + +App::post('/v1/region/sync') + ->desc('Purge cache keys') + ->groups(['region']) + ->label('scope', 'public') + ->param('keys', '', new ArrayList(new Text(9012), 600), 'Cache keys. an array containing alphanumerical cache keys') + ->inject('request') + ->inject('response') + ->inject('queueForEdgeSyncIn') + ->action(function (array $keys, Request $request, Response $response, Client $queueForEdgeSyncIn) { + + if (empty($keys)) { + throw new Exception(Exception::KEY_NOT_FOUND); + } + + $originEdgeUrl = $request->getHeader('origin-region-url'); + $time = DateTime::now(); + + var_dump('[' . $time . '] incoming request from:' . $originEdgeUrl); + var_dump($keys); + + foreach ($keys as $parts) { + $key = json_decode($parts); + $queueForEdgeSyncIn + ->enqueue([ + 'type' => $key->type, + 'key' => $key->key + ]); + } + + $response->dynamic(new Document([ + 'keys' => $keys + ]), Response::MODEL_REGION_SYNC); + }); diff --git a/app/controllers/shared/api.php b/app/controllers/shared/api.php index bb9408e692c..321724ff46c 100644 --- a/app/controllers/shared/api.php +++ b/app/controllers/shared/api.php @@ -539,7 +539,8 @@ ->inject('queueForFunctions') ->inject('mode') ->inject('dbForConsole') - ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsole) use ($parseLabel) { + ->inject('queueForSyncOutAggregation') + ->action(function (App $utopia, Request $request, Response $response, Document $project, Document $user, Event $queueForEvents, Audit $queueForAudits, Usage $queueForUsage, Delete $queueForDeletes, EventDatabase $queueForDatabase, Build $queueForBuilds, Messaging $queueForMessaging, Database $dbForProject, Func $queueForFunctions, string $mode, Database $dbForConsoleClient, $queueForSyncOutAggregation) use ($parseLabel) { $responsePayload = $response->getPayload(); @@ -595,9 +596,32 @@ 'userId' => $queueForEvents->getParam('userId') ] ); + //Sync with other regions + $queueForSyncOutAggregation->enqueue([ + 'type' => 'realtime', + 'key' => [ + 'projectId' => $target['projectId'] ?? $project->getId(), + 'payload' => $queueForEvents->getPayload(), + 'events' => $allEvents, + 'channels' => $target['channels'], + 'roles' => $target['roles'], + 'options' => [ + 'permissionsChanged' => $target['permissionsChanged'], + 'userId' => $queueForEvents->getParam('userId') + ] + ] + ]); } } + // $queueForSyncOutAggregation->enqueue([ + // 'type' => 'certificate', + // 'key' => [ + // 'domain' => 'appwrite.io', + // 'contents' => base64_encode(file_get_contents(APP_STORAGE_CERTIFICATES . '/appwrite.io.tar.gz')), + // ] + // ]); + $route = $utopia->getRoute(); $requestParams = $route->getParamsValues(); diff --git a/app/init.php b/app/init.php index a86156c7501..a9b019bcb79 100644 --- a/app/init.php +++ b/app/init.php @@ -67,6 +67,7 @@ use Utopia\Pools\Group; use Utopia\Pools\Pool; use Utopia\Queue; +use Utopia\Queue\Client; use Utopia\Queue\Connection; use Utopia\Registry\Registry; use Utopia\Storage\Device; @@ -129,6 +130,7 @@ const APP_STORAGE_CACHE = '/storage/cache'; const APP_STORAGE_CERTIFICATES = '/storage/certificates'; const APP_STORAGE_CONFIG = '/storage/config'; +const APP_STORAGE_SYNCS = '/storage/syncs'; const APP_STORAGE_READ_BUFFER = 20 * (1000 * 1000); //20MB other names `APP_STORAGE_MEMORY_LIMIT`, `APP_STORAGE_MEMORY_BUFFER`, `APP_STORAGE_READ_LIMIT`, `APP_STORAGE_BUFFER_LIMIT` const APP_SOCIAL_TWITTER = 'https://twitter.com/appwrite'; const APP_SOCIAL_TWITTER_HANDLE = 'appwrite'; @@ -1035,6 +1037,15 @@ function (mixed $value) { App::setResource('queue', function (Group $pools) { return $pools->get('queue')->pop()->getResource(); }, ['pools']); +App::setResource('queueForSyncOutAggregation', function (Connection $queue) { + return new Client('v1-sync-out-aggregation', $queue); +}, ['queue']); +App::setResource('queueForSyncOutDelivery', function (Connection $queue) { + return new Client('v1-sync-out-delivery', $queue); +}, ['queue']); +App::setResource('queueForEdgeSyncIn', function (Connection $queue) { + return new Client('v1-sync-in', $queue); +}, ['queue']); App::setResource('queueForMessaging', function (Connection $queue) { return new Messaging($queue); }, ['queue']); @@ -1420,7 +1431,7 @@ function (mixed $value) { }; }, ['pools', 'dbForConsole', 'cache']); -App::setResource('cache', function (Group $pools) { +App::setResource('cache', function (Group $pools, Client $queueForSyncOutAggregation) { $list = Config::getParam('pools-cache', []); $adapters = []; @@ -1431,9 +1442,26 @@ function (mixed $value) { ->getResource() ; } + $cache = new Cache(new Sharding($adapters)); - return new Cache(new Sharding($adapters)); -}, ['pools']); + $cache->on(cache::EVENT_SAVE, function ($key) use ($queueForSyncOutAggregation) { + $queueForSyncOutAggregation + ->enqueue([ + 'type' => 'cache', + 'key' => $key + ]); + }); + + $cache->on(cache::EVENT_PURGE, function ($key) use ($queueForSyncOutAggregation) { + $queueForSyncOutAggregation + ->enqueue([ + 'type' => 'cache', + 'key' => $key + ]); + }); + + return $cache; +}, ['pools', 'queueForSyncOutAggregation']); App::setResource('deviceForLocal', function () { return new Local(); diff --git a/app/worker.php b/app/worker.php index bca92c433fe..87b82363776 100644 --- a/app/worker.php +++ b/app/worker.php @@ -2,12 +2,6 @@ require_once __DIR__ . '/init.php'; -use Appwrite\Event\Audit; -use Appwrite\Event\Build; -use Appwrite\Event\Certificate; -use Appwrite\Event\Database as EventDatabase; -use Appwrite\Event\Delete; -use Appwrite\Event\Event; use Appwrite\Event\Func; use Appwrite\Event\Mail; use Appwrite\Event\Messaging; @@ -30,6 +24,7 @@ use Utopia\Logger\Logger; use Utopia\Platform\Service; use Utopia\Pools\Group; +use Utopia\Queue\Client; use Utopia\Queue\Connection; use Utopia\Queue\Message; use Utopia\Queue\Server; @@ -194,6 +189,12 @@ return new Cache(new Sharding($adapters)); }, ['register']); +Server::setResource('queueForSyncOutAggregation', function (Connection $queue) { + return new Client('v1-sync-out-aggregation', $queue); +}, ['queue']); +Server::setResource('queueForSyncOutDelivery', function (Connection $queue) { + return new Client('v1-sync-out-delivery', $queue); +}, ['queue']); Server::setResource('log', fn () => new Log()); Server::setResource('queueForUsage', function (Connection $queue) { diff --git a/bin/benchmark b/bin/benchmark new file mode 100644 index 00000000000..de9bee100c2 --- /dev/null +++ b/bin/benchmark @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +/usr/src/code/vendor/bin/phpbench run --config /usr/src/code/phpbench.json --report appwrite $@ \ No newline at end of file diff --git a/bin/region-sync b/bin/region-sync new file mode 100644 index 00000000000..c1cbc8ba9d3 --- /dev/null +++ b/bin/region-sync @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/cli.php region-sync $@ \ No newline at end of file diff --git a/bin/worker-sync-in b/bin/worker-sync-in new file mode 100644 index 00000000000..0c2bff58857 --- /dev/null +++ b/bin/worker-sync-in @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php sync-in $@ \ No newline at end of file diff --git a/bin/worker-sync-out-aggregation b/bin/worker-sync-out-aggregation new file mode 100644 index 00000000000..b01c4274685 --- /dev/null +++ b/bin/worker-sync-out-aggregation @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php sync-out-aggregation $@ diff --git a/bin/worker-sync-out-delivery b/bin/worker-sync-out-delivery new file mode 100644 index 00000000000..d3ceec49a18 --- /dev/null +++ b/bin/worker-sync-out-delivery @@ -0,0 +1,3 @@ +#!/bin/sh + +php /usr/src/code/app/worker.php sync-out-delivery $@ diff --git a/bla.tar.gz b/bla.tar.gz new file mode 100644 index 00000000000..e197de043ea Binary files /dev/null and b/bla.tar.gz differ diff --git a/composer.json b/composer.json index 192b3118221..790f396d877 100644 --- a/composer.json +++ b/composer.json @@ -47,7 +47,7 @@ "utopia-php/abuse": "0.37.*", "utopia-php/analytics": "0.10.*", "utopia-php/audit": "0.39.*", - "utopia-php/cache": "0.10.*", + "utopia-php/cache": "dev-feat-redis-sync as 0.10.0", "utopia-php/cli": "0.15.*", "utopia-php/config": "0.2.*", "utopia-php/database": "0.49.*", diff --git a/composer.lock b/composer.lock index aee6c86d535..7973fa57ab2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e002600539435ca8eaaace6e73b4004d", + "content-hash": "eb71c276dc4c579c30d72b8ec4ff9f08", "packages": [ { "name": "adhocore/jwt", @@ -1569,16 +1569,16 @@ }, { "name": "utopia-php/cache", - "version": "0.10.0", + "version": "dev-feat-redis-sync", "source": { "type": "git", "url": "https://github.com/utopia-php/cache.git", - "reference": "313bcdfbb166f75c2c205a59d1467cead63a9626" + "reference": "e0a88c4a6568a84d40482ebd880e73b040fc0b79" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/cache/zipball/313bcdfbb166f75c2c205a59d1467cead63a9626", - "reference": "313bcdfbb166f75c2c205a59d1467cead63a9626", + "url": "https://api.github.com/repos/utopia-php/cache/zipball/e0a88c4a6568a84d40482ebd880e73b040fc0b79", + "reference": "e0a88c4a6568a84d40482ebd880e73b040fc0b79", "shasum": "" }, "require": { @@ -1613,9 +1613,9 @@ ], "support": { "issues": "https://github.com/utopia-php/cache/issues", - "source": "https://github.com/utopia-php/cache/tree/0.10.0" + "source": "https://github.com/utopia-php/cache/tree/feat-redis-sync" }, - "time": "2024-06-05T16:40:43+00:00" + "time": "2024-06-03T07:42:02+00:00" }, { "name": "utopia-php/cli", @@ -2988,16 +2988,16 @@ "packages-dev": [ { "name": "appwrite/sdk-generator", - "version": "0.38.6", + "version": "0.38.7", "source": { "type": "git", "url": "https://github.com/appwrite/sdk-generator.git", - "reference": "d7016d6d72545e84709892faca972eb4bf5bd699" + "reference": "0a66c1149ef05ed9f45ce1c897c4a0ce9ee5e95a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/d7016d6d72545e84709892faca972eb4bf5bd699", - "reference": "d7016d6d72545e84709892faca972eb4bf5bd699", + "url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0a66c1149ef05ed9f45ce1c897c4a0ce9ee5e95a", + "reference": "0a66c1149ef05ed9f45ce1c897c4a0ce9ee5e95a", "shasum": "" }, "require": { @@ -3033,9 +3033,9 @@ "description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms", "support": { "issues": "https://github.com/appwrite/sdk-generator/issues", - "source": "https://github.com/appwrite/sdk-generator/tree/0.38.6" + "source": "https://github.com/appwrite/sdk-generator/tree/0.38.7" }, - "time": "2024-05-20T18:00:16+00:00" + "time": "2024-06-10T00:23:02+00:00" }, { "name": "doctrine/deprecations", @@ -3346,16 +3346,16 @@ }, { "name": "myclabs/deep-copy", - "version": "1.11.1", + "version": "1.12.0", "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "7284c22080590fb39f2ffa3e9057f10a4ddd0e0c" + "reference": "3a6b9a42cd8f8771bd4295d13e1423fa7f3d942c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/7284c22080590fb39f2ffa3e9057f10a4ddd0e0c", - "reference": "7284c22080590fb39f2ffa3e9057f10a4ddd0e0c", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/3a6b9a42cd8f8771bd4295d13e1423fa7f3d942c", + "reference": "3a6b9a42cd8f8771bd4295d13e1423fa7f3d942c", "shasum": "" }, "require": { @@ -3363,11 +3363,12 @@ }, "conflict": { "doctrine/collections": "<1.6.8", - "doctrine/common": "<2.13.3 || >=3,<3.2.2" + "doctrine/common": "<2.13.3 || >=3 <3.2.2" }, "require-dev": { "doctrine/collections": "^1.6.8", "doctrine/common": "^2.13.3 || ^3.2.2", + "phpspec/prophecy": "^1.10", "phpunit/phpunit": "^7.5.20 || ^8.5.23 || ^9.5.13" }, "type": "library", @@ -3393,7 +3394,7 @@ ], "support": { "issues": "https://github.com/myclabs/DeepCopy/issues", - "source": "https://github.com/myclabs/DeepCopy/tree/1.11.1" + "source": "https://github.com/myclabs/DeepCopy/tree/1.12.0" }, "funding": [ { @@ -3401,7 +3402,7 @@ "type": "tidelift" } ], - "time": "2023-03-08T13:26:56+00:00" + "time": "2024-06-12T14:39:25+00:00" }, { "name": "nikic/php-parser", @@ -5588,9 +5589,18 @@ "time": "2023-11-21T18:54:41+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/cache", + "version": "dev-feat-redis-sync", + "alias": "0.10.0", + "alias_normalized": "0.10.0.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/cache": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -5614,5 +5624,5 @@ "platform-overrides": { "php": "8.3" }, - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.2.0" } diff --git a/docker-compose.yml b/docker-compose.yml index b86af1c12b6..f026a459563 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,6 +82,9 @@ services: - ./public:/usr/src/code/public - ./src:/usr/src/code/src - ./dev:/usr/src/code/dev + #- ./vendor/utopia-php/framework:/usr/src/code/vendor/utopia-php/framework + #- ./vendor/utopia-php/cache:/usr/src/code/vendor/utopia-php/cache + depends_on: - mariadb - redis @@ -271,6 +274,107 @@ services: - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES + appwrite-worker-sync-out-aggregation: + entrypoint: worker-sync-out-aggregation + <<: *x-logging + container_name: worker-sync-out-aggregation + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + #- appwrite-certificates:/storage/certificates:rw + - appwrite-syncs:/storage/syncs:rw + depends_on: + - mariadb + - redis + environment: + - _APP_ENV + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE + - _APP_WORKER_PER_CORE + - _APP_REGION + + appwrite-worker-sync-out-delivery: + entrypoint: worker-sync-out-delivery + <<: *x-logging + container_name: worker-sync-out-delivery + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + #- appwrite-certificates:/storage/certificates:rw + - appwrite-syncs:/storage/syncs:rw + depends_on: + - mariadb + - redis + environment: + - _APP_OPENSSL_KEY_V1 + - _APP_ENV + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE + - _APP_WORKER_PER_CORE + - _APP_REGION + + appwrite-worker-sync-in: + entrypoint: worker-sync-in + <<: *x-logging + container_name: appwrite-worker-sync-in + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + - appwrite-certificates:/storage/certificates:rw + #- ./vendor/utopia-php/cache:/usr/src/code/vendor/utopia-php/cache + + depends_on: + - redis + environment: + - _APP_ENV + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_MAX + - _APP_POOL_CLIENTS + - _APP_CONNECTIONS_QUEUE + - _APP_WORKER_PER_CORE + - _APP_REGION + appwrite-worker-webhooks: entrypoint: worker-webhooks <<: *x-logging @@ -360,6 +464,7 @@ services: - _APP_LOGGING_CONFIG - _APP_EXECUTOR_SECRET - _APP_EXECUTOR_HOST + - _APP_REGION - _APP_DATABASE_SHARED_TABLES appwrite-worker-databases: @@ -495,6 +600,7 @@ services: - _APP_LOGGING_PROVIDER - _APP_LOGGING_CONFIG - _APP_DATABASE_SHARED_TABLES + - _APP_REGION appwrite-worker-functions: entrypoint: worker-functions @@ -700,6 +806,40 @@ services: - _APP_MAINTENANCE_DELAY - _APP_DATABASE_SHARED_TABLES + appwrite-task-region-sync: + entrypoint: region-sync + <<: *x-logging + container_name: appwrite-task-region-sync + image: appwrite-dev + networks: + - appwrite + volumes: + - ./app:/usr/src/code/app + - ./src:/usr/src/code/src + - appwrite-syncs:/storage/syncs:rw + depends_on: + - mariadb + - redis + environment: + - _APP_CONNECTIONS_MAX + - _APP_CONNECTIONS_DB_PROJECT + - _APP_CONNECTIONS_DB_CONSOLE + - _APP_CONNECTIONS_CACHE + - _APP_CONNECTIONS_QUEUE + - _APP_POOL_CLIENTS + - _APP_ENV + - _APP_REDIS_HOST + - _APP_REDIS_PORT + - _APP_REDIS_USER + - _APP_REDIS_PASS + - _APP_DB_HOST + - _APP_DB_PORT + - _APP_DB_SCHEMA + - _APP_DB_USER + - _APP_DB_PASS + - _APP_SYNC_EDGE_INTERVAL + - _APP_REGION + appwrite-worker-usage: entrypoint: worker-usage <<: *x-logging @@ -1015,3 +1155,4 @@ volumes: appwrite-functions: appwrite-builds: appwrite-config: + appwrite-syncs: diff --git a/phpbench.json b/phpbench.json new file mode 100644 index 00000000000..e3868519849 --- /dev/null +++ b/phpbench.json @@ -0,0 +1,17 @@ +{ + "$schema": "./vendor/phpbench/phpbench/phpbench.schema.json", + "runner.path": "tests/benchmarks", + "runner.file_pattern": "*Bench.php", + "runner.bootstrap": "app/init.php", + "runner.revs": 1000, + "runner.iterations": 3, + "runner.retry_threshold": 5, + "runner.warmup": 1, + "report.generators": { + "appwrite": { + "extends": "aggregate", + "cols": ["benchmark", "subject", "set" ,"revs", "its", "worst", "best", "mean"], + "break": ["benchmark"] + } + } +} \ No newline at end of file diff --git a/src/Appwrite/Platform/Services/Tasks.php b/src/Appwrite/Platform/Services/Tasks.php index 31465d2f26e..4a0d0332bab 100644 --- a/src/Appwrite/Platform/Services/Tasks.php +++ b/src/Appwrite/Platform/Services/Tasks.php @@ -8,6 +8,7 @@ use Appwrite\Platform\Tasks\Migrate; use Appwrite\Platform\Tasks\QueueCount; use Appwrite\Platform\Tasks\QueueRetry; +use Appwrite\Platform\Tasks\RegionSync; use Appwrite\Platform\Tasks\ScheduleFunctions; use Appwrite\Platform\Tasks\ScheduleMessages; use Appwrite\Platform\Tasks\SDKs; @@ -38,6 +39,8 @@ public function __construct() ->addAction(Upgrade::getName(), new Upgrade()) ->addAction(Vars::getName(), new Vars()) ->addAction(Version::getName(), new Version()) + ->addAction(RegionSync::getName(), new RegionSync()); + ; } } diff --git a/src/Appwrite/Platform/Services/Workers.php b/src/Appwrite/Platform/Services/Workers.php index 0e79f4257cc..66fcc1f1faf 100644 --- a/src/Appwrite/Platform/Services/Workers.php +++ b/src/Appwrite/Platform/Services/Workers.php @@ -11,6 +11,9 @@ use Appwrite\Platform\Workers\Mails; use Appwrite\Platform\Workers\Messaging; use Appwrite\Platform\Workers\Migrations; +use Appwrite\Platform\Workers\SyncIn; +use Appwrite\Platform\Workers\SyncOutAggregation; +use Appwrite\Platform\Workers\SyncOutDelivery; use Appwrite\Platform\Workers\Usage; use Appwrite\Platform\Workers\UsageDump; use Appwrite\Platform\Workers\Webhooks; @@ -34,6 +37,9 @@ public function __construct() ->addAction(UsageDump::getName(), new UsageDump()) ->addAction(Usage::getName(), new Usage()) ->addAction(Migrations::getName(), new Migrations()) + ->addAction(SyncIn::getName(), new SyncIn()) + ->addAction(SyncOutAggregation::getName(), new SyncOutAggregation()) + ->addAction(SyncOutDelivery::getName(), new SyncOutDelivery()) ; } diff --git a/src/Appwrite/Platform/Tasks/RegionSync.php b/src/Appwrite/Platform/Tasks/RegionSync.php new file mode 100644 index 00000000000..2648b53fc05 --- /dev/null +++ b/src/Appwrite/Platform/Tasks/RegionSync.php @@ -0,0 +1,84 @@ +desc('Region sync task, resend failed payloads and delete successful ones') + ->inject('dbForConsole') + ->inject('queueForSyncOutDelivery') + ->callback(fn (Database $dbForConsole, Client $queueForSyncOutDelivery) => $this->action($dbForConsole, $queueForSyncOutDelivery)); + } + + public function action(Database $dbForConsole, Client $queueForSyncOutDelivery): void + { + Console::title('Edge-sync V1'); + Console::success(APP_NAME . ' Region sync v1 has started'); + + $interval = (int) App::getEnv('_APP_SYNC_EDGE_INTERVAL', '20'); + Console::loop(function () use ($interval, $dbForConsole, $queueForSyncOutDelivery) { + $time = DateTime::now(); + + Console::success("[{$time}] New task every {$interval} seconds"); + + $chunk = 0; + $limit = 500; + $sum = $limit; + while ($sum === $limit) { + $chunk++; + $count = 0; + $results = $dbForConsole->find('syncs', [ + Query::equal('sourceRegion', [System::getEnv('_APP_REGION', 'fra')]), + Query::limit($limit) + ]); + + $sum = count($results); + if ($sum > 0) { + foreach ($results as $sync) { + try { + if ($sync->getAttribute('status') === 200) { + $dbForConsole->deleteDocument('syncs', $sync->getId()); + unlink(APP_STORAGE_SYNCS . '/' . $sync->getAttribute('filename') . '.log'); + Console::log("[{$time}] Deleting {$sync->getId()}"); + } else { + Console::log("[{$time}] Enqueueing {$sync->getId()} to {$sync->getAttribute('destRegion')}"); + + $sync->setAttribute('attempts', $sync->getAttribute('attempts') + 1); + $dbForConsole->updateDocument('syncs', $sync->getId(), $sync); + + $queueForSyncOutDelivery + ->enqueue([ + 'syncId' => $sync->getId(), + ]); + } + + } catch(\Throwable $th) { + Console::log("[{$time}] Error: {$th->getMessage()}"); + } + + $count++; + } + } + + } + + }, $interval); + } +} diff --git a/src/Appwrite/Platform/Workers/Certificates.php b/src/Appwrite/Platform/Workers/Certificates.php index 1fbbd3b2ecd..dd09bfeb0b4 100644 --- a/src/Appwrite/Platform/Workers/Certificates.php +++ b/src/Appwrite/Platform/Workers/Certificates.php @@ -25,6 +25,7 @@ use Utopia\Locale\Locale; use Utopia\Logger\Log; use Utopia\Platform\Action; +use Utopia\Queue\Client; use Utopia\Queue\Message; use Utopia\System\System; @@ -47,8 +48,9 @@ public function __construct() ->inject('queueForMails') ->inject('queueForEvents') ->inject('queueForFunctions') + ->inject('queueForSyncOutAggregation') ->inject('log') - ->callback(fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log)); + ->callback(fn (Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, Client $queueForSyncOutAggregation) => $this->action($message, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $queueForSyncOutAggregation)); } /** @@ -58,11 +60,15 @@ public function __construct() * @param Event $queueForEvents * @param Func $queueForFunctions * @param Log $log + * @param Client $queueForSyncOutAggregation * @return void + * @throws Authorization + * @throws Conflict + * @throws Structure * @throws Throwable * @throws \Utopia\Database\Exception */ - public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log): void + public function action(Message $message, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, Client $queueForSyncOutAggregation): void { $payload = $message->getPayload() ?? []; @@ -76,7 +82,7 @@ public function action(Message $message, Database $dbForConsole, Mail $queueForM $log->addTag('domain', $domain->get()); - $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $skipRenewCheck); + $this->execute($domain, $dbForConsole, $queueForMails, $queueForEvents, $queueForFunctions, $log, $queueForSyncOutAggregation, $skipRenewCheck); } /** @@ -85,12 +91,17 @@ public function action(Message $message, Database $dbForConsole, Mail $queueForM * @param Mail $queueForMails * @param Event $queueForEvents * @param Func $queueForFunctions + * @param Log $log + * @param Client $queueForSyncOutAggregation * @param bool $skipRenewCheck * @return void + * @throws Authorization + * @throws Conflict + * @throws Structure * @throws Throwable * @throws \Utopia\Database\Exception */ - private function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, bool $skipRenewCheck = false): void + protected function execute(Domain $domain, Database $dbForConsole, Mail $queueForMails, Event $queueForEvents, Func $queueForFunctions, Log $log, Client $queueForSyncOutAggregation, bool $skipRenewCheck = false): void { /** * 1. Read arguments and validate domain @@ -171,6 +182,20 @@ private function execute(Domain $domain, Database $dbForConsole, Mail $queueForM $certificate->setAttribute('attempts', 0); $certificate->setAttribute('issueDate', DateTime::now()); $success = true; + + // Enqueue certificate for regional sync + $filename = APP_STORAGE_CERTIFICATES . '/' . $domain . '.tar.gz'; + if (file_exists($filename)) { + $queueForSyncOutAggregation->enqueue([ + 'type' => 'certificate', + 'key' => [ + 'domain' => $domain, + 'contents' => base64_encode(file_get_contents($filename)), + ] + ]); + } + + } catch (Throwable $e) { $logs = $e->getMessage(); @@ -212,7 +237,7 @@ private function execute(Domain $domain, Database $dbForConsole, Mail $queueForM * @throws Conflict * @throws Structure */ - private function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void + protected function saveCertificateDocument(string $domain, Document $certificate, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void { // Check if update or insert required $certificateDocument = $dbForConsole->findOne('certificates', [Query::equal('domain', [$domain])]); @@ -234,7 +259,7 @@ private function saveCertificateDocument(string $domain, Document $certificate, * * @return null|string Returns main domain. If null, there is no main domain yet. */ - private function getMainDomain(): ?string + protected function getMainDomain(): ?string { $envDomain = System::getEnv('_APP_DOMAIN', ''); if (!empty($envDomain) && $envDomain !== 'localhost') { @@ -255,7 +280,7 @@ private function getMainDomain(): ?string * @return void * @throws Exception */ - private function validateDomain(Domain $domain, bool $isMainDomain, Log $log): void + protected function validateDomain(Domain $domain, bool $isMainDomain, Log $log): void { if (empty($domain->get())) { throw new Exception('Missing certificate domain.'); @@ -299,7 +324,7 @@ private function validateDomain(Domain $domain, bool $isMainDomain, Log $log): v * @return bool True, if certificate needs to be renewed * @throws Exception */ - private function isRenewRequired(string $domain, Log $log): bool + protected function isRenewRequired(string $domain, Log $log): bool { $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; if (\file_exists($certPath)) { @@ -333,7 +358,7 @@ private function isRenewRequired(string $domain, Log $log): bool * @return array Named array with keys 'stdout' and 'stderr', both string * @throws Exception */ - private function issueCertificate(string $folder, string $domain, string $email): array + protected function issueCertificate(string $folder, string $domain, string $email): array { $stdout = ''; $stderr = ''; @@ -363,7 +388,7 @@ private function issueCertificate(string $folder, string $domain, string $email) * @return string * @throws \Utopia\Database\Exception */ - private function getRenewDate(string $domain): string + protected function getRenewDate(string $domain): string { $certPath = APP_STORAGE_CERTIFICATES . '/' . $domain . '/cert.pem'; $certData = openssl_x509_parse(file_get_contents($certPath)); @@ -381,7 +406,7 @@ private function getRenewDate(string $domain): string * @return void * @throws Exception */ - private function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void + protected function applyCertificateFiles(string $folder, string $domain, array $letsEncryptData): void { // Prepare folder in storage for domain @@ -432,7 +457,7 @@ private function applyCertificateFiles(string $folder, string $domain, array $le * @return void * @throws Exception */ - private function notifyError(string $domain, string $errorMessage, int $attempt, Mail $queueForMails): void + protected function notifyError(string $domain, string $errorMessage, int $attempt, Mail $queueForMails): void { // Log error into console Console::warning('Cannot renew domain (' . $domain . ') on attempt no. ' . $attempt . ' certificate: ' . $errorMessage); @@ -491,7 +516,7 @@ private function notifyError(string $domain, string $errorMessage, int $attempt, * * @return void */ - private function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void + protected function updateDomainDocuments(string $certificateId, string $domain, bool $success, Database $dbForConsole, Event $queueForEvents, Func $queueForFunctions): void { $rule = $dbForConsole->findOne('rules', [ diff --git a/src/Appwrite/Platform/Workers/SyncIn.php b/src/Appwrite/Platform/Workers/SyncIn.php new file mode 100644 index 00000000000..1215457cc6e --- /dev/null +++ b/src/Appwrite/Platform/Workers/SyncIn.php @@ -0,0 +1,94 @@ +desc('Sync in worker') + ->inject('message') + ->inject('cache') + ->callback(fn (Message $message, Cache $cache) => $this->action($message, $cache)); + } + + + /** + * @param Message $message + * @param Cache $cache + * @throws Exception + */ + public function action(Message $message, Cache $cache): void + { + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + $type = $payload['type']; + $key = $payload['key']; + $time = DateTime::now(); + + switch ($type) { + case 'cache': + $cache->setListenersStatus(false); + Console::log("[{$time}] Purging cache key {$key}"); + $cache->purge($key); + $cache->setListenersStatus(true); + break; + case 'realtime': + Console::log("[{$time}] Sending realtime message"); + Realtime::send( + projectId: $key['projectId'], + payload: $key['payload'], + events: $key['events'], + channels: $key['channels'], + roles: $key['roles'], + options: $key['options'] + ); + break; + case 'certificate': + Console::log("[{$time}] Writing certificate for domain [{$key['domain']}]"); + + $path = APP_STORAGE_CERTIFICATES . '/__' . $key['domain']; + $filename = $key['domain'] . 'tar.gz'; + if (!file_exists($path)) { + mkdir($path, 0755, true); + } + + $result = file_put_contents($path . '/' . $filename, base64_decode($key['contents'])); + if (empty($result)) { + Console::error('Can not write ' . $key['filename']); + break; + } + + $stdout = ''; + $stderr = ''; + $result = Console::execute('cd ' . $path . ' && tar xvzf ' . $filename, '', $stdout, $stderr); + if ($result === 1) { + Console::error('Can not open ' . $filename); + } + break; + default: + break; + } + } +} diff --git a/src/Appwrite/Platform/Workers/SyncOutAggregation.php b/src/Appwrite/Platform/Workers/SyncOutAggregation.php new file mode 100644 index 00000000000..7f5c9952e8b --- /dev/null +++ b/src/Appwrite/Platform/Workers/SyncOutAggregation.php @@ -0,0 +1,116 @@ +desc('Sync out aggregation worker') + ->inject('message') + ->inject('dbForConsole') + ->inject('queueForSyncOutDelivery') + ->callback(fn (Message $message, Database $dbForConsole, Client $queueForSyncOutDelivery) => $this->action($message, $dbForConsole, $queueForSyncOutDelivery)); + } + + + /** + * @param Message $message + * @param Database $dbForConsole + * @param Client $queueForSyncOutDelivery + * @throws Exception + * @throws JsonException + * @throws \Utopia\Database\Exception + * @throws Authorization + * @throws Structure + */ + public function action(Message $message, Database $dbForConsole, Client $queueForSyncOutDelivery): void + { + + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + if (!empty($payload['key'])) { + $this->keys[] = [ + 'time' => time(), + 'type' => $payload['type'], + 'key' => $payload['key'], + ]; + } + + $destRegions = []; + $currentRegion = System::getEnv('_APP_REGION', 'fra'); + foreach(Config::getParam('regions', []) as $destRegion) { + if($currentRegion !== $destRegion['$id'] && $destRegion['disabled'] === false) { + $destRegions[] = $destRegion['$id']; + } + } + + if ( + count($this->keys) >= self::KEYS_THRESHOLD || + (time() - $this->lastTriggeredTime > self::AGGREGATION_INTERVAL) + ) { + $chunk = array_slice($this->keys, 0, self::KEYS_THRESHOLD, true); + array_splice($this->keys, 0, self::KEYS_THRESHOLD); + + $filename = (string)time(); + $chunk = json_encode($chunk, flags: JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT); + $path = APP_STORAGE_SYNCS . '/' . $filename . '.log'; + + Console::log('Writing log '. $path); + + if (\file_put_contents($path, $chunk)) { + foreach($destRegions as $destRegion) { + Console::log('Creating documents to '. $destRegion); + $sync = $dbForConsole->createDocument('syncs', new Document([ + 'sourceRegion' => $currentRegion, + 'destRegion' => $destRegion, + 'filename' => $filename, + 'logCreatedAt' => DateTime::now() + ])); + + $queueForSyncOutDelivery + ->enqueue([ + 'syncId' => $sync->getId(), + ]); + + $this->lastTriggeredTime = time(); + } + } else { + Console::error('Failed to save log : ' . $path); + } + } + } +} diff --git a/src/Appwrite/Platform/Workers/SyncOutDelivery.php b/src/Appwrite/Platform/Workers/SyncOutDelivery.php new file mode 100644 index 00000000000..5c3d49bd1e6 --- /dev/null +++ b/src/Appwrite/Platform/Workers/SyncOutDelivery.php @@ -0,0 +1,126 @@ +desc('Region Syncs out worker') + ->inject('message') + ->inject('dbForConsole') + ->callback(fn (Message $message, Database $dbForConsole) => $this->action($message, $dbForConsole)); + } + + + /** + * @param Message $message + * @param Database $dbForConsole + * @throws Exception + */ + public function action(Message $message, Database $dbForConsole): void + { + + $payload = $message->getPayload() ?? []; + + if (empty($payload)) { + throw new Exception('Missing payload'); + } + + if (!empty($payload['syncId'])) { + try { + $sync = $dbForConsole->getDocument('syncs', $payload['syncId']); + //$regions = Config::getParam('regions', []); + $destRegion = $sync->getAttribute('destRegion'); + $chunk = file_get_contents(APP_STORAGE_SYNCS . '/' . $sync->getAttribute('filename') . '.log'); + $jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 600, 10); + $token = $jwt->encode([]); + //$status = $this->send($regions[$destRegion]['domain'] . '/v1/region/sync', $token, json_decode($chunk)); + $status = $this->send('http://appwrite/v1/region/sync', $token, json_decode($chunk)); + Console::log('[' . DateTime::now() . '] Request ' . $sync->getId() . ' to ' . $destRegion . ' returned status ' . $status); + + $sync->setAttribute('logSentAt', DateTime::now()); + $sync->setAttribute('status', $status); + $dbForConsole->updateDocument('syncs', $sync->getId(), $sync); + } catch (\Throwable $th) { + Console::log('[' . DateTime::now() . '] Error: ' .$th->getMessage()); + } + } + } + + /** + * @param string $url + * @param string $token + * @param array data + * @return int + */ + public function send(string $url, string $token, array $data): int + { + + $boundary = uniqid(); + $delimiter = '-------------' . $boundary; + $payload = ''; + $eol = "\r\n"; + + console::warning('Sending ' . count($data) . 'to ' . $url); + + foreach ($data as $keys) { + $payload .= "--" . $delimiter . $eol + . 'Content-Disposition: form-data; name="keys[]"' . $eol . $eol + . json_encode($keys) . $eol; + } + $payload .= "--" . $delimiter . "--" . $eol; + $status = 404; + $ch = curl_init($url); + curl_setopt($ch, CURLOPT_HTTPHEADER, [ + 'Authorization: Bearer ' . $token, + 'Origin-region-url: ' . App::getEnv('_APP_REGION'), + 'Content-type: multipart/form-data; boundary=' . $delimiter, + 'Content-Length: ' . strlen($payload) + ]); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_TIMEOUT, 5); + curl_setopt($ch, CURLOPT_POST, true); + curl_setopt($ch, CURLOPT_POSTFIELDS, $payload); + //curl_setopt($ch, CURLOPT_VERBOSE, true); + curl_setopt($ch, CURLOPT_HEADER, false); + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST"); + + for ($attempts = 0; $attempts < self::MAX_SEND_ATTEMPTS; $attempts++) { + curl_exec($ch); + $status = curl_getinfo($ch, CURLINFO_HTTP_CODE); + + if ($status === 200) { + return $status; + } + + sleep(1); + } + + curl_close($ch); + + return $status; + } +} diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 6601a360754..d12fc5d3645 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -79,6 +79,7 @@ use Appwrite\Utopia\Response\Model\Project; use Appwrite\Utopia\Response\Model\Provider; use Appwrite\Utopia\Response\Model\ProviderRepository; +use Appwrite\Utopia\Response\Model\RegionSync; use Appwrite\Utopia\Response\Model\Rule; use Appwrite\Utopia\Response\Model\Runtime; use Appwrite\Utopia\Response\Model\Session; @@ -275,6 +276,7 @@ class Response extends SwooleResponse public const MODEL_VCS = 'vcs'; public const MODEL_SMS_TEMPLATE = 'smsTemplate'; public const MODEL_EMAIL_TEMPLATE = 'emailTemplate'; + public const MODEL_REGION_SYNC = 'regionSync'; // Health public const MODEL_HEALTH_STATUS = 'healthStatus'; @@ -456,6 +458,7 @@ public function __construct(SwooleHTTPResponse $response) ->setModel(new Migration()) ->setModel(new MigrationReport()) ->setModel(new MigrationFirebaseProject()) + ->setModel(new RegionSync()) // Tests (keep last) ->setModel(new Mock()); diff --git a/src/Appwrite/Utopia/Response/Model/RegionSync.php b/src/Appwrite/Utopia/Response/Model/RegionSync.php new file mode 100644 index 00000000000..d5680164477 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/RegionSync.php @@ -0,0 +1,43 @@ +addRule('keys', [ + 'type' => self::TYPE_STRING, + 'description' => 'Resources keys array to be purged.', + 'default' => '', + 'example' => '["cache-console:_metadata:users", "cache-console:_metadata:buckets"]', + ]) + + ; + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'EdgeSync'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_REGION_SYNC; + } +} diff --git a/tests/benchmarks/Scopes/Scope.php b/tests/benchmarks/Scopes/Scope.php new file mode 100644 index 00000000000..03b2d7f1ef3 --- /dev/null +++ b/tests/benchmarks/Scopes/Scope.php @@ -0,0 +1,14 @@ +client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'documentId' => ID::unique(), + 'data' => [ + 'title' => 'The Matrix', + ], + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::write(Role::user($this->getUser()['$id'])), + ], + ]); + } + + #[ParamProviders(['provideCounts'])] + #[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])] + public function benchDocumentReadList(array $params) + { + $this->client->call(Client::METHOD_GET, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => ['limit(' . $params['documents'] . ')'], + ]); + } + + #[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])] + public function benchDocumentRead() + { + $this->client->call(Client::METHOD_GET, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents/' . static::$documentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + } + + #[BeforeMethods(['createDatabase', 'createCollection', 'createDocuments'])] + public function benchDocumentUpdate() + { + $this->client->call(Client::METHOD_PATCH, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents/' . static::$documentId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'data' => [ + 'title' => 'The Matrix Reloaded', + ], + ]); + } + + public function provideCounts(): array + { + return [ + '1 Document' => ['documents' => 1], + '10 Documents' => ['documents' => 10], + '100 Documents' => ['documents' => 100], + ]; + } + + public function createDatabase(array $params = []) + { + $database = $this->client->call(Client::METHOD_POST, '/databases', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'databaseId' => ID::unique(), + 'name' => 'Test Database' + ]); + static::$databaseId = $database['body']['$id']; + } + + public function createCollection(array $params = []) + { + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'collectionId' => ID::unique(), + 'name' => 'Movies', + 'documentSecurity' => true, + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::write(Role::user($this->getUser()['$id'])), + ], + ]); + static::$collectionId = $collection['body']['$id']; + + // Create attribute + $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/attributes/string', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'key' => 'title', + 'size' => 256, + 'required' => true, + ]); + + // Wait for attribute to be ready + sleep(2); + } + + public function createDocuments(array $params = []) + { + $count = $params['documents'] ?? 1; + + for ($i = 0; $i < $count; $i++) { + $response = $this->client->call(Client::METHOD_POST, '/databases/' . static::$databaseId . '/collections/' . static::$collectionId . '/documents', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'documentId' => ID::unique(), + 'data' => [ + 'title' => 'Captain America' . $i, + ], + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::write(Role::user($this->getUser()['$id'])), + ] + ]); + + static::$documentId = $response['body']['$id']; + } + } +} diff --git a/tests/benchmarks/Services/Databases/DatabasesCustomClientBench.php b/tests/benchmarks/Services/Databases/DatabasesCustomClientBench.php new file mode 100644 index 00000000000..658ddfd98cb --- /dev/null +++ b/tests/benchmarks/Services/Databases/DatabasesCustomClientBench.php @@ -0,0 +1,10 @@ +client->call(Client::METHOD_POST, '/functions/' . static::$functionId . '/executions', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + } + + public function createFunction() + { + $response = $this->client->call(Client::METHOD_POST, '/functions', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'functionId' => ID::unique(), + 'name' => 'Test', + 'runtime' => 'php-8.0', + 'timeout' => 10, + 'execute' => [Role::users()->toString()] + ]); + static::$functionId = $response['body']['$id']; + } + + public function prepareDeployment() + { + $stdout = ''; + $stderr = ''; + + Console::execute( + 'cd ' . realpath(__DIR__ . "/../../../resources/functions/php") . " && \ + tar --exclude code.tar.gz -czf code.tar.gz .", + '', + $stdout, + $stderr + ); + } + + public function createDeployment() + { + $code = realpath(__DIR__ . '/../../../resources/functions/php/code.tar.gz'); + + $response = $this->client->call(Client::METHOD_POST, '/functions/' . static::$functionId . '/deployments', [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'entrypoint' => 'index.php', + 'code' => new CURLFile( + $code, + 'application/x-gzip', + \basename($code) + ), + ]); + + static::$deploymentId = $response['body']['$id']; + + while (true) { + $response = $this->client->call(Client::METHOD_GET, '/functions/' . static::$functionId . '/deployments/' . static::$deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ]); + + $status = $response['body']['status'] ?? ''; + + switch ($status) { + case '': + case 'processing': + case 'building': + usleep(200); + break; + case 'ready': + break 2; + case 'failed': + throw new \Exception('Failed to build function'); + } + } + + sleep(1); + } + + public function patchDeployment() + { + $this->client->call(Client::METHOD_PATCH, '/functions/' . static::$functionId . '/deployments/' . static::$deploymentId, [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'], + ], []); + } +} diff --git a/tests/benchmarks/Services/Functions/FunctionsCustomClientBench.php b/tests/benchmarks/Services/Functions/FunctionsCustomClientBench.php new file mode 100644 index 00000000000..ec5e0b8f294 --- /dev/null +++ b/tests/benchmarks/Services/Functions/FunctionsCustomClientBench.php @@ -0,0 +1,10 @@ +createDeployment(); + } +} diff --git a/tests/benchmarks/Services/Storage/Base.php b/tests/benchmarks/Services/Storage/Base.php new file mode 100644 index 00000000000..6496490a6f2 --- /dev/null +++ b/tests/benchmarks/Services/Storage/Base.php @@ -0,0 +1,117 @@ +client->call(Client::METHOD_POST, '/storage/buckets/' . static::$bucketId . '/files', array_merge([ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'fileId' => ID::unique(), + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::write(Role::user($this->getUser()['$id'])), + ], + + 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/logo.png'), 'image/png', 'logo.png'), + ]); + } + + #[ParamProviders(['provideCounts'])] + #[BeforeMethods(['createBucket', 'createFiles'])] + public function benchFileReadList(array $params) + { + $this->client->call(Client::METHOD_GET, '/storage/buckets/' . static::$bucketId . '/files', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => ['limit(' . $params['files'] . ')'], + ]); + } + + #[BeforeMethods(['createBucket', 'createFiles'])] + public function benchFileRead() + { + $this->client->call(Client::METHOD_GET, '/storage/buckets/' . static::$bucketId . '/files/' . static::$fileId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + } + + #[BeforeMethods(['createBucket', 'createFiles'])] + public function benchFileUpdate() + { + $this->client->call(Client::METHOD_PUT, '/storage/buckets/' . static::$bucketId . '/files/' . static::$fileId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'Updated name', + 'permissions' => [], + ]); + } + + public function provideCounts(): array + { + return [ + '10 Files' => ['files' => 10], + '100 Files' => ['files' => 100], + ]; + } + + public function createBucket(array $params = []) + { + // Create bucket + $bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'bucketId' => ID::unique(), + 'name' => 'Test Bucket', + 'permissions' => [ + Permission::read(Role::user($this->getUser()['$id'])), + Permission::write(Role::user($this->getUser()['$id'])), + ], + 'fileSecurity' => true + ]); + static::$bucketId = $bucket['body']['$id']; + } + + public function createFiles(array $params = []) + { + $count = $params['files'] ?? 1; + + // Create files + for ($i = 0; $i < $count; $i++) { + $response = $this->client->call(Client::METHOD_POST, '/storage/buckets/' . static::$bucketId . '/files', [ + 'content-type' => 'multipart/form-data', + 'x-appwrite-project' => $this->getProject()['$id'], + 'x-appwrite-key' => $this->getProject()['apiKey'] + ], [ + 'fileId' => ID::unique(), + 'file' => new CURLFile(realpath(__DIR__ . '/../../../resources/logo.png'), 'image/png', 'logo.png'), + ]); + + static::$fileId = $response['body']['$id']; + } + } +} diff --git a/tests/benchmarks/Services/Storage/StorageCustomClientBench.php b/tests/benchmarks/Services/Storage/StorageCustomClientBench.php new file mode 100644 index 00000000000..585182c1da8 --- /dev/null +++ b/tests/benchmarks/Services/Storage/StorageCustomClientBench.php @@ -0,0 +1,10 @@ +client->call(Client::METHOD_POST, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'userId' => $id, + 'email' => 'test' . $id . '@example.com', + 'password' => 'password', + ]); + } + + #[ParamProviders(['provideCounts'])] + #[BeforeMethods(['createUsers'])] + public function benchUserReadList(array $params) + { + $this->client->call(Client::METHOD_GET, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'queries' => ['limit(' . $params['users'] . ')'], + ]); + } + + #[BeforeMethods(['createUsers'])] + public function benchUserRead() + { + $this->client->call(Client::METHOD_GET, '/users/' . static::$userId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + } + + #[BeforeMethods(['createUsers'])] + public function benchUserUpdate() + { + $this->client->call(Client::METHOD_PUT, '/users/' . static::$userId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'name' => 'New Name', + ]); + } + + public function createUsers(array $params = []) + { + $count = $params['documents'] ?? 1; + + for ($i = 0; $i < $count; $i++) { + $id = ID::unique(); + + $response = $this->client->call(Client::METHOD_POST, '/users', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'userId' => $id, + 'email' => 'test' . $id . '@example.com', + 'password' => 'password', + ]); + + static::$userId = $response['body']['$id']; + } + } + + public function provideCounts(): array + { + return [ + '1 User' => ['users' => 1], + '10 Users' => ['users' => 10], + '100 Users' => ['users' => 100], + ]; + } +} diff --git a/tests/benchmarks/http.js b/tests/benchmarks/http.js deleted file mode 100644 index 799c8fb23c7..00000000000 --- a/tests/benchmarks/http.js +++ /dev/null @@ -1,34 +0,0 @@ -import http from 'k6/http'; -import { check } from 'k6'; -import { Counter } from 'k6/metrics'; - -// A simple counter for http requests -export const requests = new Counter('http_reqs'); - -// you can specify stages of your test (ramp up/down patterns) through the options object -// target is the number of VUs you are aiming for - -export const options = { - stages: [ - { target: 50, duration: '1m' }, - // { target: 15, duration: '1m' }, - // { target: 0, duration: '1m' }, - ], - thresholds: { - requests: ['count < 100'], - }, -}; - -export default function () { - const config = { - headers: { - 'X-Appwrite-Key': '24356eb021863f81eb7dd77c7750304d0464e141cad6e9a8befa1f7d2b066fde190df3dab1e8d2639dbb82ee848da30501424923f4cd80d887ee40ad77ded62763ee489448523f6e39667f290f9a54b2ab8fad131a0bc985e6c0f760015f7f3411e40626c75646bb19d2bb2f7bf2f63130918220a206758cbc48845fd725a695', - 'X-Appwrite-Project': '60479fe35d95d' - }} - - const resDb = http.get('http://localhost:9501/', config); - - check(resDb, { - 'status is 200': (r) => r.status === 200, - }); -} \ No newline at end of file diff --git a/tests/benchmarks/ws.js b/tests/benchmarks/ws.js deleted file mode 100644 index 916458856fb..00000000000 --- a/tests/benchmarks/ws.js +++ /dev/null @@ -1,59 +0,0 @@ -// k6 run tests/benchmarks/ws.js - -import { URL } from 'https://jslib.k6.io/url/1.0.0/index.js'; -import ws from 'k6/ws'; -import { check } from 'k6'; - -export let options = { - stages: [ - { - duration: '10s', - target: 500 - }, - { - duration: '1m', - target: 500 - }, - ], -} - -export default function () { - // const url = new URL('wss://appwrite-realtime.monitor-api.com/v1/realtime'); - // url.searchParams.append('project', '604249e6b1a9f'); - const url = new URL('ws://localhost/v1/realtime'); - url.searchParams.append('project', 'console'); - url.searchParams.append('channels[]', 'files'); - - const res = ws.connect(url.toString(), function (socket) { - let connection = false; - let checked = false; - let payload = null; - socket.on('open', () => { - connection = true; - }); - - socket.on('message', (data) => { - payload = data; - checked = true; - }); - - socket.setTimeout(function () { - check(payload, { - 'connection opened': (r) => connection, - 'message received': (r) => checked, - 'channels are right': (r) => r === JSON.stringify({ - "type": "connected", - "data": { - "channels": [ - "files" - ], - "user": null - } - }) - }) - socket.close(); - }, 5000); - }); - - check(res, { 'status is 101': (r) => r && r.status === 101 }); -} \ No newline at end of file diff --git a/tests/e2e/General/AbuseTest.php b/tests/e2e/General/AbuseTest.php index 898fbd4aff8..b8b3fae334c 100644 --- a/tests/e2e/General/AbuseTest.php +++ b/tests/e2e/General/AbuseTest.php @@ -17,7 +17,7 @@ class AbuseTest extends Scope use ProjectCustom; use SideNone; - protected function setUp(): void + public function setUp(): void { parent::setUp(); diff --git a/tests/e2e/Scopes/Scope.php b/tests/e2e/Scopes/Scope.php index 3213ff4c5d0..2bbb176ac89 100644 --- a/tests/e2e/Scopes/Scope.php +++ b/tests/e2e/Scopes/Scope.php @@ -14,13 +14,13 @@ abstract class Scope extends TestCase protected ?Client $client = null; protected string $endpoint = 'http://localhost/v1'; - protected function setUp(): void + public function setUp(): void { $this->client = new Client(); $this->client->setEndpoint($this->endpoint); } - protected function tearDown(): void + public function tearDown(): void { $this->client = null; }