From d3741e03ba743752b678df98b2920a6602b809eb Mon Sep 17 00:00:00 2001 From: Lukas Rosenstock Date: Sun, 18 Oct 2026 16:46:17 +0000 Subject: [PATCH] Added fallback store feature --- CloudObjects/SDK/ObjectRetriever.php | 113 ++++++++++- docs/caching.md | 99 +++++++++- docs/options.md | 2 + .../OfflineTests/ObjectRetrieverMockTest.php | 178 +++++++++++++++++- tests/OfflineTests/ObjectRetrieverTest.php | 29 +++ 5 files changed, 408 insertions(+), 13 deletions(-) create mode 100644 tests/OfflineTests/ObjectRetrieverTest.php diff --git a/CloudObjects/SDK/ObjectRetriever.php b/CloudObjects/SDK/ObjectRetriever.php index fb622b0..2b5d66b 100644 --- a/CloudObjects/SDK/ObjectRetriever.php +++ b/CloudObjects/SDK/ObjectRetriever.php @@ -11,7 +11,8 @@ use ML\IRI\IRI; use ML\JsonLD\JsonLD, ML\JsonLD\Node; use Psr\Log\LoggerInterface, Psr\Log\LoggerAwareTrait; use GuzzleHttp\ClientInterface, GuzzleHttp\Client, GuzzleHttp\HandlerStack; -use GuzzleHttp\Exception\RequestException; +use GuzzleHttp\Exception\ConnectException, + GuzzleHttp\Exception\RequestException; use GuzzleHttp\Psr7\Request, GuzzleHttp\Psr7\Response; use Kevinrob\GuzzleCache\CacheMiddleware; @@ -41,6 +42,7 @@ class ObjectRetriever implements CustomCacheAndLogInterface { private $objects = []; private $defaultReader; private $customCacheDefaultRequest; + private $fallbackStoreScope; const CO_API_URL = 'https://od.coid.link/'; @@ -56,7 +58,9 @@ class ObjectRetriever implements CustomCacheAndLogInterface { 'api_base_url' => null, 'logger' => null, 'timeout' => 20, - 'connect_timeout' => 5 + 'connect_timeout' => 5, + 'fallback_store_directory' => null, + 'fallback_store_scope' => null ], $options); // Check object cache configuration @@ -76,6 +80,21 @@ class ObjectRetriever implements CustomCacheAndLogInterface { $this->cache = new Psr16CacheStorage($this->options['cache_storage']); } + // Check fallback store configuration + if (isset($this->options['fallback_store_directory']) + && isset($this->options['fallback_store_scope']) + && !is_dir($this->options['fallback_store_directory']) + ) { + throw new InvalidSDKConfigurationException('Invalid fallback_store_directory specified; must be an existing directory.'); + } + + if (isset($this->options['fallback_store_directory']) + && isset($this->options['fallback_store_scope'])) { + $this->fallbackStoreScope = is_array($this->options['fallback_store_scope']) + ? $this->options['fallback_store_scope'] + : array_map('trim', explode(',', $this->options['fallback_store_scope'])); + } + // Set up logger if (is_a($this->options['logger'], LoggerInterface::class)) { $this->setLogger($this->options['logger']); @@ -159,6 +178,57 @@ class ObjectRetriever implements CustomCacheAndLogInterface { $this->cache->save($this->getCacheKey($id), $entry, $ttl); } + + $this->putIntoFallbackStore($id, $data); + } + + /** + * Get the file location of an object in the fallback store, or null if + * the fallback store is not configured or the object is out of scope. + */ + private function getFallbackStoreLocation($id) : ?string { + if (!isset($this->fallbackStoreScope)) + return null; + + // Only objects are stored, not custom data or attachments + $coid = new IRI($id); + if (!COIDParser::isValidCOID($coid) || $coid->getFragment() !== null || $coid->getQuery() !== null) + return null; + + if (!in_array('*', $this->fallbackStoreScope) + && !in_array($coid->getHost(), $this->fallbackStoreScope)) + return null; + + return $this->options['fallback_store_directory'].DIRECTORY_SEPARATOR. + $coid->getHost().str_replace('/', DIRECTORY_SEPARATOR, $coid->getPath()) + .DIRECTORY_SEPARATOR.'object.jsonld'; + } + + private function putIntoFallbackStore($id, $data) { + $location = $this->getFallbackStoreLocation($id); + if (!isset($location)) + return; + + // Write to a temporary file and rename it so that concurrent readers + // never see a partially written object + $directory = dirname($location); + $tempFile = $location.'.'.uniqid('', true).'.tmp'; + if ((!is_dir($directory) && !@mkdir($directory, 0777, true) && !is_dir($directory)) + || @file_put_contents($tempFile, $data) === false + || !@rename($tempFile, $location)) { + @unlink($tempFile); + if (isset($this->logger)) + $this->logger->warning('Could not write <'.$id.'> to fallback store.'); + } + } + + private function getFromFallbackStore($id) : ?string { + $location = $this->getFallbackStoreLocation($id); + if (!isset($location) || !is_file($location)) + return null; + + $data = @file_get_contents($location); + return ($data === false) ? null : $data; } public function getFromCacheCustom($id) { @@ -282,21 +352,44 @@ class ObjectRetriever implements CustomCacheAndLogInterface { $object = (string)$response->getBody(); $this->putIntoCache($uriString, $object, $this->options['cache_ttl']); $this->logInfoWithTime('Fetched <'.$uriString.'> from Core API ['.$response->getStatusCode().'].', $ts); + } catch (ConnectException $e) { + // connection failed or timed out + $object = $this->getFromFallbackStore($uriString); + if (!isset($object)) { + $this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API (connection failed).', $ts); + throw new CoreAPIException($e->getMessage(), $e); + } + $this->logInfoWithTime('Fetched <'.$uriString.'> from fallback store (connection failed).', $ts); } catch (RequestException $e) { if ($e->hasResponse() && $e->getResponse()->getStatusCode() == 404) { - // not found + // not found (404) $this->logInfoWithTime('Object <'.$uriString.'> not found in Core API.', $ts); return null; } else - if ($e->hasResponse()) { - // other API error - $this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API ['.$e->getResponse()->getStatusCode().'].', $ts); - throw new CoreAPIException($e->getMessage(), $e); + if ($e->hasResponse() && $e->getResponse()->getStatusCode() >= 500) { + // other API error (5xx) + $object = $this->getFromFallbackStore($uriString); + $statusCode = $e->getResponse()->getStatusCode(); + if (!isset($object)) { + $this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API ['.$statusCode.'].', $ts); + throw new CoreAPIException($e->getMessage(), $e); + } + $this->logInfoWithTime('Fetched <'.$uriString.'> from fallback store (Core API returned ['.$statusCode.']).', $ts); + } else + if (!$e->hasResponse()) { + // API error without response + $object = $this->getFromFallbackStore($uriString); + if (!isset($object)) { + $this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API (no status code).', $ts); + throw new CoreAPIException($e->getMessage(), $e); + } + $this->logInfoWithTime('Fetched <'.$uriString.'> from fallback store (Core API returned no status code).', $ts); } else { - // API error without response - $this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API (no status code).', $ts); + // other API error (e.g. 4xx) + $statusCode = $e->getResponse()->getStatusCode(); + $this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API ['.$statusCode.'].', $ts); throw new CoreAPIException($e->getMessage(), $e); - } + } } catch (Exception $e) { // other exceptions $this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API (unexpected exception).', $ts); diff --git a/docs/caching.md b/docs/caching.md index a6decc5..2ffc4c1 100644 --- a/docs/caching.md +++ b/docs/caching.md @@ -1,9 +1,18 @@ ## Caching -# Standard Caching +### Standard Caching Object descriptions that are requested through the ObjectRetriever are fetched through the CloudObjects Core Object API. They are cached in-memory inside the class for a default of 60 seconds. This can be changed using the `cache_ttl` option, which takes a number of seconds. +```php +use CloudObjects\SDK\ObjectRetriever; + +// Keep objects in memory for 5 minutes +$retriever = new ObjectRetriever([ + 'cache_ttl' => 300, +]); +``` + This in-class memory obviously only works as long as the class is alive and prevents subsequent requests for the same object hitting the API, but the cache will not exist between multiple requests handled in different processes. For this purpose, an external cache should be added. To add an external cache, provide a storage using the `cache_storage` interface. The storage must be a class that implements either of the following interfaces: @@ -15,4 +24,90 @@ To add an external cache, provide a storage using the `cache_storage` interface. External storage is used twice: - Storing the HTTP request according to the APIs caching headers. -- Storing the object data according to `cache_ttl`. \ No newline at end of file +- Storing the object data according to `cache_ttl`. + +All keys written to the external cache are prefixed with the value of the `cache_prefix` option (default: `clobj:`), so you can share a cache with other parts of your application. + +The following examples use the [Symfony Cache component](https://symfony.com/doc/current/components/cache.html), but any library implementing one of the interfaces above works. + +Using a PSR-6 cache pool, e.g. on the filesystem: + +```php +use CloudObjects\SDK\ObjectRetriever; +use Symfony\Component\Cache\Adapter\FilesystemAdapter; + +$retriever = new ObjectRetriever([ + 'cache_storage' => new FilesystemAdapter('cloudobjects', 0, '/var/cache/myapp'), + 'cache_ttl' => 300, +]); +``` + +Using a PSR-16 simple cache, e.g. in Redis: + +```php +use CloudObjects\SDK\ObjectRetriever; +use Symfony\Component\Cache\Adapter\RedisAdapter; +use Symfony\Component\Cache\Psr16Cache; + +$redis = RedisAdapter::createConnection('redis://localhost'); + +$retriever = new ObjectRetriever([ + 'cache_storage' => new Psr16Cache(new RedisAdapter($redis)), + 'cache_prefix' => 'myapp:clobj:', +]); +``` + +### Fallback Store + +Fallback store is a mechanism that can be used by applications that require high availability but may lose connectivity to CloudObjects Core. With fallback store, a copy of an object is saved in a directory. If the API fails with a 5xx error, times out, or the request fails without a response, an `ObjectRetriever` will return the stored copy instead. Be advised that fallback storage will, once configured, result in your application not noticing when the Object API is unreachable - at least for objects in the store - and will continue to deliver the saved copy even if the original in CloudObjects Core is changed or removed. + +It is possible for the fallback store to be configured for all objects or only for objects in specified namespaces. Since the fallback store is never purged except by the SDK itself, it can grow large; hence it should be limited to specific namespaces in applications where many random objects may be retrieved. + +To enable fallback store, set the `fallback_store_directory` option to a writeable path on the filesystem. Also, set `fallback_store_scope` to either the string `*` to save all retrieved objects, or a list of namespaces (domains only without `coid://` prefix); the list of namespaces can be provided as a comma-separated string or an array of strings. + +Storing all retrieved objects: + +```php +use CloudObjects\SDK\ObjectRetriever; + +$retriever = new ObjectRetriever([ + 'fallback_store_directory' => '/var/lib/myapp/cloudobjects-fallback', + 'fallback_store_scope' => '*', +]); +``` + +Storing only objects from specific namespaces, provided as an array: + +```php +use CloudObjects\SDK\ObjectRetriever; + +$retriever = new ObjectRetriever([ + 'fallback_store_directory' => '/var/lib/myapp/cloudobjects-fallback', + 'fallback_store_scope' => [ 'example.com', 'cloudobjects.io' ], +]); +``` + +Or as a comma-separated string, which is convenient when the value comes from an environment variable: + +```php +use CloudObjects\SDK\ObjectRetriever; + +$retriever = new ObjectRetriever([ + 'fallback_store_directory' => getenv('FALLBACK_STORE_DIRECTORY'), + 'fallback_store_scope' => 'example.com,cloudobjects.io', +]); +``` + +The fallback store can be combined with an external cache. The cache is still consulted first; the fallback store only comes into play when an object is neither cached nor retrievable from the API. + +```php +use CloudObjects\SDK\ObjectRetriever; +use Symfony\Component\Cache\Adapter\FilesystemAdapter; + +$retriever = new ObjectRetriever([ + 'cache_storage' => new FilesystemAdapter('cloudobjects', 0, '/var/cache/myapp'), + 'cache_ttl' => 300, + 'fallback_store_directory' => '/var/lib/myapp/cloudobjects-fallback', + 'fallback_store_scope' => 'example.com', +]); +``` diff --git a/docs/options.md b/docs/options.md index 0850c69..b8e8862 100644 --- a/docs/options.md +++ b/docs/options.md @@ -14,3 +14,5 @@ The `ObjectRetriever` constructor accepts an associative array of options. All o | `logger` | Logger implementing `Psr\Log\LoggerInterface`, which receives info messages about object retrieval. | `null` | | `timeout` | Total timeout for API requests, in seconds. | `20` | | `connect_timeout` | Timeout for establishing a connection to the API, in seconds. | `5` | +| `fallback_store_directory` | Writeable directory in which copies of retrieved objects are saved and returned when the API fails with a 5xx error, times out, or the request fails without a response. Requires `fallback_store_scope`. See [caching](caching.md#fallback-store). | `null` | +| `fallback_store_scope` | Which objects are saved in the fallback store: `'*'` for all objects, or a list of namespaces (domains without `coid://` prefix) as a comma-separated string or an array of strings. | `null` | diff --git a/tests/OfflineTests/ObjectRetrieverMockTest.php b/tests/OfflineTests/ObjectRetrieverMockTest.php index 170fcf5..7754a56 100644 --- a/tests/OfflineTests/ObjectRetrieverMockTest.php +++ b/tests/OfflineTests/ObjectRetrieverMockTest.php @@ -9,7 +9,7 @@ namespace CloudObjects\SDK; use ML\IRI\IRI; use GuzzleHttp\Client, GuzzleHttp\Handler\MockHandler, GuzzleHttp\HandlerStack, GuzzleHttp\Psr7\Request, GuzzleHttp\Psr7\Response; -use GuzzleHttp\Exception\ConnectException; +use GuzzleHttp\Exception\ConnectException, GuzzleHttp\Exception\RequestException; use CloudObjects\SDK\Exceptions\CoreAPIException; class ObjectRetrieverMockTest extends \PHPUnit\Framework\TestCase { @@ -90,4 +90,180 @@ class ObjectRetrieverMockTest extends \PHPUnit\Framework\TestCase { } } + private function createFallbackStoreDirectory() : string { + $directory = sys_get_temp_dir().DIRECTORY_SEPARATOR.'clobj-fallback-'.uniqid(); + mkdir($directory); + return $directory; + } + + private function removeDirectory(string $directory) { + $files = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($directory, \FilesystemIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST + ); + foreach ($files as $file) + $file->isDir() ? rmdir($file->getPathname()) : unlink($file->getPathname()); + rmdir($directory); + } + + public function testFallbackStoreSavesObjectInScope() { + $directory = $this->createFallbackStoreDirectory(); + $this->retriever = new ObjectRetriever([ + 'fallback_store_directory' => $directory, + 'fallback_store_scope' => 'example.com, cloudobjects.io', + ]); + $body = '{"@id":"coid://cloudobjects.io/Test","http://www.w3.org/2000/01/rdf-schema#label":"Test"}'; + $this->setMockResponse(new Response(200, ['Content-Type' => 'application/ld+json'], $body)); + + try { + $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test')); + + $location = $directory.'/cloudobjects.io/Test/object.jsonld'; + $this->assertFileExists($location); + $this->assertEquals($body, file_get_contents($location)); + } finally { + $this->removeDirectory($directory); + } + } + + public function testFallbackStoreSkipsObjectOutOfScope() { + $directory = $this->createFallbackStoreDirectory(); + $this->retriever = new ObjectRetriever([ + 'fallback_store_directory' => $directory, + 'fallback_store_scope' => [ 'example.com' ], + ]); + $this->setMockResponse(new Response(200, ['Content-Type' => 'application/ld+json'], + '{"@id":"coid://cloudobjects.io/Test"}')); + + try { + $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test')); + $this->assertFileDoesNotExist($directory.'/cloudobjects.io/Test/object.jsonld'); + } finally { + $this->removeDirectory($directory); + } + } + + public function testFallbackStoreSavesAllObjectsWithWildcard() { + $directory = $this->createFallbackStoreDirectory(); + $this->retriever = new ObjectRetriever([ + 'fallback_store_directory' => $directory, + 'fallback_store_scope' => '*', + ]); + $this->setMockResponse(new Response(200, ['Content-Type' => 'application/ld+json'], + '{"@id":"coid://cloudobjects.io"}')); + + try { + $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io')); + $this->assertFileExists($directory.'/cloudobjects.io/object.jsonld'); + } finally { + $this->removeDirectory($directory); + } + } + + private function createRetrieverWithStoredObject(string $directory) { + $this->retriever = new ObjectRetriever([ + 'fallback_store_directory' => $directory, + 'fallback_store_scope' => '*', + ]); + mkdir($directory.'/cloudobjects.io/Test', 0777, true); + file_put_contents($directory.'/cloudobjects.io/Test/object.jsonld', + '{"@id":"coid://cloudobjects.io/Test","http://www.w3.org/2000/01/rdf-schema#label":"Stored"}'); + } + + public function testFallbackStoreUsedWhenAPIReturns500() { + $directory = $this->createFallbackStoreDirectory(); + $this->createRetrieverWithStoredObject($directory); + $this->setMockResponse(new Response(500)); + + try { + $object = $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test')); + $this->assertNotNull($object); + $this->assertEquals('Stored', $object->getProperty('http://www.w3.org/2000/01/rdf-schema#label')->getValue()); + } finally { + $this->removeDirectory($directory); + } + } + + public function testFallbackStoreUsedWhenAPITimesOut() { + $directory = $this->createFallbackStoreDirectory(); + $this->createRetrieverWithStoredObject($directory); + $this->setMockResponse(new ConnectException( + 'cURL error 28: Operation timed out', + new Request('GET', 'cloudobjects.io') + )); + + try { + $object = $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test')); + $this->assertNotNull($object); + $this->assertEquals('Stored', $object->getProperty('http://www.w3.org/2000/01/rdf-schema#label')->getValue()); + } finally { + $this->removeDirectory($directory); + } + } + + public function testFallbackStoreUsedWhenRequestFailsWithoutResponse() { + $directory = $this->createFallbackStoreDirectory(); + $this->createRetrieverWithStoredObject($directory); + $this->setMockResponse(new RequestException( + 'Request failed', + new Request('GET', 'cloudobjects.io') + )); + + try { + $object = $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test')); + $this->assertNotNull($object); + $this->assertEquals('Stored', $object->getProperty('http://www.w3.org/2000/01/rdf-schema#label')->getValue()); + } finally { + $this->removeDirectory($directory); + } + } + + public function testErrorWhenRequestFailsWithoutResponseAndObjectNotInFallbackStore() { + $directory = $this->createFallbackStoreDirectory(); + $this->retriever = new ObjectRetriever([ + 'fallback_store_directory' => $directory, + 'fallback_store_scope' => '*', + ]); + $this->setMockResponse(new RequestException( + 'Request failed', + new Request('GET', 'cloudobjects.io') + )); + + try { + $this->expectException(CoreAPIException::class); + $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test')); + } finally { + $this->removeDirectory($directory); + } + } + + public function testFallbackStoreNotUsedOnClientError() { + $directory = $this->createFallbackStoreDirectory(); + $this->createRetrieverWithStoredObject($directory); + $this->setMockResponse(new Response(403)); + + try { + $this->expectException(CoreAPIException::class); + $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test')); + } finally { + $this->removeDirectory($directory); + } + } + + public function testErrorWhenAPIThrowsErrorAndObjectNotInFallbackStore() { + $directory = $this->createFallbackStoreDirectory(); + $this->retriever = new ObjectRetriever([ + 'fallback_store_directory' => $directory, + 'fallback_store_scope' => '*', + ]); + $this->setMockResponse(new Response(500)); + + try { + $this->expectException(CoreAPIException::class); + $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test')); + } finally { + $this->removeDirectory($directory); + } + } + } diff --git a/tests/OfflineTests/ObjectRetrieverTest.php b/tests/OfflineTests/ObjectRetrieverTest.php new file mode 100644 index 0000000..338543d --- /dev/null +++ b/tests/OfflineTests/ObjectRetrieverTest.php @@ -0,0 +1,29 @@ + sys_get_temp_dir(), + 'fallback_store_scope' => '*', + ]); + $this->assertInstanceOf(ObjectRetriever::class, $retriever); + } + + public function testFallbackStoreWithMissingDirectory() { + $this->expectException(InvalidSDKConfigurationException::class); + new ObjectRetriever([ + 'fallback_store_directory' => sys_get_temp_dir().'/does-not-exist-'.uniqid(), + 'fallback_store_scope' => '*', + ]); + } + +}