CO/CloudObjects-PHP-SDK
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Added fallback store feature

Browse Source
This commit is contained in:
lukas committed 2026-10-18 16:46:17 +00:00
1 parent b3c810473f
commit d3741e03ba
5 files changed
+408 -13
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

CloudObjects/SDK/ObjectRetriever.php
+103 -10
View File
@@ -11,7 +11,8 @@ use ML\IRI\IRI;
use ML\JsonLD\JsonLD, ML\JsonLD\Node;
use Psr\Log\LoggerInterface, Psr\Log\LoggerAwareTrait;
use GuzzleHttp\ClientInterface, GuzzleHttp\Client, GuzzleHttp\HandlerStack;
use GuzzleHttp\Exception\RequestException;
use GuzzleHttp\Exception\ConnectException,
GuzzleHttp\Exception\RequestException;
use GuzzleHttp\Psr7\Request,
GuzzleHttp\Psr7\Response;
use Kevinrob\GuzzleCache\CacheMiddleware;
@@ -41,6 +42,7 @@ class ObjectRetriever implements CustomCacheAndLogInterface {
private $objects = [];
private $defaultReader;
private $customCacheDefaultRequest;
private $fallbackStoreScope;
const CO_API_URL = 'https://od.coid.link/';
@@ -56,7 +58,9 @@ class ObjectRetriever implements CustomCacheAndLogInterface {
'api_base_url' => null,
'logger' => null,
'timeout' => 20,
'connect_timeout' => 5
'connect_timeout' => 5,
'fallback_store_directory' => null,
'fallback_store_scope' => null
], $options);
// Check object cache configuration
@@ -76,6 +80,21 @@ class ObjectRetriever implements CustomCacheAndLogInterface {
$this->cache = new Psr16CacheStorage($this->options['cache_storage']);
}
// Check fallback store configuration
if (isset($this->options['fallback_store_directory'])
&& isset($this->options['fallback_store_scope'])
&& !is_dir($this->options['fallback_store_directory'])
) {
throw new InvalidSDKConfigurationException('Invalid fallback_store_directory specified; must be an existing directory.');
}
if (isset($this->options['fallback_store_directory'])
&& isset($this->options['fallback_store_scope'])) {
$this->fallbackStoreScope = is_array($this->options['fallback_store_scope'])
? $this->options['fallback_store_scope']
: array_map('trim', explode(',', $this->options['fallback_store_scope']));
}
// Set up logger
if (is_a($this->options['logger'], LoggerInterface::class)) {
$this->setLogger($this->options['logger']);
@@ -159,6 +178,57 @@ class ObjectRetriever implements CustomCacheAndLogInterface {
$this->cache->save($this->getCacheKey($id), $entry, $ttl);
}
$this->putIntoFallbackStore($id, $data);
}
/**
* Get the file location of an object in the fallback store, or null if
* the fallback store is not configured or the object is out of scope.
*/
private function getFallbackStoreLocation($id) : ?string {
if (!isset($this->fallbackStoreScope))
return null;
// Only objects are stored, not custom data or attachments
$coid = new IRI($id);
if (!COIDParser::isValidCOID($coid) || $coid->getFragment() !== null || $coid->getQuery() !== null)
return null;
if (!in_array('*', $this->fallbackStoreScope)
&& !in_array($coid->getHost(), $this->fallbackStoreScope))
return null;
return $this->options['fallback_store_directory'].DIRECTORY_SEPARATOR.
$coid->getHost().str_replace('/', DIRECTORY_SEPARATOR, $coid->getPath())
.DIRECTORY_SEPARATOR.'object.jsonld';
}
private function putIntoFallbackStore($id, $data) {
$location = $this->getFallbackStoreLocation($id);
if (!isset($location))
return;
// Write to a temporary file and rename it so that concurrent readers
// never see a partially written object
$directory = dirname($location);
$tempFile = $location.'.'.uniqid('', true).'.tmp';
if ((!is_dir($directory) && !@mkdir($directory, 0777, true) && !is_dir($directory))
|| @file_put_contents($tempFile, $data) === false
|| !@rename($tempFile, $location)) {
@unlink($tempFile);
if (isset($this->logger))
$this->logger->warning('Could not write <'.$id.'> to fallback store.');
}
}
private function getFromFallbackStore($id) : ?string {
$location = $this->getFallbackStoreLocation($id);
if (!isset($location) || !is_file($location))
return null;
$data = @file_get_contents($location);
return ($data === false) ? null : $data;
}
public function getFromCacheCustom($id) {
@@ -282,21 +352,44 @@ class ObjectRetriever implements CustomCacheAndLogInterface {
$object = (string)$response->getBody();
$this->putIntoCache($uriString, $object, $this->options['cache_ttl']);
$this->logInfoWithTime('Fetched <'.$uriString.'> from Core API ['.$response->getStatusCode().'].', $ts);
} catch (ConnectException $e) {
// connection failed or timed out
$object = $this->getFromFallbackStore($uriString);
if (!isset($object)) {
$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API (connection failed).', $ts);
throw new CoreAPIException($e->getMessage(), $e);
}
$this->logInfoWithTime('Fetched <'.$uriString.'> from fallback store (connection failed).', $ts);
} catch (RequestException $e) {
if ($e->hasResponse() && $e->getResponse()->getStatusCode() == 404) {
// not found
// not found (404)
$this->logInfoWithTime('Object <'.$uriString.'> not found in Core API.', $ts);
return null;
} else
if ($e->hasResponse()) {
// other API error
$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API ['.$e->getResponse()->getStatusCode().'].', $ts);
throw new CoreAPIException($e->getMessage(), $e);
if ($e->hasResponse() && $e->getResponse()->getStatusCode() >= 500) {
// other API error (5xx)
$object = $this->getFromFallbackStore($uriString);
$statusCode = $e->getResponse()->getStatusCode();
if (!isset($object)) {
$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API ['.$statusCode.'].', $ts);
throw new CoreAPIException($e->getMessage(), $e);
}
$this->logInfoWithTime('Fetched <'.$uriString.'> from fallback store (Core API returned ['.$statusCode.']).', $ts);
} else
if (!$e->hasResponse()) {
// API error without response
$object = $this->getFromFallbackStore($uriString);
if (!isset($object)) {
$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API (no status code).', $ts);
throw new CoreAPIException($e->getMessage(), $e);
}
$this->logInfoWithTime('Fetched <'.$uriString.'> from fallback store (Core API returned no status code).', $ts);
} else {
// API error without response
$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API (no status code).', $ts);
// other API error (e.g. 4xx)
$statusCode = $e->getResponse()->getStatusCode();
$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API ['.$statusCode.'].', $ts);
throw new CoreAPIException($e->getMessage(), $e);
}
}
} catch (Exception $e) {
// other exceptions
$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API (unexpected exception).', $ts);
docs/caching.md
+97 -2
View File
@@ -1,9 +1,18 @@
## Caching
# Standard Caching
### Standard Caching
Object descriptions that are requested through the ObjectRetriever are fetched through the CloudObjects Core Object API. They are cached in-memory inside the class for a default of 60 seconds. This can be changed using the `cache_ttl` option, which takes a number of seconds.
```php
use CloudObjects\SDK\ObjectRetriever;
// Keep objects in memory for 5 minutes
$retriever = new ObjectRetriever([
'cache_ttl' => 300,
]);
```
This in-class memory obviously only works as long as the class is alive and prevents subsequent requests for the same object hitting the API, but the cache will not exist between multiple requests handled in different processes. For this purpose, an external cache should be added.
To add an external cache, provide a storage using the `cache_storage` interface. The storage must be a class that implements either of the following interfaces:
@@ -15,4 +24,90 @@ To add an external cache, provide a storage using the `cache_storage` interface.
External storage is used twice:
- Storing the HTTP request according to the APIs caching headers.
- Storing the object data according to `cache_ttl`.
- Storing the object data according to `cache_ttl`.
All keys written to the external cache are prefixed with the value of the `cache_prefix` option (default: `clobj:`), so you can share a cache with other parts of your application.
The following examples use the [Symfony Cache component](https://symfony.com/doc/current/components/cache.html), but any library implementing one of the interfaces above works.
Using a PSR-6 cache pool, e.g. on the filesystem:
```php
use CloudObjects\SDK\ObjectRetriever;
use Symfony\Component\Cache\Adapter\FilesystemAdapter;
$retriever = new ObjectRetriever([
'cache_storage' => new FilesystemAdapter('cloudobjects', 0, '/var/cache/myapp'),
'cache_ttl' => 300,
]);
```
Using a PSR-16 simple cache, e.g. in Redis:
```php
use CloudObjects\SDK\ObjectRetriever;
use Symfony\Component\Cache\Adapter\RedisAdapter;
use Symfony\Component\Cache\Psr16Cache;
$redis = RedisAdapter::createConnection('redis://localhost');
$retriever = new ObjectRetriever([
'cache_storage' => new Psr16Cache(new RedisAdapter($redis)),
'cache_prefix' => 'myapp:clobj:',
]);
```
### Fallback Store
Fallback store is a mechanism that can be used by applications that require high availability but may lose connectivity to CloudObjects Core. With fallback store, a copy of an object is saved in a directory. If the API fails with a 5xx error, times out, or the request fails without a response, an `ObjectRetriever` will return the stored copy instead. Be advised that fallback storage will, once configured, result in your application not noticing when the Object API is unreachable - at least for objects in the store - and will continue to deliver the saved copy even if the original in CloudObjects Core is changed or removed.
It is possible for the fallback store to be configured for all objects or only for objects in specified namespaces. Since the fallback store is never purged except by the SDK itself, it can grow large; hence it should be limited to specific namespaces in applications where many random objects may be retrieved.
To enable fallback store, set the `fallback_store_directory` option to a writeable path on the filesystem. Also, set `fallback_store_scope` to either the string `*` to save all retrieved objects, or a list of namespaces (domains only without `coid://` prefix); the list of namespaces can be provided as a comma-separated string or an array of strings.
Storing all retrieved objects:
```php
use CloudObjects\SDK\ObjectRetriever;
$retriever = new ObjectRetriever([
'fallback_store_directory' => '/var/lib/myapp/cloudobjects-fallback',
'fallback_store_scope' => '*',
]);
```
Storing only objects from specific namespaces, provided as an array:
```php
use CloudObjects\SDK\ObjectRetriever;
$retriever = new ObjectRetriever([
'fallback_store_directory' => '/var/lib/myapp/cloudobjects-fallback',
'fallback_store_scope' => [ 'example.com', 'cloudobjects.io' ],
]);
```
Or as a comma-separated string, which is convenient when the value comes from an environment variable:
```php
use CloudObjects\SDK\ObjectRetriever;
$retriever = new ObjectRetriever([
'fallback_store_directory' => getenv('FALLBACK_STORE_DIRECTORY'),
'fallback_store_scope' => 'example.com,cloudobjects.io',
]);
```
The fallback store can be combined with an external cache. The cache is still consulted first; the fallback store only comes into play when an object is neither cached nor retrievable from the API.
```php
use CloudObjects\SDK\ObjectRetriever;
use Symfony\Component\Cache\Adapter\FilesystemAdapter;
$retriever = new ObjectRetriever([
'cache_storage' => new FilesystemAdapter('cloudobjects', 0, '/var/cache/myapp'),
'cache_ttl' => 300,
'fallback_store_directory' => '/var/lib/myapp/cloudobjects-fallback',
'fallback_store_scope' => 'example.com',
]);
```
docs/options.md
+2
View File
@@ -14,3 +14,5 @@ The `ObjectRetriever` constructor accepts an associative array of options. All o
| `logger` | Logger implementing `Psr\Log\LoggerInterface`, which receives info messages about object retrieval. | `null` |
| `timeout` | Total timeout for API requests, in seconds. | `20` |
| `connect_timeout` | Timeout for establishing a connection to the API, in seconds. | `5` |
| `fallback_store_directory` | Writeable directory in which copies of retrieved objects are saved and returned when the API fails with a 5xx error, times out, or the request fails without a response. Requires `fallback_store_scope`. See [caching](caching.md#fallback-store). | `null` |
| `fallback_store_scope` | Which objects are saved in the fallback store: `'*'` for all objects, or a list of namespaces (domains without `coid://` prefix) as a comma-separated string or an array of strings. | `null` |
tests/OfflineTests/ObjectRetrieverMockTest.php
+177 -1
View File
@@ -9,7 +9,7 @@ namespace CloudObjects\SDK;
use ML\IRI\IRI;
use GuzzleHttp\Client, GuzzleHttp\Handler\MockHandler,
GuzzleHttp\HandlerStack, GuzzleHttp\Psr7\Request, GuzzleHttp\Psr7\Response;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Exception\ConnectException, GuzzleHttp\Exception\RequestException;
use CloudObjects\SDK\Exceptions\CoreAPIException;
class ObjectRetrieverMockTest extends \PHPUnit\Framework\TestCase {
@@ -90,4 +90,180 @@ class ObjectRetrieverMockTest extends \PHPUnit\Framework\TestCase {
}
}
private function createFallbackStoreDirectory() : string {
$directory = sys_get_temp_dir().DIRECTORY_SEPARATOR.'clobj-fallback-'.uniqid();
mkdir($directory);
return $directory;
}
private function removeDirectory(string $directory) {
$files = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($directory, \FilesystemIterator::SKIP_DOTS),
\RecursiveIteratorIterator::CHILD_FIRST
);
foreach ($files as $file)
$file->isDir() ? rmdir($file->getPathname()) : unlink($file->getPathname());
rmdir($directory);
}
public function testFallbackStoreSavesObjectInScope() {
$directory = $this->createFallbackStoreDirectory();
$this->retriever = new ObjectRetriever([
'fallback_store_directory' => $directory,
'fallback_store_scope' => 'example.com, cloudobjects.io',
]);
$body = '{"@id":"coid://cloudobjects.io/Test","http://www.w3.org/2000/01/rdf-schema#label":"Test"}';
$this->setMockResponse(new Response(200, ['Content-Type' => 'application/ld+json'], $body));
try {
$this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test'));
$location = $directory.'/cloudobjects.io/Test/object.jsonld';
$this->assertFileExists($location);
$this->assertEquals($body, file_get_contents($location));
} finally {
$this->removeDirectory($directory);
}
}
public function testFallbackStoreSkipsObjectOutOfScope() {
$directory = $this->createFallbackStoreDirectory();
$this->retriever = new ObjectRetriever([
'fallback_store_directory' => $directory,
'fallback_store_scope' => [ 'example.com' ],
]);
$this->setMockResponse(new Response(200, ['Content-Type' => 'application/ld+json'],
'{"@id":"coid://cloudobjects.io/Test"}'));
try {
$this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test'));
$this->assertFileDoesNotExist($directory.'/cloudobjects.io/Test/object.jsonld');
} finally {
$this->removeDirectory($directory);
}
}
public function testFallbackStoreSavesAllObjectsWithWildcard() {
$directory = $this->createFallbackStoreDirectory();
$this->retriever = new ObjectRetriever([
'fallback_store_directory' => $directory,
'fallback_store_scope' => '*',
]);
$this->setMockResponse(new Response(200, ['Content-Type' => 'application/ld+json'],
'{"@id":"coid://cloudobjects.io"}'));
try {
$this->retriever->getObjectNode(new IRI('coid://cloudobjects.io'));
$this->assertFileExists($directory.'/cloudobjects.io/object.jsonld');
} finally {
$this->removeDirectory($directory);
}
}
private function createRetrieverWithStoredObject(string $directory) {
$this->retriever = new ObjectRetriever([
'fallback_store_directory' => $directory,
'fallback_store_scope' => '*',
]);
mkdir($directory.'/cloudobjects.io/Test', 0777, true);
file_put_contents($directory.'/cloudobjects.io/Test/object.jsonld',
'{"@id":"coid://cloudobjects.io/Test","http://www.w3.org/2000/01/rdf-schema#label":"Stored"}');
}
public function testFallbackStoreUsedWhenAPIReturns500() {
$directory = $this->createFallbackStoreDirectory();
$this->createRetrieverWithStoredObject($directory);
$this->setMockResponse(new Response(500));
try {
$object = $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test'));
$this->assertNotNull($object);
$this->assertEquals('Stored', $object->getProperty('http://www.w3.org/2000/01/rdf-schema#label')->getValue());
} finally {
$this->removeDirectory($directory);
}
}
public function testFallbackStoreUsedWhenAPITimesOut() {
$directory = $this->createFallbackStoreDirectory();
$this->createRetrieverWithStoredObject($directory);
$this->setMockResponse(new ConnectException(
'cURL error 28: Operation timed out',
new Request('GET', 'cloudobjects.io')
));
try {
$object = $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test'));
$this->assertNotNull($object);
$this->assertEquals('Stored', $object->getProperty('http://www.w3.org/2000/01/rdf-schema#label')->getValue());
} finally {
$this->removeDirectory($directory);
}
}
public function testFallbackStoreUsedWhenRequestFailsWithoutResponse() {
$directory = $this->createFallbackStoreDirectory();
$this->createRetrieverWithStoredObject($directory);
$this->setMockResponse(new RequestException(
'Request failed',
new Request('GET', 'cloudobjects.io')
));
try {
$object = $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test'));
$this->assertNotNull($object);
$this->assertEquals('Stored', $object->getProperty('http://www.w3.org/2000/01/rdf-schema#label')->getValue());
} finally {
$this->removeDirectory($directory);
}
}
public function testErrorWhenRequestFailsWithoutResponseAndObjectNotInFallbackStore() {
$directory = $this->createFallbackStoreDirectory();
$this->retriever = new ObjectRetriever([
'fallback_store_directory' => $directory,
'fallback_store_scope' => '*',
]);
$this->setMockResponse(new RequestException(
'Request failed',
new Request('GET', 'cloudobjects.io')
));
try {
$this->expectException(CoreAPIException::class);
$this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test'));
} finally {
$this->removeDirectory($directory);
}
}
public function testFallbackStoreNotUsedOnClientError() {
$directory = $this->createFallbackStoreDirectory();
$this->createRetrieverWithStoredObject($directory);
$this->setMockResponse(new Response(403));
try {
$this->expectException(CoreAPIException::class);
$this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test'));
} finally {
$this->removeDirectory($directory);
}
}
public function testErrorWhenAPIThrowsErrorAndObjectNotInFallbackStore() {
$directory = $this->createFallbackStoreDirectory();
$this->retriever = new ObjectRetriever([
'fallback_store_directory' => $directory,
'fallback_store_scope' => '*',
]);
$this->setMockResponse(new Response(500));
try {
$this->expectException(CoreAPIException::class);
$this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test'));
} finally {
$this->removeDirectory($directory);
}
}
}
tests/OfflineTests/ObjectRetrieverTest.php
+29
View File
@@ -0,0 +1,29 @@
<?php
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
namespace CloudObjects\SDK;
use CloudObjects\SDK\Exceptions\InvalidSDKConfigurationException;
class ObjectRetrieverTest extends \PHPUnit\Framework\TestCase {
public function testFallbackStoreWithExistingDirectory() {
$retriever = new ObjectRetriever([
'fallback_store_directory' => sys_get_temp_dir(),
'fallback_store_scope' => '*',
]);
$this->assertInstanceOf(ObjectRetriever::class, $retriever);
}
public function testFallbackStoreWithMissingDirectory() {
$this->expectException(InvalidSDKConfigurationException::class);
new ObjectRetriever([
'fallback_store_directory' => sys_get_temp_dir().'/does-not-exist-'.uniqid(),
'fallback_store_scope' => '*',
]);
}
}