From 1f938cf19d5a9d26c96b1cd5a78cae807e1e295b Mon Sep 17 00:00:00 2001 From: Lukas Rosenstock Date: Sun, 18 Oct 2026 17:10:08 +0000 Subject: [PATCH] Added permanent store feature --- CloudObjects/SDK/ObjectRetriever.php | 91 +++++++++++++--- .../OfflineTests/ObjectRetrieverMockTest.php | 100 ++++++++++++++++++ tests/OfflineTests/ObjectRetrieverTest.php | 26 +++++ 3 files changed, 200 insertions(+), 17 deletions(-) diff --git a/CloudObjects/SDK/ObjectRetriever.php b/CloudObjects/SDK/ObjectRetriever.php index 2b5d66b..8a7c55b 100644 --- a/CloudObjects/SDK/ObjectRetriever.php +++ b/CloudObjects/SDK/ObjectRetriever.php @@ -43,6 +43,7 @@ class ObjectRetriever implements CustomCacheAndLogInterface { private $defaultReader; private $customCacheDefaultRequest; private $fallbackStoreScope; + private $permanentStoreScope; const CO_API_URL = 'https://od.coid.link/'; @@ -60,7 +61,9 @@ class ObjectRetriever implements CustomCacheAndLogInterface { 'timeout' => 20, 'connect_timeout' => 5, 'fallback_store_directory' => null, - 'fallback_store_scope' => null + 'fallback_store_scope' => null, + 'permanent_store_directory' => null, + 'permanent_store_scope' => null ], $options); // Check object cache configuration @@ -95,6 +98,27 @@ class ObjectRetriever implements CustomCacheAndLogInterface { : array_map('trim', explode(',', $this->options['fallback_store_scope'])); } + // Check permanent store configuration + if (isset($this->options['permanent_store_directory']) + && isset($this->options['permanent_store_scope']) + && !is_dir($this->options['permanent_store_directory']) + ) { + throw new InvalidSDKConfigurationException('Invalid permanent_store_directory specified; must be an existing directory.'); + } + + if (isset($this->options['permanent_store_directory']) + && isset($this->options['permanent_store_scope'])) { + $this->permanentStoreScope = is_array($this->options['permanent_store_scope']) + ? $this->options['permanent_store_scope'] + : array_map('trim', explode(',', $this->options['permanent_store_scope'])); + } + + if (isset($this->fallbackStoreScope) && isset($this->permanentStoreScope) + && realpath($this->options['fallback_store_directory']) === realpath($this->options['permanent_store_directory']) + ) { + throw new InvalidSDKConfigurationException('The permanent_store_directory and the fallback_store_directory must be different directories.'); + } + // Set up logger if (is_a($this->options['logger'], LoggerInterface::class)) { $this->setLogger($this->options['logger']); @@ -180,35 +204,32 @@ class ObjectRetriever implements CustomCacheAndLogInterface { } $this->putIntoFallbackStore($id, $data); + $this->putIntoPermanentStore($id, $data); } /** - * Get the file location of an object in the fallback store, or null if - * the fallback store is not configured or the object is out of scope. + * Get the file location of an object in a store, or null if the store + * is not configured or the object is out of scope. */ - private function getFallbackStoreLocation($id) : ?string { - if (!isset($this->fallbackStoreScope)) + private function getStoreLocation($directory, ?array $scope, $id) : ?string { + if (!isset($scope)) return null; // Only objects are stored, not custom data or attachments $coid = new IRI($id); - if (!COIDParser::isValidCOID($coid) || $coid->getFragment() !== null || $coid->getQuery() !== null) + if (!COIDParser::isValidCOID($coid)) return null; - if (!in_array('*', $this->fallbackStoreScope) - && !in_array($coid->getHost(), $this->fallbackStoreScope)) + if (!in_array('*', $scope) + && !in_array($coid->getHost(), $scope)) return null; - return $this->options['fallback_store_directory'].DIRECTORY_SEPARATOR. + return $directory.DIRECTORY_SEPARATOR. $coid->getHost().str_replace('/', DIRECTORY_SEPARATOR, $coid->getPath()) .DIRECTORY_SEPARATOR.'object.jsonld'; } - private function putIntoFallbackStore($id, $data) { - $location = $this->getFallbackStoreLocation($id); - if (!isset($location)) - return; - + private function putIntoStore($location, $id, $data, $storeName) { // Write to a temporary file and rename it so that concurrent readers // never see a partially written object $directory = dirname($location); @@ -218,12 +239,11 @@ class ObjectRetriever implements CustomCacheAndLogInterface { || !@rename($tempFile, $location)) { @unlink($tempFile); if (isset($this->logger)) - $this->logger->warning('Could not write <'.$id.'> to fallback store.'); + $this->logger->warning('Could not write <'.$id.'> to '.$storeName.' store.'); } } - private function getFromFallbackStore($id) : ?string { - $location = $this->getFallbackStoreLocation($id); + private function getFromStore($location) : ?string { if (!isset($location) || !is_file($location)) return null; @@ -231,6 +251,37 @@ class ObjectRetriever implements CustomCacheAndLogInterface { return ($data === false) ? null : $data; } + private function getFallbackStoreLocation($id) : ?string { + return $this->getStoreLocation($this->options['fallback_store_directory'], + $this->fallbackStoreScope, $id); + } + + private function putIntoFallbackStore($id, $data) { + $location = $this->getFallbackStoreLocation($id); + if (isset($location)) + $this->putIntoStore($location, $id, $data, 'fallback'); + } + + private function getFromFallbackStore($id) : ?string { + return $this->getFromStore($this->getFallbackStoreLocation($id)); + } + + private function getPermanentStoreLocation($id) : ?string { + return $this->getStoreLocation($this->options['permanent_store_directory'], + $this->permanentStoreScope, $id); + } + + private function putIntoPermanentStore($id, $data) { + // Objects in the permanent store are never overwritten + $location = $this->getPermanentStoreLocation($id); + if (isset($location) && !is_file($location)) + $this->putIntoStore($location, $id, $data, 'permanent'); + } + + private function getFromPermanentStore($id) : ?string { + return $this->getFromStore($this->getPermanentStoreLocation($id)); + } + public function getFromCacheCustom($id) { return $this->getFromCache('custom/'.$id); } @@ -343,6 +394,12 @@ class ObjectRetriever implements CustomCacheAndLogInterface { $this->logInfoWithTime('Fetched <'.$uriString.'> from object cache.', $ts); } + if (!isset($object)) { + $object = $this->getFromPermanentStore($uriString); + if (isset($object)) + $this->logInfoWithTime('Fetched <'.$uriString.'> from permanent store.', $ts); + } + if (!isset($object)) { try { $response = $this->client diff --git a/tests/OfflineTests/ObjectRetrieverMockTest.php b/tests/OfflineTests/ObjectRetrieverMockTest.php index 7754a56..06dfe7e 100644 --- a/tests/OfflineTests/ObjectRetrieverMockTest.php +++ b/tests/OfflineTests/ObjectRetrieverMockTest.php @@ -266,4 +266,104 @@ class ObjectRetrieverMockTest extends \PHPUnit\Framework\TestCase { } } + private function createRetrieverWithPermanentObject(string $directory, array $options = []) { + $this->retriever = new ObjectRetriever(array_merge([ + 'permanent_store_directory' => $directory, + 'permanent_store_scope' => '*', + ], $options)); + mkdir($directory.'/cloudobjects.io/Test', 0777, true); + file_put_contents($directory.'/cloudobjects.io/Test/object.jsonld', + '{"@id":"coid://cloudobjects.io/Test","http://www.w3.org/2000/01/rdf-schema#label":"Stored"}'); + } + + public function testPermanentStoreSavesObjectInScope() { + $directory = $this->createFallbackStoreDirectory(); + $this->retriever = new ObjectRetriever([ + 'permanent_store_directory' => $directory, + 'permanent_store_scope' => 'example.com, cloudobjects.io', + ]); + $body = '{"@id":"coid://cloudobjects.io/Test","http://www.w3.org/2000/01/rdf-schema#label":"Test"}'; + $this->setMockResponse(new Response(200, ['Content-Type' => 'application/ld+json'], $body)); + + try { + $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test')); + + $location = $directory.'/cloudobjects.io/Test/object.jsonld'; + $this->assertFileExists($location); + $this->assertEquals($body, file_get_contents($location)); + } finally { + $this->removeDirectory($directory); + } + } + + public function testPermanentStoreSkipsObjectOutOfScope() { + $directory = $this->createFallbackStoreDirectory(); + $this->retriever = new ObjectRetriever([ + 'permanent_store_directory' => $directory, + 'permanent_store_scope' => [ 'example.com' ], + ]); + $this->setMockResponse(new Response(200, ['Content-Type' => 'application/ld+json'], + '{"@id":"coid://cloudobjects.io/Test"}')); + + try { + $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test')); + $this->assertFileDoesNotExist($directory.'/cloudobjects.io/Test/object.jsonld'); + } finally { + $this->removeDirectory($directory); + } + } + + public function testPermanentStoreUsedWithoutContactingAPI() { + $directory = $this->createFallbackStoreDirectory(); + $this->createRetrieverWithPermanentObject($directory); + $mock = new MockHandler([ new Response(500) ]); + $this->retriever->setClient(new Client(['handler' => HandlerStack::create($mock)])); + + try { + $object = $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test')); + $this->assertNotNull($object); + $this->assertEquals('Stored', $object->getProperty('http://www.w3.org/2000/01/rdf-schema#label')->getValue()); + // The mocked response has not been consumed + $this->assertCount(1, $mock); + } finally { + $this->removeDirectory($directory); + } + } + + public function testPermanentStoreNotOverwritten() { + $directory = $this->createFallbackStoreDirectory(); + $this->createRetrieverWithPermanentObject($directory); + $this->setMockResponse(new Response(200, ['Content-Type' => 'application/ld+json'], + '[{"@id":"coid://cloudobjects.io/Test","http://www.w3.org/2000/01/rdf-schema#label":"Updated"}]')); + + try { + $this->retriever->fetchAllObjectsInNamespace(new IRI('coid://cloudobjects.io')); + $this->assertStringContainsString('Stored', + file_get_contents($directory.'/cloudobjects.io/Test/object.jsonld')); + } finally { + $this->removeDirectory($directory); + } + } + + public function testPermanentStoreTakesPrecedenceOverFallbackStore() { + $permanentDirectory = $this->createFallbackStoreDirectory(); + $fallbackDirectory = $this->createFallbackStoreDirectory(); + $this->createRetrieverWithPermanentObject($permanentDirectory, [ + 'fallback_store_directory' => $fallbackDirectory, + 'fallback_store_scope' => '*', + ]); + mkdir($fallbackDirectory.'/cloudobjects.io/Test', 0777, true); + file_put_contents($fallbackDirectory.'/cloudobjects.io/Test/object.jsonld', + '{"@id":"coid://cloudobjects.io/Test","http://www.w3.org/2000/01/rdf-schema#label":"Fallback"}'); + $this->setMockResponse(new Response(500)); + + try { + $object = $this->retriever->getObjectNode(new IRI('coid://cloudobjects.io/Test')); + $this->assertEquals('Stored', $object->getProperty('http://www.w3.org/2000/01/rdf-schema#label')->getValue()); + } finally { + $this->removeDirectory($permanentDirectory); + $this->removeDirectory($fallbackDirectory); + } + } + } diff --git a/tests/OfflineTests/ObjectRetrieverTest.php b/tests/OfflineTests/ObjectRetrieverTest.php index 338543d..b8e97d6 100644 --- a/tests/OfflineTests/ObjectRetrieverTest.php +++ b/tests/OfflineTests/ObjectRetrieverTest.php @@ -26,4 +26,30 @@ class ObjectRetrieverTest extends \PHPUnit\Framework\TestCase { ]); } + public function testPermanentStoreWithExistingDirectory() { + $retriever = new ObjectRetriever([ + 'permanent_store_directory' => sys_get_temp_dir(), + 'permanent_store_scope' => '*', + ]); + $this->assertInstanceOf(ObjectRetriever::class, $retriever); + } + + public function testPermanentStoreWithMissingDirectory() { + $this->expectException(InvalidSDKConfigurationException::class); + new ObjectRetriever([ + 'permanent_store_directory' => sys_get_temp_dir().'/does-not-exist-'.uniqid(), + 'permanent_store_scope' => '*', + ]); + } + + public function testPermanentStoreWithSameDirectoryAsFallbackStore() { + $this->expectException(InvalidSDKConfigurationException::class); + new ObjectRetriever([ + 'fallback_store_directory' => sys_get_temp_dir(), + 'fallback_store_scope' => '*', + 'permanent_store_directory' => sys_get_temp_dir(), + 'permanent_store_scope' => '*', + ]); + } + }