Added fallback store feature

CloudObjects/SDK/ObjectRetriever.php

@@ -11,7 +11,8 @@ use ML\IRI\IRI;
 use ML\JsonLD\JsonLD, ML\JsonLD\Node;
 use Psr\Log\LoggerInterface, Psr\Log\LoggerAwareTrait;
 use GuzzleHttp\ClientInterface, GuzzleHttp\Client, GuzzleHttp\HandlerStack;
-use GuzzleHttp\Exception\RequestException;
+use GuzzleHttp\Exception\ConnectException,
+	GuzzleHttp\Exception\RequestException;
 use GuzzleHttp\Psr7\Request,
 	GuzzleHttp\Psr7\Response;
 use Kevinrob\GuzzleCache\CacheMiddleware;
@@ -41,6 +42,7 @@ class ObjectRetriever implements CustomCacheAndLogInterface {
 	private $objects = [];
 	private $defaultReader;
 	private $customCacheDefaultRequest;
+	private $fallbackStoreScope;
 
 	const CO_API_URL = 'https://od.coid.link/';
 		
@@ -56,7 +58,9 @@ class ObjectRetriever implements CustomCacheAndLogInterface {
 			'api_base_url' => null,
 			'logger' => null,
 			'timeout' => 20,
-			'connect_timeout' => 5
+			'connect_timeout' => 5,
+			'fallback_store_directory' => null,
+			'fallback_store_scope' => null
 		], $options);
 
 		// Check object cache configuration
@@ -76,6 +80,21 @@ class ObjectRetriever implements CustomCacheAndLogInterface {
 			$this->cache = new Psr16CacheStorage($this->options['cache_storage']);
 		}
 
+		// Check fallback store configuration
+		if (isset($this->options['fallback_store_directory'])
+				&& isset($this->options['fallback_store_scope'])
+				&& !is_dir($this->options['fallback_store_directory'])
+			) {
+			throw new InvalidSDKConfigurationException('Invalid fallback_store_directory specified; must be an existing directory.');
+		}
+
+		if (isset($this->options['fallback_store_directory'])
+				&& isset($this->options['fallback_store_scope'])) {
+			$this->fallbackStoreScope = is_array($this->options['fallback_store_scope'])
+				? $this->options['fallback_store_scope']
+				: array_map('trim', explode(',', $this->options['fallback_store_scope']));
+		}
+
 		// Set up logger
 		if (is_a($this->options['logger'], LoggerInterface::class)) {
 			$this->setLogger($this->options['logger']);
@@ -159,6 +178,57 @@ class ObjectRetriever implements CustomCacheAndLogInterface {
 
 			$this->cache->save($this->getCacheKey($id), $entry, $ttl);
 		}
+
+		$this->putIntoFallbackStore($id, $data);
+	}
+
+	/**
+	 * Get the file location of an object in the fallback store, or null if
+	 * the fallback store is not configured or the object is out of scope.
+	 */
+	private function getFallbackStoreLocation($id) : ?string {
+		if (!isset($this->fallbackStoreScope))
+			return null;
+
+		// Only objects are stored, not custom data or attachments
+		$coid = new IRI($id);
+		if (!COIDParser::isValidCOID($coid) || $coid->getFragment() !== null || $coid->getQuery() !== null)
+			return null;
+
+		if (!in_array('*', $this->fallbackStoreScope)
+				&& !in_array($coid->getHost(), $this->fallbackStoreScope))
+			return null;
+
+		return $this->options['fallback_store_directory'].DIRECTORY_SEPARATOR.
+			$coid->getHost().str_replace('/', DIRECTORY_SEPARATOR, $coid->getPath())
+			.DIRECTORY_SEPARATOR.'object.jsonld';
+	}
+
+	private function putIntoFallbackStore($id, $data) {
+		$location = $this->getFallbackStoreLocation($id);
+		if (!isset($location))
+			return;
+
+		// Write to a temporary file and rename it so that concurrent readers
+		// never see a partially written object
+		$directory = dirname($location);
+		$tempFile = $location.'.'.uniqid('', true).'.tmp';
+		if ((!is_dir($directory) && !@mkdir($directory, 0777, true) && !is_dir($directory))
+				|| @file_put_contents($tempFile, $data) === false
+				|| !@rename($tempFile, $location)) {
+			@unlink($tempFile);
+			if (isset($this->logger))
+				$this->logger->warning('Could not write <'.$id.'> to fallback store.');
+		}
+	}
+
+	private function getFromFallbackStore($id) : ?string {
+		$location = $this->getFallbackStoreLocation($id);
+		if (!isset($location) || !is_file($location))
+			return null;
+
+		$data = @file_get_contents($location);
+		return ($data === false) ? null : $data;
 	}
 
 	public function getFromCacheCustom($id) {
@@ -282,21 +352,44 @@ class ObjectRetriever implements CustomCacheAndLogInterface {
 				$object = (string)$response->getBody();
 				$this->putIntoCache($uriString, $object, $this->options['cache_ttl']);
 				$this->logInfoWithTime('Fetched <'.$uriString.'> from Core API ['.$response->getStatusCode().'].', $ts);
+			} catch (ConnectException $e) {
+				// connection failed or timed out
+				$object = $this->getFromFallbackStore($uriString);
+				if (!isset($object)) {
+					$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API (connection failed).', $ts);
+					throw new CoreAPIException($e->getMessage(), $e);
+				}
+				$this->logInfoWithTime('Fetched <'.$uriString.'> from fallback store (connection failed).', $ts);
 			} catch (RequestException $e) {
 				if ($e->hasResponse() && $e->getResponse()->getStatusCode() == 404) {
-					// not found
+					// not found (404)
 					$this->logInfoWithTime('Object <'.$uriString.'> not found in Core API.', $ts);
 					return null;
 				} else
-				if ($e->hasResponse()) {
-					// other API error
-					$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API ['.$e->getResponse()->getStatusCode().'].', $ts);
-					throw new CoreAPIException($e->getMessage(), $e);
+				if ($e->hasResponse() && $e->getResponse()->getStatusCode() >= 500) {
+					// other API error (5xx)
+					$object = $this->getFromFallbackStore($uriString);
+					$statusCode = $e->getResponse()->getStatusCode();
+					if (!isset($object)) {
+						$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API ['.$statusCode.'].', $ts);
+						throw new CoreAPIException($e->getMessage(), $e);
+					}
+					$this->logInfoWithTime('Fetched <'.$uriString.'> from fallback store (Core API returned ['.$statusCode.']).', $ts);
+				} else
+				if (!$e->hasResponse()) {
+					// API error without response
+					$object = $this->getFromFallbackStore($uriString);
+					if (!isset($object)) {
+						$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API (no status code).', $ts);
+						throw new CoreAPIException($e->getMessage(), $e);
+					}
+					$this->logInfoWithTime('Fetched <'.$uriString.'> from fallback store (Core API returned no status code).', $ts);
 				} else {
-					// API error without response					
-					$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API (no status code).', $ts);
+					// other API error (e.g. 4xx)
+					$statusCode = $e->getResponse()->getStatusCode();
+					$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API ['.$statusCode.'].', $ts);
 					throw new CoreAPIException($e->getMessage(), $e);
-				}					
+				}
 			} catch (Exception $e) {
 				// other exceptions
 				$this->logInfoWithTime('Object <'.$uriString.'> could not be retrieved from Core API (unexpected exception).', $ts);