From 77721ce19c5d8e3db264fd0c7260710cc8347713 Mon Sep 17 00:00:00 2001 From: John Flatness Date: Tue, 28 May 2024 17:24:00 -0400 Subject: [PATCH 1/5] Add sigv4 version of s3 service --- .../Omeka/Service/Amazon/S3V4Auth.php | 317 ++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100755 application/libraries/Omeka/Service/Amazon/S3V4Auth.php diff --git a/application/libraries/Omeka/Service/Amazon/S3V4Auth.php b/application/libraries/Omeka/Service/Amazon/S3V4Auth.php new file mode 100755 index 000000000..5a9c4776f --- /dev/null +++ b/application/libraries/Omeka/Service/Amazon/S3V4Auth.php @@ -0,0 +1,317 @@ +_region = $region; + } else { + $this->_region = 'us-east-1'; + } + + parent::__construct($accessKey, $secretKey, $region); + } + /** + * Put file to S3 as object, using streaming + * + * @param string $path File name + * @param string $object Object name + * @param array $meta Metadata + * @return boolean + */ + public function putFileStream($path, $object, $meta=null) + { + $data = @fopen($path, "rb"); + if ($data === false) { + /** + * @see Zend_Service_Amazon_S3_Exception + */ + require_once 'Zend/Service/Amazon/S3/Exception.php'; + throw new Zend_Service_Amazon_S3_Exception("Cannot open file $path"); + } + + if (!is_array($meta)) { + $meta = array(); + } + + if (!isset($meta[self::S3_CONTENT_TYPE_HEADER])) { + $meta[self::S3_CONTENT_TYPE_HEADER] = self::getMimeType($path); + } + + if (!isset($meta['Content-MD5'])) { + $meta['Content-MD5'] = base64_encode(md5_file($path, true)); + } + + if (!isset($meta['x-amz-content-sha256'])) { + $meta['x-amz-content-sha256'] = hash_file('sha256', $path); + } + + return $this->putObject($object, $data, $meta); + } + /** + * Upload an object by a PHP string + * + * @param string $object Object name + * @param string|resource $data Object data (can be string or stream) + * @param array $meta Metadata + * @return boolean + */ + public function putObject($object, $data, $meta=null) + { + $object = $this->_fixupObjectName($object); + $headers = (is_array($meta)) ? $meta : array(); + + if(!is_resource($data)) { + $headers['Content-MD5'] = base64_encode(md5($data, true)); + } + $headers['Expect'] = '100-continue'; + + if (!isset($headers[self::S3_CONTENT_TYPE_HEADER])) { + $headers[self::S3_CONTENT_TYPE_HEADER] = self::getMimeType($object); + } + + $response = $this->_makeRequest('PUT', $object, null, $headers, $data); + + // Etags aren't always the MD5, so rely on S3 checking against our headers + if ($response->getStatus() == 200) { + return true; + } + + return false; + } + + /** + * Make a request to Amazon S3 + * + * @param string $method Request method + * @param string $path Path to requested object + * @param array $params Request parameters + * @param array $headers HTTP headers + * @param string|resource $data Request data + * @return Zend_Http_Response + */ + public function _makeRequest($method, $path='', $params=null, $headers=array(), $data=null) + { + $retry_count = 0; + + if (!is_array($headers)) { + $headers = array($headers); + } + + if (!isset($headers['x-amz-content-sha256'])) { + if (is_string($data)) { + $headers['x-amz-content-sha256'] = hash('sha256', $data); + } else if (is_resource($data)) { + throw new Exception('sha256 is required but was not passed for a stream'); + } else { + // body is empty, use sha256 of the empty string + $headers['x-amz-content-sha256'] = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; + } + } + + if(is_resource($data) && $method != 'PUT') { + /** + * @see Zend_Service_Amazon_S3_Exception + */ + require_once 'Zend/Service/Amazon/S3/Exception.php'; + throw new Zend_Service_Amazon_S3_Exception("Only PUT request supports stream data"); + } + + // Move this higher so the content-type is set when we sign the request + if (($method == 'PUT') && ($data !== null)) { + if (!isset($headers['Content-type'])) { + $headers['Content-type'] = self::getMimeType($path); + } + } + + // build the end point out + $parts = explode('/', $path, 2); + $endpoint = clone($this->_endpoint); + if ($parts[0]) { + // prepend bucket name to the hostname + $endpoint->setHost($parts[0].'.'.$endpoint->getHost()); + } + if (!empty($parts[1])) { + // ZF-10218, ZF-10122 + $pathparts = explode('?',$parts[1]); + $endpath = $pathparts[0]; + $endpoint->setPath('/'.$endpath); + + } + else { + $endpoint->setPath('/'); + if ($parts[0]) { + $path = $parts[0].'/'; + } + } + // the client will add the Host header for us, but we need to sign it + $headers['host'] = $endpoint->getHost(); + self::addSignature($method, $endpoint->getPath(), $headers); + unset($headers['host']); + + $client = self::getHttpClient(); + + $client->resetParameters(true); + $client->setUri($endpoint); + $client->setAuth(false); + // Work around buglet in HTTP client - it doesn't clean headers + // Remove when ZHC is fixed + /* + $client->setHeaders(array('Content-MD5' => null, + 'Content-Encoding' => null, + 'Expect' => null, + 'Range' => null, + 'x-amz-acl' => null, + 'x-amz-copy-source' => null, + 'x-amz-metadata-directive' => null)); + */ + $client->setHeaders($headers); + + + if (is_array($params)) { + foreach ($params as $name=>$value) { + $client->setParameterGet($name, $value); + } + } + + + if (($method == 'PUT') && ($data !== null)) { + $client->setRawData($data, $headers['Content-type']); + } + do { + $retry = false; + + $response = $client->request($method); + $response_code = $response->getStatus(); + + // Some 5xx errors are expected, so retry automatically + if ($response_code >= 500 && $response_code < 600 && $retry_count <= 5) { + $retry = true; + $retry_count++; + sleep($retry_count / 4 * $retry_count); + } + else if ($response_code == 307) { + // Need to redirect, new S3 endpoint given + // This should never happen as Zend_Http_Client will redirect automatically + } + else if ($response_code == 100) { + // echo 'OK to Continue'; + } + } while ($retry); + + return $response; + } + + /** + * Add the S3 Authorization signature to the request headers + * + * @param string $method + * @param string $path + * @param array &$headers + * @return string + */ + protected function addSignature($method, $path, &$headers) + { + $sha256 = $headers['x-amz-content-sha256']; + + // use the same time for all dates/timestamps + $time = time(); + $timestamp = gmdate('Ymd\THis\Z', $time); + $date = gmdate('Ymd', $time); + // set date here so the request matches the signature + $headers['x-amz-date'] = $timestamp; + + $canonicalURI = parse_url($path, PHP_URL_PATH); + + $query = parse_url($path, PHP_URL_QUERY); + $canonicalQueryString = ''; + if ($query) { + parse_str($query, $queryArr); + ksort($queryArr); + foreach ($queryArr as $key => $value) { + $canonicalQueryString .= rawurlencode($key) . '=' . rawurlencode($value) . '&'; + } + $canonicalQueryString = substr($canonicalQueryString, 0, -1); + } + + $canonicalHeadersArr = array(); + foreach ($headers as $header => $value) { + $lowerHeader = strtolower($header); + $canonicalHeadersArr[$lowerHeader] = $lowerHeader . ':' . trim($value); + } + ksort($canonicalHeadersArr); + $canonicalHeaders = implode("\n", $canonicalHeadersArr) . "\n"; + $signedHeaders = implode(';', array_keys($canonicalHeadersArr)); + + $canonicalRequestHash = hash('sha256', "$method\n$canonicalURI\n$canonicalQueryString\n$canonicalHeaders\n$signedHeaders\n$sha256"); + + $region = $this->_region; + $scope = "$date/$region/s3/aws4_request"; + $stringToSign = "AWS4-HMAC-SHA256\n$timestamp\n$scope\n$canonicalRequestHash"; + + // Signing key is the same request-to-request as long as it's the same date + if (!($this->_signingKey && $date === $this->_signingKeyDate)) { + $dateKey = hash_hmac('sha256', $date, 'AWS4' . $this->_getSecretKey(), true); + $dateRegionKey = hash_hmac('sha256', $region, $dateKey, true); + $dateRegionServiceKey = hash_hmac('sha256', 's3', $dateRegionKey, true); + $this->_signingKey = hash_hmac('sha256', 'aws4_request', $dateRegionServiceKey, true); + $this->_signingKeyDate = $date; + } + + $signature = hash_hmac('sha256', $stringToSign, $this->_signingKey); + + $headers['Authorization'] = 'AWS4-HMAC-SHA256 Credential=' . $this->_getAccessKey() . "/$date/$region/s3/aws4_request,SignedHeaders=$signedHeaders,Signature=$signature"; + } +} From 2e4b1bd942672a85eebdce21868d3be0e5f42087 Mon Sep 17 00:00:00 2001 From: John Flatness Date: Wed, 29 May 2024 15:48:04 -0400 Subject: [PATCH 2/5] Add sigv4 option for storage adapter --- .../Omeka/Storage/Adapter/ZendS3.php | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/application/libraries/Omeka/Storage/Adapter/ZendS3.php b/application/libraries/Omeka/Storage/Adapter/ZendS3.php index aadc30fb5..d4f179177 100644 --- a/application/libraries/Omeka/Storage/Adapter/ZendS3.php +++ b/application/libraries/Omeka/Storage/Adapter/ZendS3.php @@ -25,6 +25,8 @@ class Omeka_Storage_Adapter_ZendS3 implements Omeka_Storage_Adapter_AdapterInter const FORCE_SSL = 'forceSSL'; const ACLS_OPTION = 'acls'; const STORAGE_CLASS_OPTION = 'storageClass'; + const SIGV4_OPTION = 'sigV4'; + const REGION_OPTION = 'region'; const S3_STORAGE_CLASS_HEADER = 'x-amz-storage-class'; @@ -53,6 +55,11 @@ class Omeka_Storage_Adapter_ZendS3 implements Omeka_Storage_Adapter_AdapterInter */ private $_storageClass; + /** + * @var bool + */ + private $_sigV4 = false; + /** * Set options for the storage adapter. * @@ -62,6 +69,10 @@ public function __construct(array $options = array()) { $this->_options = $options; + if (isset($this->_options[self::SIGV4_OPTION])) { + $this->_sigV4 = (bool) $this->_options[self::SIGV4_OPTION]; + } + if (array_key_exists(self::AWS_KEY_OPTION, $options) && array_key_exists(self::AWS_SECRET_KEY_OPTION, $options)) { $awsKey = $options[self::AWS_KEY_OPTION]; @@ -79,7 +90,17 @@ public function __construct(array $options = array()) $client->setMaxRetries(3); Zend_Service_Amazon_S3::setHttpClient($client); - $this->_s3 = new Zend_Service_Amazon_S3($awsKey, $awsSecretKey); + if ($this->_sigV4) { + if (isset($this->_options[self::REGION_OPTION])) { + $region = $this->_options[self::REGION_OPTION]; + } else { + $region = null; + } + $this->_s3 = new Omeka_Service_Amazon_S3V4Auth($awsKey, $awsSecretKey, $region); + } else { + $this->_s3 = new Zend_Service_Amazon_S3($awsKey, $awsSecretKey); + } + if (!empty($options[self::ENDPOINT_OPTION])) { $this->_s3->setEndpoint($options[self::ENDPOINT_OPTION]); } @@ -95,6 +116,7 @@ public function __construct(array $options = array()) if (isset($this->_options[self::STORAGE_CLASS_OPTION])) { $this->_storageClass = $this->_options[self::STORAGE_CLASS_OPTION]; } + } public function setUp() From 106fdb3842a8e61e86fb72193479e1d41cb2a30d Mon Sep 17 00:00:00 2001 From: John Flatness Date: Wed, 29 May 2024 18:31:39 -0400 Subject: [PATCH 3/5] Clean up S3V4Auth class --- .../Omeka/Service/Amazon/S3V4Auth.php | 52 +------------------ 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/application/libraries/Omeka/Service/Amazon/S3V4Auth.php b/application/libraries/Omeka/Service/Amazon/S3V4Auth.php index 5a9c4776f..394d4164c 100755 --- a/application/libraries/Omeka/Service/Amazon/S3V4Auth.php +++ b/application/libraries/Omeka/Service/Amazon/S3V4Auth.php @@ -1,44 +1,7 @@ resetParameters(true); $client->setUri($endpoint); $client->setAuth(false); - // Work around buglet in HTTP client - it doesn't clean headers - // Remove when ZHC is fixed - /* - $client->setHeaders(array('Content-MD5' => null, - 'Content-Encoding' => null, - 'Expect' => null, - 'Range' => null, - 'x-amz-acl' => null, - 'x-amz-copy-source' => null, - 'x-amz-metadata-directive' => null)); - */ $client->setHeaders($headers); - if (is_array($params)) { foreach ($params as $name=>$value) { $client->setParameterGet($name, $value); } } - if (($method == 'PUT') && ($data !== null)) { $client->setRawData($data, $headers['Content-type']); } From 037a8d7653f3443e106a3369ca1e0ac1108ee3d2 Mon Sep 17 00:00:00 2001 From: John Flatness Date: Wed, 29 May 2024 19:54:19 -0400 Subject: [PATCH 4/5] S3: refactor for v4 presigned URL support --- .../Omeka/Service/Amazon/S3V4Auth.php | 59 ++++++++++++++++--- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/application/libraries/Omeka/Service/Amazon/S3V4Auth.php b/application/libraries/Omeka/Service/Amazon/S3V4Auth.php index 394d4164c..d191fb974 100755 --- a/application/libraries/Omeka/Service/Amazon/S3V4Auth.php +++ b/application/libraries/Omeka/Service/Amazon/S3V4Auth.php @@ -216,10 +216,10 @@ protected function addSignature($method, $path, &$headers) { $sha256 = $headers['x-amz-content-sha256']; - // use the same time for all dates/timestamps - $time = time(); - $timestamp = gmdate('Ymd\THis\Z', $time); - $date = gmdate('Ymd', $time); + $timestamp = gmdate('Ymd\THis\Z'); + $date = substr($timestamp, 0, 8); + $region = $this->_region; + // set date here so the request matches the signature $headers['x-amz-date'] = $timestamp; @@ -245,9 +245,54 @@ protected function addSignature($method, $path, &$headers) $canonicalHeaders = implode("\n", $canonicalHeadersArr) . "\n"; $signedHeaders = implode(';', array_keys($canonicalHeadersArr)); - $canonicalRequestHash = hash('sha256', "$method\n$canonicalURI\n$canonicalQueryString\n$canonicalHeaders\n$signedHeaders\n$sha256"); + $signature = $this->_getSignature($method, $canonicalURI, $canonicalQueryString, $canonicalHeaders, $signedHeaders, $sha256, $timestamp, $date, $region); + + $headers['Authorization'] = 'AWS4-HMAC-SHA256 Credential=' . $this->_getAccessKey() . "/$date/$region/s3/aws4_request,SignedHeaders=$signedHeaders,Signature=$signature"; + } + /** + * Get the query string for a presigned SigV4 URL for the given path + * + * @param string $path Path portion of the URL (including the leading slash) + * @param int $expires Time, in seconds, the URL should be valid for (max is 7 days) + * @return string + */ + public function getPresignedURLQuery($path, $expires) + { + $timestamp = gmdate('Ymd\THis\Z'); + $date = substr($timestamp, 0, 8); $region = $this->_region; + + $accessKey = rawurlencode($this->_getAccessKey()); + $region = rawurlencode($this->_region); + $expires = (int) $expires; + + $query = "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=$accessKey%2F$date%2F$region%2Fs3%2Faws4_request&X-Amz-Date=$timestamp&X-Amz-Expires=$expires&X-Amz-SignedHeaders=host"; + $headers = 'host:' . $this->_endpoint->getHost() . "\n"; + $signature = $this->_getSignature('GET', $path, $query, $headers, 'host', 'UNSIGNED-PAYLOAD', $timestamp, $date, $region); + $query .= "&X-Amz-Signature=$signature"; + + return $query; + } + + /** + * Get signature for a given request + * + * @param string $method + * @param string $canonicalURI + * @param string $canonicalQueryString + * @param string $canonicalHeaders + * @param string $signedHeaders + * @param string $sha256 + * @param string $timestamp + * @param string $date + * @param string $region + * @return string + */ + protected function _getSignature($method, $canonicalURI, $canonicalQueryString, $canonicalHeaders, $signedHeaders, $sha256, $timestamp, $date, $region) + { + $canonicalRequestHash = hash('sha256', "$method\n$canonicalURI\n$canonicalQueryString\n$canonicalHeaders\n$signedHeaders\n$sha256"); + $scope = "$date/$region/s3/aws4_request"; $stringToSign = "AWS4-HMAC-SHA256\n$timestamp\n$scope\n$canonicalRequestHash"; @@ -260,8 +305,6 @@ protected function addSignature($method, $path, &$headers) $this->_signingKeyDate = $date; } - $signature = hash_hmac('sha256', $stringToSign, $this->_signingKey); - - $headers['Authorization'] = 'AWS4-HMAC-SHA256 Credential=' . $this->_getAccessKey() . "/$date/$region/s3/aws4_request,SignedHeaders=$signedHeaders,Signature=$signature"; + return hash_hmac('sha256', $stringToSign, $this->_signingKey); } } From 1ce89ad5c0ffbc28272b495b7b556879ab931465 Mon Sep 17 00:00:00 2001 From: John Flatness Date: Wed, 29 May 2024 19:54:54 -0400 Subject: [PATCH 5/5] S3: connect v4 presigned URL support to storage --- .../Omeka/Storage/Adapter/ZendS3.php | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/application/libraries/Omeka/Storage/Adapter/ZendS3.php b/application/libraries/Omeka/Storage/Adapter/ZendS3.php index d4f179177..8d902156e 100644 --- a/application/libraries/Omeka/Storage/Adapter/ZendS3.php +++ b/application/libraries/Omeka/Storage/Adapter/ZendS3.php @@ -221,30 +221,46 @@ public function getUri($path) $uri = "$endpoint/$object"; if ($expiration = $this->_getExpiration()) { - $timestamp = time(); - $expirationSeconds = $expiration * 60; - $expires = $timestamp + $expirationSeconds; - // "Chunk" expirations to allow browser caching - $expires = $expires + $expirationSeconds - ($expires % $expirationSeconds); + if ($this->_sigV4) { + $uri .= '?' . $this->_getPresignedQueryV4("/$object", $expiration); + } else { + $uri .= '?' . $this->_getPresignedQueryV2("/$object", $expiration); + } + } + + return $uri; + } - $accessKeyId = $this->_options[self::AWS_KEY_OPTION]; - $secretKey = $this->_options[self::AWS_SECRET_KEY_OPTION]; + protected function _getPresignedQueryV2($path, $expiration) + { + $timestamp = time(); + $expirationSeconds = $expiration * 60; + $expires = $timestamp + $expirationSeconds; + // "Chunk" expirations to allow browser caching + $expires = $expires + $expirationSeconds - ($expires % $expirationSeconds); - $stringToSign = "GET\n\n\n$expires\n/$object"; + $accessKeyId = $this->_options[self::AWS_KEY_OPTION]; + $secretKey = $this->_options[self::AWS_SECRET_KEY_OPTION]; - $signature = base64_encode( - Zend_Crypt_Hmac::compute($secretKey, 'sha1', $stringToSign, Zend_Crypt_Hmac::BINARY)); + $stringToSign = "GET\n\n\n$expires\n$path"; - $query['AWSAccessKeyId'] = $accessKeyId; - $query['Expires'] = $expires; - $query['Signature'] = $signature; + $signature = base64_encode( + Zend_Crypt_Hmac::compute($secretKey, 'sha1', $stringToSign, Zend_Crypt_Hmac::BINARY)); - $queryString = http_build_query($query); + $query['AWSAccessKeyId'] = $accessKeyId; + $query['Expires'] = $expires; + $query['Signature'] = $signature; - $uri .= "?$queryString"; - } + return http_build_query($query); + } - return $uri; + protected function _getPresignedQueryV4($path, $expiration) + { + $expirationSeconds = $expiration * 60; + // SigV4 expirations are limited to 7 days + $expires = min($expirationSeconds, 604800); + + return $this->_s3->getPresignedURLQuery($path, $expires); } /**