Merge pull request #284 from NYPL/SFR-1829_Complete_fulfill_endpoint

Sfr 1829 complete fulfill endpoint

.gitignore

@@ -13,3 +13,4 @@ config/local*
 tags
 # Vim
 *.swp
+/*venv*

CHANGELOG.md

@@ -2,6 +2,7 @@
 ## unreleased version -- v0.12.4
 ## Added
 New /fulfill endpoint with ability to check for NYPL login in Bearer authorization header
+Fulfill endpoint returns pre-signed URLs for objects in private buckets when user is logged in
 
 ## unreleased version -- v0.12.4
 ## Added

api/blueprints/drbFulfill.py

@@ -3,60 +3,86 @@
 
 import jwt
 
-from flask import Blueprint, request
+from flask import Blueprint, request, redirect, current_app
+from ..db import DBClient
 from ..utils import APIUtils
+from managers import S3Manager
 from logger import createLog
 
-JWT_ALGORITHM = ''
 logger = createLog(__name__)
 
-fulfill = Blueprint('fulfill', __name__, url_prefix='/fulfill')
+fulfill = Blueprint("fulfill", __name__, url_prefix="/fulfill")
 
-@fulfill.route('/<uuid>', methods=['GET'])
-def workFulfill(uuid):
-    logger.info('Checking if authorization is needed for work {}'.format(uuid))
 
-    requires_authorization = True
+@fulfill.route("/<link_id>", methods=["GET"])
+def itemFulfill(link_id):
+    with DBClient(current_app.config["DB_CLIENT"]) as dbClient:
+        link = dbClient.fetchSingleLink(link_id)
+        if not link:
+            return APIUtils.formatResponseObject(
+                404, "fulfill", "No link exists for this ID"
+            )
 
-    if requires_authorization:
-        try:
-            bearer = request.headers.get('Authorization')
-            token = bearer.split()[1]
+        requires_authorization = (
+            # Might not have edd flag if edd is not true
+            link.flags.get("edd", False) is False and link.flags["nypl_login"] is True
+        )
 
-            jwt_secret = os.environ['NYPL_API_CLIENT_PUBLIC_KEY']
-            decoded_token =(jwt.decode(token, jwt_secret, 'RS256', 
-                                   audience="app_myaccount"))
-            if json.loads(json.dumps(decoded_token))['iss'] == "https://www.nypl.org":
-                statusCode = 200
-                responseBody = uuid
+        if requires_authorization:
+            decodedToken = None
+            try:
+                bearer = request.headers.get("Authorization")
+                if bearer is None:
+                    return APIUtils.formatResponseObject(
+                        401,
+                        "fulfill",
+                        "Invalid access token",
+                        headers={"WWW-Authenticate": "Bearer"},
+                    )
+                token = bearer.split()[1]
+
+                jwt_secret = os.environ["NYPL_API_CLIENT_PUBLIC_KEY"]
+                decodedToken = jwt.decode(
+                    token, jwt_secret, algorithms=["RS256"], audience="app_myaccount"
+                )
+
+            except jwt.exceptions.ExpiredSignatureError:
+                return APIUtils.formatResponseObject(
+                    401,
+                    "fulfill",
+                    "Expired access token",
+                    headers={"WWW-Authenticate": "Bearer"},
+                )
+            except (
+                jwt.exceptions.DecodeError,
+                UnicodeDecodeError,
+                IndexError,
+                AttributeError,
+            ):
+                return APIUtils.formatResponseObject(
+                    401,
+                    "fulfill",
+                    "Invalid access token",
+                    headers={"WWW-Authenticate": "Bearer"},
+                )
+
+            if decodedToken["iss"] == "https://www.nypl.org":
+                storageManager = S3Manager()
+                storageManager.createS3Client()
+                presignedObjectUrl = APIUtils.getPresignedUrlFromObjectUrl(
+                    storageManager.s3Client, link.url
+                )
+                return redirect(presignedObjectUrl)
             else:
-                statusCode = 401
-                responseBody = 'Invalid access token'
-
-        except jwt.exceptions.ExpiredSignatureError:
-            statusCode = 401
-            responseBody = 'Expired access token'
-        except (jwt.exceptions.DecodeError, UnicodeDecodeError, IndexError, AttributeError):
-            statusCode = 401
-            responseBody = 'Invalid access token'
-        except ValueError:
-            logger.warning("Could not deserialize NYPL-issued public key")
-            statusCode = 500
-            responseBody = 'Server error'
-
-    else:
-        # TODO: In the future, this could record an analytics timestamp
-        # and redirect to URL of a work if authentication is not required. 
-        # For now, only use /fulfill endpoint in response if authentication is required.
-        statusCode = 400
-        responseBody = "Bad Request"
-
-    response = APIUtils.formatResponseObject(
-        statusCode, 'fulfill', responseBody
-    )
-
-    if statusCode == 401:
-        response[0].headers['WWW-Authenticate'] = 'Bearer'
-
-    return response
+                return APIUtils.formatResponseObject(
+                    401,
+                    "fulfill",
+                    "Invalid access token",
+                    headers={"WWW-Authenticate": "Bearer"},
+                )
 
+        else:
+            # TODO: In the future, this could record an analytics timestamp
+            # and redirect to URL of an item if authentication is not required.
+            # For now, only use /fulfill endpoint in response if authentication is required.
+            return APIUtils.formatResponseObject(400, "fulfill", "Bad request")

api/db.py

@@ -18,6 +18,13 @@ def createSession(self):
     def closeSession(self):
         self.session.close()
 
+    def __enter__(self):
+        self.createSession()
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        self.closeSession()
+
     def fetchSearchedWorks(self, ids):
         uuids = [i[0] for i in ids]
         editionIds = list(set(APIUtils.flatten([i[1] for i in ids])))

api/utils.py

@@ -8,6 +8,7 @@
 from model.postgres.collection import COLLECTION_EDITIONS
 from logger import createLog
 from botocore.exceptions import ClientError
+from urllib.parse import urlparse
 
 
 logger = createLog(__name__)
@@ -481,14 +482,16 @@ def flatten(cls, nested):
                 yield from cls.flatten(elem)
 
     @staticmethod
-    def formatResponseObject(status, responseType, datablock):
+    def formatResponseObject(status, responseType, datablock, headers = {}):
+        response = jsonify({
+            'status': status,
+            'timestamp': datetime.utcnow(),
+            'responseType': responseType,
+            'data': datablock
+        })
+        response.headers.extend(headers)
         return (
-            jsonify({
-                'status': status,
-                'timestamp': datetime.utcnow(),
-                'responseType': responseType,
-                'data': datablock
-            }),
+            response,
             status
         )
 
@@ -535,4 +538,35 @@ def generate_presigned_url(s3_client, client_method, method_parameters, expires_
                 "Couldn't get a presigned URL for client method '%s'.", client_method
             )
             raise
-        return url
+        return url
+
+    @staticmethod
+    def getPresignedUrlFromObjectUrl(s3Client, url):
+        """
+        Given the URL of an S3 resource, generate a presigned Amazon S3 URL
+        that can be used to access that resource.
+
+        :param s3_client: A Boto3 Amazon S3 client
+        :param url: The URL of the desired resource
+        """
+
+        if "//" not in url:
+            url = "//" + url
+
+        parsedUrl = urlparse(url)
+
+        if "s3" not in parsedUrl.hostname:
+            raise ValueError(
+                "s3 helper function given a non-s3 or malformed URL"
+            )
+
+        bucketName = parsedUrl.hostname.split('.')[0]
+        objectKey = parsedUrl.path[1:]
+        timeValid = 1000 * 30
+
+        return APIUtils.generate_presigned_url(
+            s3Client,
+            "get_object",
+            {'Bucket': bucketName,'Key': objectKey},
+            timeValid
+        )

config/sample-compose.yaml

@@ -41,14 +41,25 @@ HATHI_API_ROOT: https://babel.hathitrust.org/cgi/htd
 #HATHI_API_KEY: 
 #HATHI_API_SECRET: 
 
+# AWS CONFIGURATION
+#AWS_ACCESS:
+#AWS_SECRET:
+#AWS_REGION:
+
 # OCLC CONFIGURATION
 OCLC_QUERY_LIMIT: '390000'
 #OCLC_API_KEY: 
 
 # NYPL AUTH CONFIGURATION
-NYPL_API_CLIENT_PUBLIC_KEY: >
+NYPL_API_CLIENT_PUBLIC_KEY: |
   -----BEGIN PUBLIC KEY-----
-  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA44ilHg/PxcJYsISHMRyoxsmez178qZpkJVXg7rOMVTLZuf05an7Pl+lX4nw/rqcvGQDXyrimciLgLkWu00xhm6h6klTeJSNq2DgseF8OMw2olfuBKq1NBQ/vC8U0l5NJu34oSN4/iipgpovqAHHBGV4zDt0EWSXE5xpnBWi+w1NMAX/muB2QRfRxkkhueDkAmwKvz5MXJPay7FB/WRjf+7r2EN78x5iQKyCw0tpEZ5hpBX831SEnVULCnpFOcJWMPLdg0Ff6tBmgDxKQBVFIQ9RrzMLTqxKnVVn2+hVpk4F/8tMsGCdd4s/AJqEQBy5lsq7ji1B63XYqi5fc1SnJEQIDAQAB
+  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA44ilHg/PxcJYsISHMRyo
+  xsmez178qZpkJVXg7rOMVTLZuf05an7Pl+lX4nw/rqcvGQDXyrimciLgLkWu00xh
+  m6h6klTeJSNq2DgseF8OMw2olfuBKq1NBQ/vC8U0l5NJu34oSN4/iipgpovqAHHB
+  GV4zDt0EWSXE5xpnBWi+w1NMAX/muB2QRfRxkkhueDkAmwKvz5MXJPay7FB/WRjf
+  +7r2EN78x5iQKyCw0tpEZ5hpBX831SEnVULCnpFOcJWMPLdg0Ff6tBmgDxKQBVFI
+  Q9RrzMLTqxKnVVn2+hVpk4F/8tMsGCdd4s/AJqEQBy5lsq7ji1B63XYqi5fc1SnJ
+  EQIDAQAB
   -----END PUBLIC KEY-----
 
 # Bardo CCE API URL

tests/unit/test_api_fulfill_blueprint.py

@@ -3,53 +3,90 @@
 
 import jwt
 
-from api.blueprints.drbFulfill import workFulfill
+from api.blueprints.drbFulfill import itemFulfill, fulfill
 from api.utils import APIUtils
+from model.postgres.link import Link
 
 
 class TestSearchBlueprint:
     @pytest.fixture
     def mockUtils(self, mocker):
         return mocker.patch.multiple(
             APIUtils,
-            formatResponseObject=mocker.DEFAULT
+            formatResponseObject=mocker.DEFAULT,
+            getPresignedUrlFromObjectUrl="example.com/example.pdf"
         )
 
     @pytest.fixture
     def testApp(self):
         flaskApp = Flask('test')
         flaskApp.config['DB_CLIENT'] = 'testDBClient'
         flaskApp.config['READER_VERSION'] = 'test'
+        flaskApp.register_blueprint(fulfill)
 
         return flaskApp
 
-    def test_workFulfill_invalid_token(self, testApp, mockUtils, monkeypatch):
-        with testApp.test_request_context('/fulfill/12345', 
+    @pytest.fixture
+    def mockDB(self, mocker, monkeypatch):
+        mockDB = mocker.MagicMock()
+        mockDB.__enter__.return_value = mockDB
+        mockDB.fetchSingleLink.return_value = Link(flags = {"nypl_login": True},
+                            url="https://doc-example-bucket1.s3.us-west-2.amazonaws.com/puppy.png")
+        mocker.patch('api.blueprints.drbFulfill.DBClient').return_value = mockDB
+
+        monkeypatch.setenv('NYPL_API_CLIENT_PUBLIC_KEY', "SomeKeyValue")
+
+    @pytest.fixture
+    def mockS3(self, mocker):
+        mockS3Manager = mocker.MagicMock()
+        mocker.patch('api.blueprints.drbFulfill.S3Manager').return_value = mockS3Manager
+        return mockS3Manager
+
+    def test_itemFulfill_invalid_token(self, testApp, mockUtils, mockDB):
+        with testApp.test_request_context('/fulfill/12345',
                                           headers={'Authorization': 'Bearer Whatever'}):
-            monkeypatch.setenv('NYPL_API_CLIENT_PUBLIC_KEY', "SomeKeyValue")
-            workFulfill('12345')
+            itemFulfill('12345')
             mockUtils['formatResponseObject'].assert_called_once_with(
-                    401, 'fulfill', 'Invalid access token')
+                    401, 'fulfill', 'Invalid access token', headers={'WWW-Authenticate': 'Bearer'})
 
-    def test_workFulfill_no_bearer_auth(self, testApp, mockUtils):
-        with testApp.test_request_context('/fulfill/12345', 
+    def test_itemFulfill_no_bearer_auth(self, testApp, mockUtils, mockDB):
+        with testApp.test_request_context('/fulfill/12345',
                                           headers={'Authorization': 'Whatever'}):
-            workFulfill('12345')
+            itemFulfill('12345')
             mockUtils['formatResponseObject'].assert_called_once_with(
-                    401, 'fulfill', 'Invalid access token')
+                    401, 'fulfill', 'Invalid access token', headers={'WWW-Authenticate': 'Bearer'})
 
-    def test_workFulfill_empty_token(self, testApp, mockUtils):
-        with testApp.test_request_context('/fulfill/12345', 
+    def test_itemFulfill_empty_token(self, testApp, mockUtils, mockDB):
+        with testApp.test_request_context('/fulfill/12345',
                                           headers={'Authorization': ''}):
-            workFulfill('12345')
+            itemFulfill('12345')
             mockUtils['formatResponseObject'].assert_called_once_with(
-                    401, 'fulfill', 'Invalid access token')
+                    401, 'fulfill', 'Invalid access token', headers={'WWW-Authenticate': 'Bearer'})
 
-    def test_workFulfill_no_header(self, testApp, mockUtils):
+    def test_itemFulfill_no_header(self, testApp, mockUtils, mockDB):
         with testApp.test_request_context('/fulfill/12345'):
-            workFulfill('12345')
+            itemFulfill('12345')
             mockUtils['formatResponseObject'].assert_called_once_with(
-                    401, 'fulfill', 'Invalid access token')
-
+                    401, 'fulfill', 'Invalid access token', headers={'WWW-Authenticate': 'Bearer'})
 
-
+    def test_itemFulfill_invalid_iss(self, testApp, mockUtils, mockDB, mocker):
+        with testApp.test_request_context('/fulfill/12345'):
+            mocker.patch("jwt.decode", return_value={
+                "iss": "https://example.com"
+            })
+            itemFulfill('12345')
+            mockUtils['formatResponseObject'].assert_called_once_with(
+                    401, 'fulfill', 'Invalid access token', headers={'WWW-Authenticate': 'Bearer'})
+
+    def test_itemFulfill_redirect(self, testApp, mockS3, mockDB, mocker):
+        mocker.patch("api.utils.APIUtils.getPresignedUrlFromObjectUrl", return_value="example.com/example.pdf")
+        mocker.patch("jwt.decode", return_value={
+            "iss": "https://www.nypl.org"
+        })
+        response = testApp.test_client().get(
+            '/fulfill/12345',
+            follow_redirects=False,
+            headers={'Authorization': 'Bearer Whatever'}
+        )
+        assert response.status_code == 302
+        assert response.location == "example.com/example.pdf"