From a555d6f31cb7c6c168c903da0f90ff5c61eebc34 Mon Sep 17 00:00:00 2001 From: "J. Bukhari" Date: Tue, 27 Feb 2024 20:14:42 -0500 Subject: [PATCH 1/4] add Lambda function code --- aws-lambda/README.md | 21 +++++++++++++++++++++ aws-lambda/auth-event.json | 5 +++++ aws-lambda/bib-event.json | 5 +++++ aws-lambda/config.yaml | 13 +++++++++++++ aws-lambda/requirements.txt | 2 ++ aws-lambda/service.py | 13 +++++++++++++ requirements.txt | 2 +- setup.py | 4 +--- 8 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 aws-lambda/README.md create mode 100644 aws-lambda/auth-event.json create mode 100644 aws-lambda/bib-event.json create mode 100644 aws-lambda/config.yaml create mode 100644 aws-lambda/requirements.txt create mode 100644 aws-lambda/service.py diff --git a/aws-lambda/README.md b/aws-lambda/README.md new file mode 100644 index 0000000..538bd48 --- /dev/null +++ b/aws-lambda/README.md @@ -0,0 +1,21 @@ + +### About + +This is a Lambda function for running dlx-dl-sync. + +### Deployment + +This describes how to deploy this function to AWS Lambda using the Python library [python-lambda](https://pypi.org/project/python-lambda/). The files in this directory are the standard files necessary for deploying Python code to Lambda. + +AWS credentials must be configured in the deployment environment. + +1. Navigate to this directory (`dlx-dl/aws-lambda`) +2. Make sure a virtual environment (venv) is activated. +3. ```pip install -r requirements.txt``` +4. ```lambda deploy``` + +Note that once the Lambda function is deployed, it will not run until it is "invoked". [Several methods exist to invoke the function](https://docs.aws.amazon.com/lambda/latest/dg/lambda-invocation.html). + +### Running the Lambda function locally (for development/testing purposes) + +```lambda invoke -v --event-file=bib-event.json``` \ No newline at end of file diff --git a/aws-lambda/auth-event.json b/aws-lambda/auth-event.json new file mode 100644 index 0000000..e1cb67a --- /dev/null +++ b/aws-lambda/auth-event.json @@ -0,0 +1,5 @@ +{ + "source": "dlx-dl-lambda", + "type": "auth", + "modified_within": 86400 +} diff --git a/aws-lambda/bib-event.json b/aws-lambda/bib-event.json new file mode 100644 index 0000000..6a8f1d3 --- /dev/null +++ b/aws-lambda/bib-event.json @@ -0,0 +1,5 @@ +{ + "source": "dlx-dl-lambda", + "type": "bib", + "modified_within": 86400 +} diff --git a/aws-lambda/config.yaml b/aws-lambda/config.yaml new file mode 100644 index 0000000..262b01b --- /dev/null +++ b/aws-lambda/config.yaml @@ -0,0 +1,13 @@ +region: us-east-1 + +function_name: dlx-dl +handler: service.handler +description: DLX-DL Periodic Task Runner +runtime: python3.8 +role: service-role/dlx-dl-role-ggxqj9bh +timeout: 900 + +# Build options +build: + source_directories: lib # a comma delimited list of directories in your project root that contains source to package. + diff --git a/aws-lambda/requirements.txt b/aws-lambda/requirements.txt new file mode 100644 index 0000000..59d2ddb --- /dev/null +++ b/aws-lambda/requirements.txt @@ -0,0 +1,2 @@ +../. # installs dlx-dl from the directory above +python-lambda==11.8.0 \ No newline at end of file diff --git a/aws-lambda/service.py b/aws-lambda/service.py new file mode 100644 index 0000000..2d2bde3 --- /dev/null +++ b/aws-lambda/service.py @@ -0,0 +1,13 @@ +import json +from dlx_dl.scripts import sync + +def handler(event, context): + # event dict should contain the params expected by dlx.scripts.sync + + print(f'running with args: {json.dumps(event)}') + + try: + sync.run(**event) + except Exception as exc: + print('; '.join(str(exc).split('\n'))) # puts exception text on one line for CloudWatch logs + diff --git a/requirements.txt b/requirements.txt index ad8e978..f684332 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ cffi==1.15.1 charset-normalizer==2.1.1 click==8.1.3 cryptography==41.0.2 -dlx @ git+https://github.com/dag-hammarskjold-library/dlx@95972bbbd3505fc7be2c44cca85b71ca87f56464 +dlx @ git+https://github.com/dag-hammarskjold-library/dlx@b4f554c454449f23850941af53debd605fc033b6 idna==3.4 Jinja2==3.1.2 jmespath==1.0.1 diff --git a/setup.py b/setup.py index f1b5a9c..10eea15 100644 --- a/setup.py +++ b/setup.py @@ -16,9 +16,7 @@ license = 'http://www.opensource.org/licenses/bsd-license.php', packages = find_packages(exclude=['test']), test_suite = 'tests', - install_requires = [ - 'dlx @ git+https://github.com/dag-hammarskjold-library/dlx#egg=dlx' - ], + install_requires = requirements, description = 'Export data fom DLX to DL', long_description = long_description, long_description_content_type = "text/markdown", From f1a32ea0c949c85bf977afdafe075ffc739cce2a Mon Sep 17 00:00:00 2001 From: jbukhari Date: Tue, 27 Feb 2024 22:17:22 -0500 Subject: [PATCH 2/4] Create deploy.yml Deploys the AWS Lambda function on push to master --- .github/workflows/deploy.yml | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..9cdbda6 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,37 @@ +on: + push: + branches: [ master ] + +jobs: + deploy Lambda function: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: 3.10 + - name: Setting up AWS Credentials + run: | + pip install awscli + aws configure set region us-east-1 + aws configure set output json + aws configure set aws_access_key_id ${{secrets.AWS_ACCESS_KEY}} + aws configure set aws_secret_access_key ${{secrets.AWS_SECRET_KEY}} + - name: Create virtual environment + working-directory: ./aws-lambda + run: | + pip install virtualenv + python3 -m virtualenv venv + - name: Install requiremenents + working-directory: ./aws-lambda + run: | + . venv/bin/activate + python3 -m pip install --upgrade pip + pip install -r requirements.txt + deactivate + - name: Deploy + working-directory: ./aws-lambda + run: | + . venv/bin/activate + lambda deploy From 240c09d015fbf16bfd3048b772f6fd211ed13d97 Mon Sep 17 00:00:00 2001 From: Aaron Helton Date: Tue, 5 Mar 2024 08:52:41 -0500 Subject: [PATCH 3/4] add gitignore and install python-lambda in deploy.yml --- .github/workflows/deploy.yml | 1 + .gitignore | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 .gitignore diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9cdbda6..a3bbb44 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -29,6 +29,7 @@ jobs: . venv/bin/activate python3 -m pip install --upgrade pip pip install -r requirements.txt + pip install python-lambda deactivate - name: Deploy working-directory: ./aws-lambda diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..68f5ecc --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ + +venv/ +env/ +__pycache__/ +*.py[co] +.vscode +_build/ +/docs/_build/ +.DS_Store +.vs \ No newline at end of file From bdfad9aa942dd941bc98adae60f6e2c0cac9db16 Mon Sep 17 00:00:00 2001 From: Aaron Helton Date: Tue, 5 Mar 2024 15:01:22 -0500 Subject: [PATCH 4/4] fix requirements --- .github/workflows/deploy.yml | 1 - aws-lambda/requirements.txt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a3bbb44..9cdbda6 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -29,7 +29,6 @@ jobs: . venv/bin/activate python3 -m pip install --upgrade pip pip install -r requirements.txt - pip install python-lambda deactivate - name: Deploy working-directory: ./aws-lambda diff --git a/aws-lambda/requirements.txt b/aws-lambda/requirements.txt index 59d2ddb..6f6fd56 100644 --- a/aws-lambda/requirements.txt +++ b/aws-lambda/requirements.txt @@ -1,2 +1,2 @@ ../. # installs dlx-dl from the directory above -python-lambda==11.8.0 \ No newline at end of file +python-lambda @ git+https://github.com/nficano/python-lambda@2f9f17a5c5993e65ee2b61d06f29ed5a6689d337 \ No newline at end of file