Merge pull request #42 from ubtue/zts_integrate_updates

Zts integrate updates

.github/workflows/ci.yml

@@ -0,0 +1,22 @@
+name: CI
+on: [push, pull_request]
+jobs:
+  build:
+    name: Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v2
+
+      - run: git submodule update --init --depth=1 modules/translate
+      - run: git submodule update --init --depth=1 modules/utilities
+      - run: git submodule update --init --depth=1 modules/zotero-schema
+      - run: git submodule update --init --depth=1 modules/translators
+
+      - name: Install Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: 16
+
+      - run: npm ci
+      - run: npm test

.gitmodules

@@ -1,6 +1,12 @@
 [submodule "modules/translators"]
 	path = modules/translators
-	url = https://github.com/ubtue/zotero-translators.git
-[submodule "modules/zotero"]
-	path = modules/zotero
-	url = https://github.com/ubtue/zotero.git
+	url = https://github.com/zotero/translators
+[submodule "modules/utilities"]
+	path = modules/utilities
+	url = https://github.com/zotero/utilities.git
+[submodule "modules/translate"]
+	path = modules/translate
+	url = https://github.com/ubtue/translate.git
+[submodule "modules/zotero-schema"]
+	path = modules/zotero-schema
+	url = https://github.com/zotero/zotero-schema.git

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:10.7.0
+FROM node:lts
 
 # install npm dependencies
 WORKDIR /app
@@ -10,7 +10,7 @@ RUN npm install
 COPY . /app
 
 # Cloning zotero translators repository
-RUN git clone --depth=1 https://github.com/zotero/translators /app/modules/translators/
+RUN git clone --depth=1 https://github.com/zotero/translators.git /app/modules/translators/
 
 # run the application
 EXPOSE 1969

README.md

@@ -1,6 +1,7 @@
 # Zotero Translation Server
 
-[![Build Status](https://travis-ci.com/zotero/translation-server.svg?branch=master)](https://travis-ci.com/zotero/translation-server)
+[![Build Status](https://img.shields.io/github/actions/workflow/status/zotero/translation-server/ci.yml?branch=master)](https://github.com/zotero/translation-server/actions)
+
 
 The Zotero translation server lets you use [Zotero translators](https://www.zotero.org/support/translators) without the Zotero client.
 
@@ -15,7 +16,8 @@ docker pull zotero/translation-server
 docker run -d -p 1969:1969 --rm --name translation-server zotero/translation-server
 ```
 
-This will pull the latest image from Docker Hub and run it as a background process on port 1969. Use `docker kill translation-server` to stop it.
+This will pull the latest image [from Docker Hub](https://registry.hub.docker.com/r/zotero/translation-server)
+and run it as a background process on port 1969. Use `docker kill translation-server` to stop it.
 
 ### Running from source
 
@@ -61,11 +63,26 @@ Deploy:
 ./lambda_deploy lambda_config.env
 ```
 
-You can view the API Gateway endpoint in the Outputs section of the CloudFormation stack in the AWS Console.
+You can view the API Gateway endpoint in the Outputs section of the console output.
+
+## User-Agent
+
+By default, translation-server uses a standard Chrome `User-Agent` string to maximize compatibility. This is fine for personal usage, but for a deployed service, it’s polite to customize `User-Agent` so that sites can identify requests and contact you in case of abuse.
+
+You can do this by setting the `USER_AGENT` environment variable:
+
+`USER_AGENT='my-custom-translation-server/2.0 ([email protected])' npm start`
+
+If you find that regular requests are being blocked with a fully custom user-agent string, you can also add an identifier and contact information to the end of a standard browser UA string:
+
+```
+export USER_AGENT='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 my-custom-translation-server/2.0 ([email protected])'
+npm start
+```
 
 ## Proxy Support
 
-You can configure `translation-server` to use a proxy server by setting the `HTTP_PROXY` and `HTTPS_PROXY` environment variables:
+You can configure translation-server to use a proxy server by setting the `HTTP_PROXY` and `HTTPS_PROXY` environment variables:
 
 `HTTP_PROXY=http://proxy.example.com:8080 HTTPS_PROXY=http://proxy.example.com:8080 npm start`
 

config/default.json → config/default.json5

@@ -5,11 +5,13 @@
 	"identifierSearchLambda": "", // Identifier search Lambda function for text search
 	"port": 1969,
 	"host": "0.0.0.0", // host to listen on
-    "textSearchTimeout": 100,
-    "serverTimeout": 900000, // Also handle long running requests (15 * 60 * 1000
+        "httpMaxResponseSize": 10, // Max size of requested response to load; triggers 400 ResponseSize error when exceeded
+        "textSearchTimeout": 100,
+        "serverTimeout": 900000, // Also handle long running requests (15 * 60 * 1000
 	"translators": {
 		"CrossrefREST.email": "" // Pass an email to Crossref REST API to utilize the faster servers pool
 	},
+	"trustProxyHeaders": false, // Trust X-Forwarded-For when logging requests
 	"userAgent": "ub_tools/zotero (see https://ixtheo.de/crawler)",
 	"translatorsDirectory": "./modules/translators",
 	"persistentCookies": true, // Remember upstream cookies between translation operations. Do not enable for a public service.

docker-entrypoint.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 
-# will be deprecated when https://github.com/zotero/translation-server-v2/issues/1 will be released
+# will be deprecated when https://github.com/zotero/translation-server/issues/1 will be released
 echo "-> Updating zotero translators"
 cd /app/modules/translators/
-git pull origin master
+git pull --ff-only origin master
 
 cd /app/
 exec npm start

lambda_deploy

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+if [ -z $1 ]; then
+	echo "Usage: $0 config.env" >&2
+	exit 1
+fi
+
+set -a
+. "$dir/$1"
+set +a
+
+./lambda_package
+
+j2 "$dir/lambda_template.yaml.j2" > "$dir/package/template.yaml"
+
+sam package --template-file "$dir/package/template.yaml" --s3-bucket $deployment_bucket_name --output-template-file "$dir/package/package.yaml"
+aws cloudformation deploy --template-file "$dir/package/package.yaml" --stack-name $stack_name --capabilities CAPABILITY_IAM
+aws cloudformation describe-stacks --stack-name $stack_name
+rm -rf "$dir/package"

lambda_package

@@ -1,3 +1,6 @@
+rm -rf package
+mkdir package
+
 # Skip files that aren't required i.e. ".git"
 cp -R config package/
 cp -R modules package/

lambda_template.yaml.j2

@@ -10,7 +10,7 @@ Resources:
       FunctionName: {{ main_function_name }}
       CodeUri: ./
       Handler: src/lambda.handler
-      Runtime: nodejs12.x
+      Runtime: nodejs14.x
       MemorySize: 2048
       Timeout: 30
       Policies: