From af726bd8559b9d7f83f126a17017696dadbd542e Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 14 May 2024 12:18:34 +0200 Subject: [PATCH] chore: add pre-commit lint to remove trailing whitespace --- .devcontainer/devcontainer.json | 2 +- .github/pull_request_template.md | 2 +- .github/workflows/build.yaml | 2 +- .github/workflows/deploy.yaml | 10 ++--- .github/workflows/distribute.yaml | 2 +- .github/workflows/lint.yaml | 26 +++++++++++++ .gitignore | 2 +- .pre-commit-config.yaml | 16 ++++++++ README.md | 2 +- integration_tests/cdk/README.md | 10 ++--- integration_tests/cdk/cdk.json | 2 +- integration_tests/cdk/package.json | 1 - lib/index.ts | 2 +- lib/ingestor-api/index.ts | 14 +++---- lib/ingestor-api/runtime/Dockerfile | 2 +- lib/stac-api/index.ts | 5 +-- lib/stac-api/runtime/Dockerfile | 2 +- lib/stac-api/runtime/requirements.txt | 2 +- lib/stac-browser/index.ts | 26 ++++++------- lib/tipg-api/index.ts | 10 ++--- lib/titiler-pgstac-api/index.ts | 47 +++++++++++------------ lib/titiler-pgstac-api/runtime/Dockerfile | 2 +- 22 files changed, 114 insertions(+), 75 deletions(-) create mode 100644 .github/workflows/lint.yaml create mode 100644 .pre-commit-config.yaml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index dd07802..4bfa3af 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,4 +1,4 @@ { "name": "VS Code DEV Container for AWS CDK development", "image": "jsii/superchain:1-buster-slim-node16" -} \ No newline at end of file +} diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 54eddf3..01aff37 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,4 @@ ## :warning: Checklist if your PR is changing anything else than documentation - [ ] Posted the link to a successful manually triggered deployment workflow (successful including the resources destruction) -## Merge request description \ No newline at end of file +## Merge request description diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4a206d1..3990633 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -12,7 +12,7 @@ on: required: false DS_RELEASE_BOT_PRIVATE_KEY: required: false - + jobs: build_and_package: name: Build and package diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 0455e7b..f662266 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -2,7 +2,7 @@ name: Deployment on: workflow_dispatch: - + jobs: build_package_and_deploy: name: Build, package and deploy @@ -30,7 +30,7 @@ jobs: - name: Generate distribution packages run: npm run package - + - name: Install deployment environment id: install_deploy_env run: | @@ -40,7 +40,7 @@ jobs: pip install dist/python/*.gz cd integration_tests/cdk pip install -r requirements.txt - npm install + npm install deactivate cd - @@ -56,7 +56,7 @@ jobs: PROJECT_ID: ${{ steps.short-sha.outputs.sha }} run: | source .deployment_venv/bin/activate - + # synthesize the stack cd integration_tests/cdk npx cdk synth --debug --all --require-approval never @@ -65,7 +65,7 @@ jobs: npx cdk deploy --ci --all --require-approval never deactivate cd - - + - name: Tear down any infrastructure if: always() env: diff --git a/.github/workflows/distribute.yaml b/.github/workflows/distribute.yaml index 960d4a4..2e604e6 100644 --- a/.github/workflows/distribute.yaml +++ b/.github/workflows/distribute.yaml @@ -18,7 +18,7 @@ jobs: with: name: python path: dist - + - run: pip install twine - run: twine upload dist/* diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 0000000..1ba24ca --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,26 @@ +name: Lint + +on: + push: + branches: + - main + pull_request: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pre-commit + + - name: Run pre-commit + run: pre-commit run --all-files diff --git a/.gitignore b/.gitignore index 0ce8378..b54838e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,4 @@ __pycache__ .tox tests/*.egg* tests/*venv* -tests/__pycache__ \ No newline at end of file +tests/__pycache__ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3ce6095 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks + +# Optionally both commit and push +default_stages: [commit] + +# Regex for files to exclude +# Don't lint the generated JSON metadata files +exclude: "diagrams/" + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer diff --git a/README.md b/README.md index 698f7a2..83f4ffc 100644 --- a/README.md +++ b/README.md @@ -56,4 +56,4 @@ _Warning_: If you rebase `main`, you must ensure that the commits referenced by ## Tests -Each pull request to `main` is added to a [merge queue](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue#triggering-merge-group-checks-with-github-actions) so that a "deployment test" workflow can run before the merge actually happens. If the deployment fails, the merge is cancelled. Here is [the definition of this workflow](https://github.com/developmentseed/eoapi-cdk/blob/main/.github/workflows/deploy.yaml) and the [tests definition](https://github.com/developmentseed/eoapi-cdk/blob/main/tests). \ No newline at end of file +Each pull request to `main` is added to a [merge queue](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue#triggering-merge-group-checks-with-github-actions) so that a "deployment test" workflow can run before the merge actually happens. If the deployment fails, the merge is cancelled. Here is [the definition of this workflow](https://github.com/developmentseed/eoapi-cdk/blob/main/.github/workflows/deploy.yaml) and the [tests definition](https://github.com/developmentseed/eoapi-cdk/blob/main/tests). diff --git a/integration_tests/cdk/README.md b/integration_tests/cdk/README.md index abc32f6..bd70e8d 100644 --- a/integration_tests/cdk/README.md +++ b/integration_tests/cdk/README.md @@ -8,11 +8,11 @@ This is a wrapper CDK code that is used to test a deployment of the `eoapi-cdk` - python - docker - node -- AWS credentials environment variables configured to point to an account. +- AWS credentials environment variables configured to point to an account. ## Installation -Install python dependencies with +Install python dependencies with ``` python -m venv .venv @@ -28,7 +28,7 @@ pip install eoapi-cdk Or alternatively, compile and package from the root of this repository to get the python version of the constructs locally. -Also install node dependencies with +Also install node dependencies with ``` npm install @@ -42,7 +42,7 @@ npx cdk --version ## Deployment -First, synthesize the app +First, synthesize the app ``` npx cdk synth --all @@ -52,4 +52,4 @@ Then, deploy ``` npx cdk deploy --all --require-approval never -``` \ No newline at end of file +``` diff --git a/integration_tests/cdk/cdk.json b/integration_tests/cdk/cdk.json index d9313bf..294adbb 100644 --- a/integration_tests/cdk/cdk.json +++ b/integration_tests/cdk/cdk.json @@ -29,4 +29,4 @@ "aws-cn" ] } -} \ No newline at end of file +} diff --git a/integration_tests/cdk/package.json b/integration_tests/cdk/package.json index f16c04f..e5a5a6b 100644 --- a/integration_tests/cdk/package.json +++ b/integration_tests/cdk/package.json @@ -5,4 +5,3 @@ "aws-cdk": "2.130.0" } } - \ No newline at end of file diff --git a/lib/index.ts b/lib/index.ts index 6f543fd..a1c5a45 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -5,4 +5,4 @@ export * from "./stac-api"; export * from "./titiler-pgstac-api"; export * from "./stac-browser"; export * from "./tipg-api"; -export * from "./utils"; \ No newline at end of file +export * from "./utils"; diff --git a/lib/ingestor-api/index.ts b/lib/ingestor-api/index.ts index e39f7ce..25ad480 100644 --- a/lib/ingestor-api/index.ts +++ b/lib/ingestor-api/index.ts @@ -46,7 +46,7 @@ export class StacIngestor extends Construct { ), ], }); - + const handler = this.buildApiLambda({ table: this.table, env, @@ -113,7 +113,7 @@ export class StacIngestor extends Construct { subnetSelection: undefined | ec2.SubnetSelection lambdaFunctionOptions?: CustomLambdaFunctionProps; }): lambda.Function { - + const handler = new lambda.Function(this, "api-handler", { // defaults runtime: lambda.Runtime.PYTHON_3_11, @@ -162,7 +162,7 @@ export class StacIngestor extends Construct { lambdaFunctionOptions?: CustomLambdaFunctionProps; }): lambda.Function { - + const handler = new lambda.Function(this, "stac-ingestor",{ // defaults runtime: lambda.Runtime.PYTHON_3_11, @@ -319,17 +319,17 @@ export interface StacIngestorProps { readonly ingestorDomainNameOptions?: apigateway.DomainNameOptions; /** - * Can be used to override the default lambda function properties. + * Can be used to override the default lambda function properties. * * @default - default settings are defined in the construct. */ readonly apiLambdaFunctionOptions?: CustomLambdaFunctionProps; /** - * Can be used to override the default lambda function properties. + * Can be used to override the default lambda function properties. * * @default - default settings are defined in the construct. */ readonly ingestorLambdaFunctionOptions?: CustomLambdaFunctionProps; - -} \ No newline at end of file + +} diff --git a/lib/ingestor-api/runtime/Dockerfile b/lib/ingestor-api/runtime/Dockerfile index 02eb7da..8bfafd2 100644 --- a/lib/ingestor-api/runtime/Dockerfile +++ b/lib/ingestor-api/runtime/Dockerfile @@ -13,4 +13,4 @@ RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset --n RUN mkdir -p /asset/src COPY runtime/src/*.py /asset/src/ -CMD ["echo", "hello world"] \ No newline at end of file +CMD ["echo", "hello world"] diff --git a/lib/stac-api/index.ts b/lib/stac-api/index.ts index 77c4bc2..9fe86d8 100644 --- a/lib/stac-api/index.ts +++ b/lib/stac-api/index.ts @@ -19,7 +19,7 @@ export class PgStacApiLambda extends Construct { constructor(scope: Construct, id: string, props: PgStacApiLambdaProps) { super(scope, id); - + console.log(props) console.log(props.lambdaFunctionOptions); this.stacApiLambdaFunction = new lambda.Function(this, "lambda", { @@ -51,7 +51,7 @@ export class PgStacApiLambda extends Construct { this.stacApiLambdaFunction.connections.allowTo(props.db, ec2.Port.tcp(5432)); } const stacApi = new HttpApi(this, `${Stack.of(this).stackName}-stac-api`, { - defaultDomainMapping: props.stacApiDomainName ? { + defaultDomainMapping: props.stacApiDomainName ? { domainName: props.stacApiDomainName } : undefined, defaultIntegration: new HttpLambdaIntegration("integration", this.stacApiLambdaFunction), @@ -104,4 +104,3 @@ export interface PgStacApiLambdaProps { */ readonly lambdaFunctionOptions?: CustomLambdaFunctionProps; } - diff --git a/lib/stac-api/runtime/Dockerfile b/lib/stac-api/runtime/Dockerfile index 3f8e0ac..380eb4e 100644 --- a/lib/stac-api/runtime/Dockerfile +++ b/lib/stac-api/runtime/Dockerfile @@ -10,4 +10,4 @@ RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset --n RUN mkdir -p /asset/src COPY runtime/src/*.py /asset/src/ -CMD ["echo", "hello world"] \ No newline at end of file +CMD ["echo", "hello world"] diff --git a/lib/stac-api/runtime/requirements.txt b/lib/stac-api/runtime/requirements.txt index 744fce5..cbc95bb 100644 --- a/lib/stac-api/runtime/requirements.txt +++ b/lib/stac-api/runtime/requirements.txt @@ -4,4 +4,4 @@ stac-fastapi.pgstac==2.4.8 stac-fastapi.types==2.4.8 # https://github.com/stac-utils/stac-fastapi/pull/466 pygeoif==0.7 -starlette_cramjam \ No newline at end of file +starlette_cramjam diff --git a/lib/stac-browser/index.ts b/lib/stac-browser/index.ts index df8beb0..d7263fc 100644 --- a/lib/stac-browser/index.ts +++ b/lib/stac-browser/index.ts @@ -28,12 +28,12 @@ export class StacBrowser extends Construct { websiteIndexDocument: props.websiteIndexDocument }) } - + // if props.cloudFrontDistributionArn is defined and props.bucketArn is not defined, add a bucket policy to allow read access from the cloudfront distribution if (props.cloudFrontDistributionArn && !props.bucketArn) { this.bucket.addToResourcePolicy(new PolicyStatement({ sid: 'AllowCloudFrontServicePrincipal', - effect: Effect.ALLOW, + effect: Effect.ALLOW, actions: ['s3:GetObject'], principals: [new ServicePrincipal('cloudfront.amazonaws.com')], resources: [this.bucket.arnForObjects('*')], @@ -44,7 +44,7 @@ export class StacBrowser extends Construct { } })); } - + // add the compiled code to the bucket as a bucket deployment this.bucketDeployment = new s3_deployment.BucketDeployment(this, 'BucketDeployment', { destinationBucket: this.bucket, @@ -59,12 +59,12 @@ export class StacBrowser extends Construct { } private buildApp(props: StacBrowserProps, cloneDirectory: string): string { - + // Define where to clone and build const githubRepoUrl = 'https://github.com/radiantearth/stac-browser.git'; - // Maybe the repo already exists in cloneDirectory. Try checking out the desired version and if it fails, delete and reclone. + // Maybe the repo already exists in cloneDirectory. Try checking out the desired version and if it fails, delete and reclone. try { console.log(`Checking if a valid cloned repo exists with version ${props.githubRepoTag}...`) execSync(`git checkout tags/${props.githubRepoTag}`, { cwd: cloneDirectory }); @@ -116,9 +116,9 @@ export class StacBrowser extends Construct { export interface StacBrowserProps { /** - * Bucket ARN. If specified, the identity used to deploy the stack must have the appropriate permissions to create a deployment for this bucket. + * Bucket ARN. If specified, the identity used to deploy the stack must have the appropriate permissions to create a deployment for this bucket. * In addition, if specified, `cloudFrontDistributionArn` is ignored since the policy of an imported resource can't be modified. - * + * * @default - No bucket ARN. A new bucket will be created. */ @@ -126,7 +126,7 @@ export interface StacBrowserProps { /** * STAC catalog URL. Overrides the catalog URL in the stac-browser configuration. - */ + */ readonly stacCatalogUrl: string; /** @@ -144,11 +144,11 @@ export interface StacBrowserProps { /** * The ARN of the cloudfront distribution that will be added to the bucket policy with read access. * If `bucketArn` is specified, this parameter is ignored since the policy of an imported bucket can't be modified. - * + * * @default - No cloudfront distribution ARN. The bucket policy will not be modified. - */ + */ readonly cloudFrontDistributionArn?: string; - + /** * The name of the index document (e.g. "index.html") for the website. Enables static website * hosting for this bucket. @@ -158,8 +158,8 @@ export interface StacBrowserProps { readonly websiteIndexDocument?: string; /** - * Location in the filesystem where to compile the browser code. - * + * Location in the filesystem where to compile the browser code. + * * @default - DEFAULT_CLONE_DIRECTORY */ readonly cloneDirectory?: string; diff --git a/lib/tipg-api/index.ts b/lib/tipg-api/index.ts index d40a3df..37f8b5c 100644 --- a/lib/tipg-api/index.ts +++ b/lib/tipg-api/index.ts @@ -49,7 +49,7 @@ import { this.tiPgLambdaFunction.connections.allowTo(props.db, ec2.Port.tcp(5432), "allow connections from tipg"); } const tipgApi = new HttpApi(this, `${Stack.of(this).stackName}-tipg-api`, { - defaultDomainMapping: props.tipgApiDomainName ? { + defaultDomainMapping: props.tipgApiDomainName ? { domainName: props.tipgApiDomainName } : undefined, defaultIntegration: new HttpLambdaIntegration("integration", this.tiPgLambdaFunction), @@ -91,11 +91,11 @@ import { * Customized environment variables to send to titiler-pgstac runtime. */ readonly apiEnv?: Record; - + /** - * Custom Domain Name for tipg API. If defined, will create the - * domain name and integrate it with the tipg API. - * + * Custom Domain Name for tipg API. If defined, will create the + * domain name and integrate it with the tipg API. + * * @default - undefined */ readonly tipgApiDomainName?: IDomainName; diff --git a/lib/titiler-pgstac-api/index.ts b/lib/titiler-pgstac-api/index.ts index 726744d..61708e5 100644 --- a/lib/titiler-pgstac-api/index.ts +++ b/lib/titiler-pgstac-api/index.ts @@ -13,12 +13,12 @@ import { import { HttpLambdaIntegration } from "@aws-cdk/aws-apigatewayv2-integrations-alpha"; import { Construct } from "constructs"; import { CustomLambdaFunctionProps } from "../utils"; - - // default settings that can be overridden by the user-provided environment. + + // default settings that can be overridden by the user-provided environment. let defaultTitilerPgstacEnv :{ [key: string]: any } = { "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": ".tif,.TIF,.tiff", - "GDAL_CACHEMAX": "200", + "GDAL_CACHEMAX": "200", "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR", "GDAL_INGESTED_BYTES_AT_OPEN": "32768", "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES", @@ -26,7 +26,7 @@ import { CustomLambdaFunctionProps } from "../utils"; "GDAL_HTTP_VERSION": "2", "PYTHONWARNINGS": "ignore", "VSI_CACHE": "TRUE", - "VSI_CACHE_SIZE": "5000000", + "VSI_CACHE_SIZE": "5000000", "DB_MIN_CONN_SIZE": "1", "DB_MAX_CONN_SIZE": "1" } @@ -34,10 +34,10 @@ import { CustomLambdaFunctionProps } from "../utils"; export class TitilerPgstacApiLambda extends Construct { readonly url: string; public titilerPgstacLambdaFunction: lambda.Function; - + constructor(scope: Construct, id: string, props: TitilerPgStacApiLambdaProps) { super(scope, id); - + this.titilerPgstacLambdaFunction = new lambda.Function(this, "lambda", { // defaults runtime: lambda.Runtime.PYTHON_3_11, @@ -55,9 +55,9 @@ import { CustomLambdaFunctionProps } from "../utils"; // if user provided environment variables, merge them with the defaults. environment: props.apiEnv ? { ...defaultTitilerPgstacEnv, ...props.apiEnv, "PGSTAC_SECRET_ARN": props.dbSecret.secretArn } : defaultTitilerPgstacEnv, // overwrites defaults with user-provided configurable properties - ...props.lambdaFunctionOptions, + ...props.lambdaFunctionOptions, }); - + // grant access to buckets using addToRolePolicy if (props.buckets) { props.buckets.forEach(bucket => { @@ -67,46 +67,46 @@ import { CustomLambdaFunctionProps } from "../utils"; })); }); } - + props.dbSecret.grantRead(this.titilerPgstacLambdaFunction); - + if (props.vpc) { this.titilerPgstacLambdaFunction.connections.allowTo(props.db, ec2.Port.tcp(5432), "allow connections from titiler"); } - + const stacApi = new HttpApi(this, `${Stack.of(this).stackName}-titiler-pgstac-api`, { - defaultDomainMapping: props.titilerPgstacApiDomainName ? { - domainName: props.titilerPgstacApiDomainName + defaultDomainMapping: props.titilerPgstacApiDomainName ? { + domainName: props.titilerPgstacApiDomainName } : undefined, defaultIntegration: new HttpLambdaIntegration("integration", this.titilerPgstacLambdaFunction), }); - + this.url = stacApi.url!; - + new CfnOutput(this, "titiler-pgstac-api-output", { exportName: `${Stack.of(this).stackName}-titiler-pgstac-url`, value: this.url, }); } } - + export interface TitilerPgStacApiLambdaProps { /** * VPC into which the lambda should be deployed. */ readonly vpc?: ec2.IVpc; - + /** * RDS Instance with installed pgSTAC. */ readonly db: rds.IDatabaseInstance; - + /** * Subnet into which the lambda should be deployed. */ readonly subnetSelection?: ec2.SubnetSelection; - + /** * Secret containing connection information for pgSTAC database. */ @@ -114,19 +114,19 @@ import { CustomLambdaFunctionProps } from "../utils"; /** * Customized environment variables to send to titiler-pgstac runtime. These will be merged with `defaultTitilerPgstacEnv`. - * The database secret arn is automatically added to the environment variables at deployment. + * The database secret arn is automatically added to the environment variables at deployment. /*/ readonly apiEnv?: Record; /** - * list of buckets the lambda will be granted access to. + * list of buckets the lambda will be granted access to. */ readonly buckets?: string[]; /** * Custom Domain Name Options for Titiler Pgstac API, - * - * @default - undefined. + * + * @default - undefined. */ readonly titilerPgstacApiDomainName?: IDomainName; @@ -138,4 +138,3 @@ import { CustomLambdaFunctionProps } from "../utils"; readonly lambdaFunctionOptions?: CustomLambdaFunctionProps; } - diff --git a/lib/titiler-pgstac-api/runtime/Dockerfile b/lib/titiler-pgstac-api/runtime/Dockerfile index e5cf74e..307425d 100644 --- a/lib/titiler-pgstac-api/runtime/Dockerfile +++ b/lib/titiler-pgstac-api/runtime/Dockerfile @@ -16,4 +16,4 @@ RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/g COPY runtime/src/*.py /asset/ -CMD ["echo", "hello world"] \ No newline at end of file +CMD ["echo", "hello world"]