diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 610a0060..7b9c9459 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -65,3 +65,10 @@ jobs: version: main secrets: security_checks_token: ${{ secrets.CRUD_SERVICE_SYSDIG_CHECK_TRIGGER }} + + perf-test: + needs: + - checks + - release-docker + - tests + uses: ./.github/workflows/perf-test.yml diff --git a/.github/workflows/perf-test.yml b/.github/workflows/perf-test.yml new file mode 100644 index 00000000..79f7719a --- /dev/null +++ b/.github/workflows/perf-test.yml @@ -0,0 +1,49 @@ +name: 'Performance Test' + +on: + workflow_call: + inputs: + node-version: + default: 20.x + required: false + type: string + push: + tags: + - "v*" + + +jobs: + k6_ci_test: + name: k6 CI Test run + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version: 20.x + cache: npm + + - name: start CRUD Service via docker file + working-directory: bench + run: docker compose --file docker-compose.yml up -d + + - name: Populate "Customer" collection and "Registered Customer" View + run: npm i && node bench/utils/generate-customer-data.js -c mongodb://localhost:27017/bench-test -d bench-test -n 100000 -s 250 + + - name: Run k6 load test (collection Items) + working-directory: bench + run: docker compose -f dc-k6.yml up k6-load-test + + - name: Run k6 spike test (collection Customers) + working-directory: bench + run: docker compose -f dc-k6.yml up k6-spike-test + + - name: Run k6 stress test (view Registered Customers) + working-directory: bench + run: docker compose -f dc-k6.yml up k6-stress-test-on-view + + diff --git a/.gitignore b/.gitignore index 19fcdd90..aa25f15d 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ kms test-collections .npmrc .tap -.local \ No newline at end of file +.local + diff --git a/README.md b/README.md index 0f07bd27..83a7e3cb 100644 --- a/README.md +++ b/README.md @@ -323,6 +323,59 @@ This API responses always in `application/application/x-ndjson` See the documentation to see which parameters are available. +## Performance test + +We use [k6](https://example.com/k6) to simulate the load of traffic directed to the CRUD Service and retrieve some performance metrics. At every version released, a workflow automatically starts executing the following tests: + +- **Load Test**: 10 virtual users execute POST requests for one minute on the same collection, then 100 virtual users execute GET, PATCH, and DELETE requests for another minute on the data created. +- **Spike Test**: We simulate a spike of activity by increasing the number of users from 5 to 500 in 30 seconds, then a decrement of activity from 500 to 5 in another 30 seconds. During this test, only GET requests are executed on a collection that includes 100,000 documents. +- **Stress Test**: We simulate a brief time of intense activity with 250 users for 90 seconds, followed by a decrement of activity to 5 in 30 seconds. During this test, only GET requests are executed on a collection that includes 100,000 documents. + +These tests are executed ahead of every version release to ensure that further updates do not cause a degradation of performance that might affect the usage of the CRUD Service. + +### Execute Performance Test on a Local Environment + +In case you want to run the tests on your local environment, follow these steps: + +- Start the CRUD Service in a Docker container. +- Have a MongoDB instance ready for use, eventually loaded with existing documents to simulate tests. + +To simplify these operations, you can use the same setup for the tests executed during the GitHub workflow, by starting an instance of the CRUD Service using collections and views included in the folder `_bench/definitions`. Use the script `bench/utils/generate-customer-data.js` to quickly include mock documents in the _customers_ collection. + +The `generate-customer-data.js` script can be executed at any time with the following command: + +```bash +node bench/utils/generate-customer-data.js -c -d -n -s +``` +Where the script arguments are the following: +- **connection string** (default: _mongodb://localhost:27017_): Connects to your MongoDB instance. +- **database name** (default: _bench-test_): Specifies the name of the database to write to. +- **number of documents** (default: _100000_): Sets the number of documents to be created and saved in the customers collection of the specified database. +- **number of total shops** (default: _250_): Defines a random value (from 1 to the specified number) applied to the shopID field of each document to be saved. + + +To simplify these operations, you can execute the command `npm run bench:init` from your shell. This command starts a container with a MongoDB 6.0 instance, a container with the CRUD Service (built from your current branch), and populates the _customers_ collection with 100,000 documents. + +To execute any test, start the k6 service with the following command: +``` +docker compose -f bench/dc-k6.yml up +``` +Remember to replace `` with one of the following: +| Service Name | Description | File name containing the test | +|--------------------------------|-------------------------------------------------------------------------------------------------|------------------------------------------| +| k6-load-test | Executes a Load Test (1 minute of POST, 1 minute of GET/PATCH/DELETE) on the _items_ collection | [load-test.js](bench/scripts/load-test.js) | +| k6-smoke-test | Executes a Smoke Test (1 minute of GET requests) on the _customers_ collection | [smoke-test.js](bench/scripts/smoke-test.js) | +| k6-stress-test-on-collections | Executes a Stress Test (GET requests for 90 seconds by 250 users) on the _customers_ collection | [stress-test-on-collections.js](bench/scripts/stress-test-on-collections.js) | +| k6-stress-test-on-view | Executes a Stress Test (GET requests for 90 seconds by 250 users) on the _registered-customers_ view | [stress-test-on-view.js](bench/scripts/stress-test-on-view.js) | +| k6-spike-test | Executes a Spike Test (simulate a spike of 500 concurrent users for GET requests) on the _customers_ collection | [spike-test.js](bench/scripts/spike-test.js) | +| runner | An empty test that can be populated for tests on local environment | [runner.js](bench/scripts/runner.js) | + +We suggest you use the runner to execute customized tests for your research. + +Also, do not run all the tests alltogether via `docker compose -f bench/dc-k6.yml up`, without specifying a test name, otherwise all the tests will run at the same time and the results will not be specific to any test but a global indication on how to service worked during the execution of **all** the tests. + +You are free to modify and improve those tests and the definitions used for them but please remember to not use any sensible data. + ## FAQ ### How do I change the Mongocryptd version on Debian diff --git a/bench/dc-k6.yml b/bench/dc-k6.yml new file mode 100755 index 00000000..8a04dfc7 --- /dev/null +++ b/bench/dc-k6.yml @@ -0,0 +1,106 @@ +services: + runner: + image: grafana/k6:0.48.0 + deploy: + resources: + limits: + memory: 512Mb + cpus: "1" + volumes: + - ./scripts:/app + networks: + - k6-net + command: [ + "run", + "--out", + "json=runner-results.json", + "/app/runner.js", + ] + k6-load-test: + image: grafana/k6:0.48.0 + deploy: + resources: + limits: + memory: 512Mb + cpus: "1" + volumes: + - ./scripts:/app + networks: + - k6-net + command: [ + "run", + "--out", + "json=load-test-results.json", + "/app/load-test.js", + ] + k6-smoke-test: + image: grafana/k6:0.48.0 + deploy: + resources: + limits: + memory: 512Mb + cpus: "1" + volumes: + - ./scripts:/app + networks: + - k6-net + command: [ + "run", + "--out", + "json=smoke-test-results.json", + "/app/smoke-test.js", + ] + k6-stress-test-on-collection: + image: grafana/k6:0.48.0 + deploy: + resources: + limits: + memory: 512Mb + cpus: "1" + volumes: + - ./scripts:/app + networks: + - k6-net + command: [ + "run", + "--out", + "json=stress-test-on-collection-results.json", + "/app/stress-test-on-collection.js", + ] + k6-stress-test-on-view: + image: grafana/k6:0.48.0 + deploy: + resources: + limits: + memory: 512Mb + cpus: "1" + volumes: + - ./scripts:/app + networks: + - k6-net + command: [ + "run", + "--out", + "json=stress-test-on-view-results.json", + "/app/stress-test-on-view.js", + ] + k6-spike-test: + image: grafana/k6:0.48.0 + deploy: + resources: + limits: + memory: 512Mb + cpus: "1" + volumes: + - ./scripts:/app + networks: + - k6-net + command: [ + "run", + "--out", + "json=spike-test-results.json", + "/app/spike-test.js", + ] + +networks: + k6-net: diff --git a/bench/definitions/collections/customers.js b/bench/definitions/collections/customers.js new file mode 100755 index 00000000..7e52ea0d --- /dev/null +++ b/bench/definitions/collections/customers.js @@ -0,0 +1,245 @@ +/* + * Copyright 2023 Mia s.r.l. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict' + +module.exports = { + name: 'customers', + endpointBasePath: '/customers', + defaultState: 'PUBLIC', + fields: [ + { + name: '_id', + type: 'ObjectId', + required: true, + }, + { + name: 'updaterId', + type: 'string', + description: 'User id that has requested the last change successfully', + required: true, + }, + { + name: 'updatedAt', + type: 'Date', + description: 'Date of the request that has performed the last change', + required: true, + }, + { + name: 'creatorId', + type: 'string', + description: 'User id that has created this object', + required: true, + }, + { + name: 'createdAt', + type: 'Date', + description: 'Date of the request that has performed the object creation', + required: true, + }, + { + name: '__STATE__', + type: 'string', + description: 'The state of the document', + required: true, + }, + { + name: 'customerId', + type: 'string', + required: true, + nullable: false, + }, + { + name: 'firstName', + type: 'string', + required: true, + nullable: false, + }, + { + name: 'lastName', + type: 'string', + required: true, + nullable: false, + }, + { + name: 'gender', + type: 'string', + required: true, + nullable: false, + }, + { + name: 'birthDate', + type: 'Date', + required: true, + nullable: false, + }, + { + name: 'creditCardDetail', + type: 'RawObject', + required: true, + nullable: false, + schema: { + properties: { + name: { type: 'string' }, + cardNo: { type: 'number' }, + expirationDate: { type: 'string' }, + cvv: { type: 'string' }, + }, + required: ['name', 'cardNo', 'expirationDate', 'cardCode'], + }, + }, + { + name: 'canBeContacted', + type: 'boolean', + required: false, + nullable: false, + }, + { + name: 'email', + type: 'string', + required: true, + nullable: false, + }, + { + name: 'phoneNumber', + type: 'string', + required: false, + nullable: false, + }, + { + name: 'address', + type: 'RawObject', + required: false, + nullable: true, + schema: { + properties: { + line: { type: 'string' }, + city: { type: 'string' }, + county: { type: 'string' }, + country: { type: 'string' }, + }, + required: ['line', 'city', 'county', 'country'], + }, + }, + { + name: 'socialNetworkProfiles', + type: 'RawObject', + required: false, + nullable: true, + schema: { + properties: { + 'twitter': { type: 'string' }, + 'instagram': { type: 'string' }, + 'facebook': { type: 'string' }, + 'threads': { type: 'string' }, + 'reddit': { type: 'string' }, + 'linkedin': { type: 'string' }, + 'tiktok': { type: 'string' }, + }, + }, + }, + { + name: 'subscriptionNumber', + type: 'string', + required: true, + nullable: false, + }, + { + name: 'shopID', + type: 'number', + required: true, + nullable: false, + }, + { + name: 'purchasesCount', + type: 'number', + required: true, + nullable: false, + }, + { + name: 'purchases', + type: 'Array', + items: { + type: 'RawObject', + schema: { + properties: { + name: { type: 'string' }, + category: { type: 'string' }, + price: { type: 'number' }, + employeeId: { type: 'string' }, + boughtOnline: { type: 'boolean' }, + }, + }, + }, + }, + { + name: 'details', + type: 'string', + required: false, + nullable: false, + }, + ], + indexes: [ + { + name: '_id', + type: 'normal', + unique: true, + fields: [ + { + name: '_id', + order: 1, + }, + ], + }, + { + name: 'shopIdIndex', + type: 'normal', + unique: false, + fields: [ + { + name: 'shopID', + order: 1, + }, + { + name: '__STATE__', + order: 1, + }, + ], + }, + { + name: 'purchasesCountIndex', + type: 'normal', + unique: true, + fields: [ + { + name: 'purchasesCount', + order: 1, + }, + ], + }, + { + name: 'canBeContactedIndex', + type: 'normal', + unique: false, + fields: [ + { + name: 'canBeContacted', + order: 1, + }, + ], + }, + ], +} diff --git a/bench/definitions/collections/items.js b/bench/definitions/collections/items.js new file mode 100755 index 00000000..64dc776d --- /dev/null +++ b/bench/definitions/collections/items.js @@ -0,0 +1,147 @@ +/* + * Copyright 2023 Mia s.r.l. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict' + +module.exports = { + name: 'items', + endpointBasePath: '/items', + defaultState: 'PUBLIC', + fields: [ + { + name: '_id', + type: 'ObjectId', + required: true, + }, + { + name: 'updaterId', + type: 'string', + description: 'User id that has requested the last change successfully', + required: true, + }, + { + name: 'updatedAt', + type: 'Date', + description: 'Date of the request that has performed the last change', + required: true, + }, + { + name: 'creatorId', + type: 'string', + description: 'User id that has created this object', + required: true, + }, + { + name: 'createdAt', + type: 'Date', + description: 'Date of the request that has performed the object creation', + required: true, + }, + { + name: '__STATE__', + type: 'string', + description: 'The state of the document', + required: true, + }, + { + name: 'string', + type: 'string', + required: false, + nullable: true, + }, + { + name: 'number', + type: 'number', + required: false, + nullable: true, + }, + { + name: 'boolean', + type: 'boolean', + required: false, + nullable: true, + }, + { + name: 'date', + type: 'Date', + required: false, + nullable: true, + }, + { + name: 'object', + type: 'RawObject', + required: false, + nullable: true, + schema: { + properties: { + string: { type: 'string' }, + number: { type: 'number' }, + boolean: { type: 'boolean' }, + counter: { type: 'number' }, + }, + }, + }, + { + name: 'array', + type: 'Array', + items: { + type: 'RawObject', + schema: { + properties: { + string: { type: 'string' }, + number: { type: 'number' }, + boolean: { type: 'boolean' }, + }, + }, + }, + }, + ], + indexes: [ + { + name: '_id', + type: 'normal', + unique: true, + fields: [ + { + name: '_id', + order: 1, + }, + ], + }, + { + name: 'numberIndex', + type: 'normal', + unique: false, + fields: [ + { + name: 'number', + order: 1, + }, + ], + }, + { + name: 'counterIndex', + type: 'normal', + unique: false, + fields: [ + { + name: 'object.counter', + order: 1, + }, + ], + }, + ], +} diff --git a/bench/definitions/collections/registered-customers.js b/bench/definitions/collections/registered-customers.js new file mode 100755 index 00000000..f452fd95 --- /dev/null +++ b/bench/definitions/collections/registered-customers.js @@ -0,0 +1,146 @@ +/* + * Copyright 2023 Mia s.r.l. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict' + +module.exports = { + name: 'registered-customers', + endpointBasePath: '/registered-customers', + defaultState: 'PUBLIC', + fields: [ + { + name: '_id', + type: 'ObjectId', + required: true, + }, + { + name: 'updaterId', + type: 'string', + description: 'User id that has requested the last change successfully', + required: true, + }, + { + name: 'updatedAt', + type: 'Date', + description: 'Date of the request that has performed the last change', + required: true, + }, + { + name: 'creatorId', + type: 'string', + description: 'User id that has created this object', + required: true, + }, + { + name: 'createdAt', + type: 'Date', + description: 'Date of the request that has performed the object creation', + required: true, + }, + { + name: '__STATE__', + type: 'string', + description: 'The state of the document', + required: true, + }, + { + name: 'customerId', + type: 'string', + required: true, + nullable: false, + }, + { + name: 'firstName', + type: 'string', + required: true, + nullable: false, + }, + { + name: 'lastName', + type: 'string', + required: true, + nullable: false, + }, + { + name: 'shopID', + type: 'number', + required: true, + nullable: false, + }, + { + name: 'canBeContacted', + type: 'boolean', + required: false, + nullable: false, + }, + { + name: 'purchasesCount', + type: 'number', + required: true, + nullable: false, + }, + ], + indexes: [ + { + name: '_id', + type: 'normal', + unique: true, + fields: [ + { + name: '_id', + order: 1, + }, + ], + }, + { + name: 'shopIdIndex', + type: 'normal', + unique: false, + fields: [ + { + name: 'shopID', + order: 1, + }, + { + name: '__STATE__', + order: 1, + }, + ], + }, + { + name: 'purchasesCountIndex', + type: 'normal', + unique: false, + fields: [ + { + name: 'purchasesCount', + order: 1, + }, + ], + }, + { + name: 'canBeContactedIndex', + type: 'normal', + unique: false, + fields: [ + { + name: 'canBeContacted', + order: 1, + }, + ], + }, + ], +} diff --git a/bench/definitions/views/registered-customers.js b/bench/definitions/views/registered-customers.js new file mode 100755 index 00000000..02dfb393 --- /dev/null +++ b/bench/definitions/views/registered-customers.js @@ -0,0 +1,42 @@ +/* + * Copyright 2023 Mia s.r.l. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict' + +module.exports = { + name: 'registered-customers', + source: 'customers', + type: 'view', + pipeline: [ + { + $project: { + _id: 1, + name: 1, + updaterId: 1, + updatedAt: 1, + creatorId: 1, + createdAt: 1, + customerId: 1, + firstName: 1, + lastName: 1, + shopID: 1, + canBeContacted: 1, + purchasesCount: 1, + __STATE__: 1, + }, + }, + ], +} diff --git a/bench/docker-compose.yml b/bench/docker-compose.yml new file mode 100755 index 00000000..44206e4a --- /dev/null +++ b/bench/docker-compose.yml @@ -0,0 +1,60 @@ +services: + database: + image: mongo:6.0 + ports: + - '27017:27017' + volumes: + - mongo:/data/db + networks: + - k6-net + deploy: + resources: + limits: + memory: 4GB + cpus: "2" + healthcheck: + test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 5s + + crud-service: + build: + context: .. + dockerfile: Dockerfile + ports: + - '3000:3000' + depends_on: + database: + condition: service_healthy + networks: + - k6-net + deploy: + resources: + limits: + memory: 500Mb + cpus: "2" + environment: + LOG_LEVEL: info + COLLECTION_DEFINITION_FOLDER: /home/node/app/definitions/collections + VIEWS_DEFINITION_FOLDER: /home/node/app/definitions/views + USER_ID_HEADER_KEY: userid + CRUD_LIMIT_CONSTRAINT_ENABLED: "true" + CRUD_MAX_LIMIT: 200 + MONGODB_URL: "mongodb://database:27017/bench-test" + volumes: + - ./definitions:/home/node/app/definitions + - ./utils/healthcheck.js:/home/node/app/healthcheck/healthcheck.js + healthcheck: + test: [ "CMD-SHELL", "node /home/node/app/healthcheck/healthcheck.js" ] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + +networks: + k6-net: + +volumes: + mongo: diff --git a/bench/queryParser.bench.js b/bench/queryParser.bench.js old mode 100644 new mode 100755 diff --git a/bench/scripts/load-test.js b/bench/scripts/load-test.js new file mode 100755 index 00000000..0e1970f7 --- /dev/null +++ b/bench/scripts/load-test.js @@ -0,0 +1,179 @@ +/* + * Copyright 2023 Mia s.r.l. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import http from 'k6/http'; +import { textSummary } from 'https://jslib.k6.io/k6-summary/0.0.2/index.js'; +import { check, sleep } from 'k6'; +import { + randomIntBetween, + randomString, +} from 'https://jslib.k6.io/k6-utils/1.4.0/index.js'; +import { is200, is200or404, is204or404, CRUD_BASE_URL } from './utils.js' + +// +// Test on collection "items" +// Type of test: load test +// +// We have a first stage where we insert documents for 1 minute +// When we're done, then we execute a round GET (list), GET by _q, PATCH by _q and DELETE by _q +// The following for 100 VUs for a minute +// + +export const options = { + scenarios: { + 'initialLoad': { + executor: 'constant-vus', + exec: 'initialLoad', + vus: 10, + duration: '1m', + tags: { test_type: 'initialLoad' } + }, + 'loadTest': { + executor: 'constant-vus', + exec: 'loadTest', + vus: 100, + startTime: '1m', + duration: '1m', + tags: { test_type: 'loadTest' } + } + }, + thresholds: { + checks: ['rate==1'], + http_req_failed: ['rate<0.01'], + 'http_req_duration{type:post}': ['p(90)<100'], + 'http_req_duration{type:getList}': ['p(90)<500'], + 'http_req_duration{type:getById}': ['p(90)<500'], + 'http_req_duration{type:patchByQuery}': ['p(90)<500'], + 'http_req_duration{type:patchById}': ['p(90)<500'], + 'http_req_duration{type:deleteByQuery}': ['p(90)<500'], + 'http_req_duration{type:deleteById}': ['p(90)<500'], + }, +} + +// #region helper fns + +let counter = 0 +const generateItem = () => { + const array = [] + const len = randomIntBetween(0, 10) + for (let i = 0; i < len; i++) { + array.push({ + string: `array-${i}-${randomString(5)}`, + number: randomIntBetween(1, 10), + boolean: randomIntBetween(0, 1) === 1, + }) + } + + counter += 1 + + return { + string: randomString(5), + number: randomIntBetween(1, 10), + boolean: randomIntBetween(0, 1) === 1, + date: new Date(randomIntBetween(0, 1705414020030)), + object: { + string: `object-${randomString(5)}`, + number: randomIntBetween(1, 10), + boolean: randomIntBetween(0, 1) === 1, + counter + }, + array + } +} +//#endregion + +export function initialLoad () { + let post = http.post( + `${CRUD_BASE_URL}/items`, + JSON.stringify(generateItem()), + { + headers: { 'Content-Type': 'application/json' }, + tags: { type: 'post' } + } + ); + check(post, { 'POST / returns status 200': is200or404 }) + + sleep(0.01) +} + +export function loadTest () { + // GET / request + const getList = http.get(`${CRUD_BASE_URL}/items?number=${randomIntBetween(1, 10)}`, { tags: { type: 'getList' }}) + check(getList, { 'GET / returns status 200': is200 }) + sleep(1) + + // Fetch for the seventh document from the getList request to get an id to use for a getById request + const getListResults = JSON.parse(getList.body) + const count = getListResults.length + if (count === 0) { + return + } + + // GET /{id} request + const documentIdToFetch = getListResults[randomIntBetween(0, count - 1)]._id + const getById = http.get(`${CRUD_BASE_URL}/items/${documentIdToFetch}`, { tags: { type: 'getById' }}) + const isGetByIdValid = check(getById, { 'GET /{id} returns status 200 or 404': is200or404 }) + if (!isGetByIdValid) { console.log({ failed: 'getById', error: getById.error, status: getById.status, documentId: documentIdToFetch })} + sleep(1) + + // PATCH /{id} request + const documentIdToPatch = getListResults[randomIntBetween(0, count - 1)]._id + const patchById = http.patch( + `${CRUD_BASE_URL}/items/${documentIdToPatch}`, + JSON.stringify({ $set: generateItem() }), + { + headers: { 'Content-Type': 'application/json' }, + tags: { type: 'patchById' } + } + ) + const isPatchByIdValid = check(patchById, { 'PATCH /{id} returns status 200 or 404': is200or404 }) + if (!isPatchByIdValid) { console.log({ failed: 'patchById', error: patchById.error, status: patchById.status, documentId: documentIdToFetch })} + sleep(1) + + // DELETE /{id} request + const documentIdToDelete = getListResults[randomIntBetween(0, count - 1)]._id + const deleteById = http.del(`${CRUD_BASE_URL}/items/${documentIdToDelete}`, null, { tags: { type: 'deleteById' }}) + const isDeleteByIdValid = check(deleteById, { 'DELETE /{id} returns status 204 or 404': is204or404 }) + if (!isDeleteByIdValid) { console.log({ failed: 'deleteById', error: deleteById.error, status: deleteById.status, documentId: documentIdToFetch })} + sleep(1) + + // PATCH /?_q=... request + const counterValueForPatch = getListResults[randomIntBetween(0, count - 1)].object.counter + const patchQuery = JSON.stringify({ 'object.counter': counterValueForPatch }) + const patch = http.patch( + `${CRUD_BASE_URL}/items?_q=${patchQuery}`, + JSON.stringify({ $set: generateItem() }), + { + headers: { 'Content-Type': 'application/json' }, + tags: { type: 'patchByQuery' } + } + ); + check(patch, { 'PATCH / returns status 200': is200 }) + sleep(1) + + // DELETE /?_q=... request + const counterValueForDelete = getListResults[randomIntBetween(0, count - 1)].object.counter + const deleteQuery = JSON.stringify({ 'object.counter': counterValueForDelete }) + const deleteReq = http.del(`${CRUD_BASE_URL}/items?_q=${deleteQuery}`, null, { tags: { type: 'deleteByQuery' }}) + check(deleteReq, { 'DELETE / returns status 200': is200 }) + sleep(1) +} + +export function handleSummary(data) { + return { + stdout: textSummary(data, { enableColors: true }) + }; +} diff --git a/bench/scripts/runner.js b/bench/scripts/runner.js new file mode 100755 index 00000000..b1fecf2e --- /dev/null +++ b/bench/scripts/runner.js @@ -0,0 +1,43 @@ +/* + * Copyright 2023 Mia s.r.l. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import http from 'k6/http'; +import { sleep } from 'k6'; + +// +// This file represent a stub of a k6 test. Feel free to modify based on your needs. +// + +export const options = { + stages: [ + { duration: '30s', target: 5 }, + { duration: '2m', target: 10 }, + { duration: '30s', target: 0 }, + ], +} + +export function setup() { + // Here it goes any code we want to execute before running our tests +} + +export default function () { + http.get('http://crud-service:3000/users/export?shopID=2') + sleep(1) +} + +export function teardown(data) { + // Here it goes any code we want to execute after running our tests +} diff --git a/bench/scripts/smoke-test.js b/bench/scripts/smoke-test.js new file mode 100755 index 00000000..ea54845b --- /dev/null +++ b/bench/scripts/smoke-test.js @@ -0,0 +1,45 @@ +/* + * Copyright 2023 Mia s.r.l. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { textSummary } from 'https://jslib.k6.io/k6-summary/0.0.2/index.js'; +import { executeGetTests } from './utils.js'; + +// +// Test on collection "customers" +// Type of test: smoke test +// +// 5 concurrent users for 1 minutes +// + +export const options = { + vus: 5, + duration: '1m', + thresholds: { + checks: ['rate==1'], + http_req_failed: ['rate<0.01'], + http_req_duration: ['p(90)<150', 'p(95)<300'], + } +} + +export default function () { + executeGetTests('customers') +} + +export function handleSummary(data) { + return { + stdout: textSummary(data, { enableColors: true }), + }; +} diff --git a/bench/scripts/spike-test.js b/bench/scripts/spike-test.js new file mode 100755 index 00000000..97f7edb8 --- /dev/null +++ b/bench/scripts/spike-test.js @@ -0,0 +1,54 @@ +/* + * Copyright 2023 Mia s.r.l. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { textSummary } from 'https://jslib.k6.io/k6-summary/0.0.2/index.js'; +import { executeGetTests } from './utils.js'; + +// +// Test on collection "customers" +// Type of test: spike test +// +// 5 concurrent users for the first 5 seconds +// Then number of users rise to 500 in a 20-seconds span +// Before to cool down to 5 users in 20 seconds to conclude the test +// + +export const options = { + stages: [ + { duration: '10s', target: 5 }, + { duration: '30s', target: 500 }, + { duration: '30s', target: 5 }, + ], + thresholds: { + checks: ['rate==1'], + http_req_failed: ['rate<0.01'], + 'http_req_duration{type:getList}': ['p(90)<250'], + 'http_req_duration{type:getListWithQueryOperator}': ['p(90)<250'], + 'http_req_duration{type:getById}': ['p(90)<250'], + 'http_req_duration{type:count}': ['p(90)<250'], + 'http_req_duration{type:export}': ['p(90)<250'], + } +} + +export default function () { + executeGetTests('customers') +} + +export function handleSummary(data) { + return { + stdout: textSummary(data, { enableColors: true }), + }; +} diff --git a/bench/scripts/stress-test-on-collection.js b/bench/scripts/stress-test-on-collection.js new file mode 100755 index 00000000..d5dcfb3a --- /dev/null +++ b/bench/scripts/stress-test-on-collection.js @@ -0,0 +1,57 @@ +/* + * Copyright 2023 Mia s.r.l. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { textSummary } from 'https://jslib.k6.io/k6-summary/0.0.2/index.js'; +import { executeGetTests } from './utils'; + +// +// Test on collection "customers" +// Type of test: stress test +// +// 5 concurrent users for the first 5 seconds +// Then number of users rise to 200 in a 10-seconds span +// It stays high for 45 seconds +// Then it goes back to 5 users in 30 seconds to conclude the test +// + +export const options = { + stages: [ + { duration: '5s', target: 5 }, + { duration: '10s', target: 250 }, + { duration: '75s', target: 250 }, + { duration: '10s', target: 5 }, + { duration: '10s', target: 5 }, + ], + thresholds: { + checks: ['rate==1'], + http_req_failed: ['rate<0.01'], + 'http_req_duration{type:getList}': ['p(90)<250'], + 'http_req_duration{type:getListWithQueryOperator}': ['p(90)<250'], + 'http_req_duration{type:getById}': ['p(90)<250'], + 'http_req_duration{type:count}': ['p(90)<250'], + 'http_req_duration{type:export}': ['p(90)<250'], + } +} + +export default function () { + executeGetTests('customers') +} + +export function handleSummary(data) { + return { + stdout: textSummary(data, { enableColors: true }), + }; +} diff --git a/bench/scripts/stress-test-on-view.js b/bench/scripts/stress-test-on-view.js new file mode 100755 index 00000000..15f640e6 --- /dev/null +++ b/bench/scripts/stress-test-on-view.js @@ -0,0 +1,57 @@ +/* + * Copyright 2023 Mia s.r.l. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { textSummary } from 'https://jslib.k6.io/k6-summary/0.0.2/index.js'; +import { executeGetTests } from './utils.js'; + +// +// Test on view "registered-customers" +// Type of test: stress test +// +// 5 concurrent users for the first 5 seconds +// Then number of users rise to 200 in a 10-seconds span +// It stays high for 45 seconds +// Then it goes back to 5 users in 30 seconds to conclude the test +// + +export const options = { + stages: [ + { duration: '5s', target: 5 }, + { duration: '10s', target: 250 }, + { duration: '75s', target: 250 }, + { duration: '10s', target: 5 }, + { duration: '10s', target: 5 }, + ], + thresholds: { + checks: ['rate==1'], + http_req_failed: ['rate<0.01'], + 'http_req_duration{type:getList}': ['p(90)<250'], + 'http_req_duration{type:getListWithQueryOperator}': ['p(90)<250'], + 'http_req_duration{type:getById}': ['p(90)<250'], + 'http_req_duration{type:count}': ['p(90)<250'], + 'http_req_duration{type:export}': ['p(90)<250'], + } +} + +export default function () { + executeGetTests('registered-customers') +} + +export function handleSummary(data) { + return { + stdout: textSummary(data, { enableColors: true }), + }; +} diff --git a/bench/scripts/utils.js b/bench/scripts/utils.js new file mode 100755 index 00000000..c90f780c --- /dev/null +++ b/bench/scripts/utils.js @@ -0,0 +1,85 @@ +/* + * Copyright 2023 Mia s.r.l. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict' + +import http from 'k6/http'; +import { check, sleep } from 'k6'; + +/** Base URL of the CRUD Service instance deployed via `/bench/docker-compose.yml */ +export const CRUD_BASE_URL = 'http://crud-service:3000' + +/** Returns `true` if the response have status 200. Can be used in any GET returning results. */ +export const is200 = res => res.status === 200 +/** Returns `true` if the response have status 200 or 404. Can be used for requests by id */ +export const is200or404 = res => [200, 404].includes(res.status) +/** Returns `true` if the response have status 204 or 404. Can be used for the `DELETE \{id}` requests */ +export const is204or404 = res => [204, 404].includes(res.status) + +/** + * Execute the following GET requests: + * - `GET /`: returns a list of documents from the collection filtered by a specified queryString + * - `GET /{id}`: returns a document with given id (this Id is found) + * - `GET /?_q=...`: returns a list of documents that satisfy the condition in the `_q` operator passed in queryString + * - `GET /count`: count the documents in the collection that satisfy the condition specified in the queryString + * - `GET /export`: request to export data from the collection filtered by a specified queryString + * + * @param {string} collectionName Name of the collection to execute the request in + * @param {object} options The following object includes a list of queries to be used for the different GET requests, and the time to pass between each request (in ms - default: 0.1ms). + * Queries have default values, but can be modified at will (in that case, refer to the property names to understand which request will be affected) + */ +export const executeGetTests = ( + collectionName, + { + getListQueryString = 'shopID=2', + getWithQueryOperatorQueryString = JSON.stringify({ purchasesCount: { $gte: 100 }}), + getCountQueryString = 'canBeContacted=true', + getExportQueryString = 'shopID=2', + sleepTime = 0.1 + } = {} +) => { + // GET / request + const getList = http.get(`${CRUD_BASE_URL}/${collectionName}?${getListQueryString}`, { tags: { type: 'getList' }}) + check(getList, { 'GET / returns status 200': is200 }) + sleep(sleepTime) + + // Fetch for the seventh document from the getList request to get an id to use for a getById request + const getListResults = JSON.parse(getList.body) + const count = getListResults.length + const document = getListResults[7 % count] + + if (document) { + // GET /{id} request + const getById = http.get(`${CRUD_BASE_URL}/${collectionName}/${document._id}`, { tags: { type: 'getById' }}) + check(getById, { 'GET/{id} returns status 200': is200 }) + sleep(sleepTime) + } + + // GET /_q=... request + const getWithQuery = http.get(`${CRUD_BASE_URL}/${collectionName}/?_q=${getWithQueryOperatorQueryString}`, { tags: { type: 'getListWithQueryOperator' }}) + check(getWithQuery, { 'GET /?_q=... returns status 200': is200 }) + sleep(sleepTime) + + // GET /count request + const getCount = http.get(`${CRUD_BASE_URL}/${collectionName}/count?${getCountQueryString}`, { tags: { type: 'count' }}) + check(getCount, { 'GET /count returns status 200': is200 }) + sleep(sleepTime) + + // GET /export request + const getExport = http.get(`${CRUD_BASE_URL}/${collectionName}/export?${getExportQueryString}`, { tags: { type: 'export' }}) + check(getExport, { 'GET /export returns status 200': is200 }) + sleep(sleepTime) +} diff --git a/bench/utils/generate-customer-data.js b/bench/utils/generate-customer-data.js new file mode 100755 index 00000000..1d6b7a19 --- /dev/null +++ b/bench/utils/generate-customer-data.js @@ -0,0 +1,157 @@ +/* eslint-disable no-console */ +/* + * Copyright 2023 Mia s.r.l. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict' + +const { Command } = require('commander') +const { MongoClient } = require('mongodb') +const { faker } = require('@faker-js/faker') + +function generateCustomers({ index, shopCount }) { + const firstName = faker.person.firstName() + const lastName = faker.person.lastName() + + const fullNameCode = `${firstName}.${lastName}` + // email is generated manually to ensure unicity + const email = `${fullNameCode}.${faker.number.int({ max: 99 })}@email.com` + + const creditCardDetail = { + name: `${firstName} ${lastName}`, + cardNo: faker.finance.creditCardNumber(), + expirationDate: `${faker.number.int({ min: 1, max: 12 })}/${faker.number.int({ min: 2024, max: 2031 })}`, + cvv: faker.finance.creditCardCVV, + } + + const address = { + line: faker.location.street(), + city: faker.location.city(), + county: faker.location.county(), + country: faker.location.country(), + } + + const socialNetworkProfiles = { + twitter: `http://www.xcom/${fullNameCode}`, + instagram: `http://www.instagram.com/${fullNameCode}`, + facebook: `http://www.facebook.com/${fullNameCode}`, + threads: `http://www.threads.com/@${fullNameCode}`, + reddit: `http://www.reddit.com/u/${fullNameCode}`, + linkedin: `http://www.linked.in/${fullNameCode}`, + tiktok: `http://www.tiktok.co/${fullNameCode}`, + } + + const purchasesCount = faker.number.int({ min: 1, max: 51 }) + const purchases = [] + for (let i = 0; i < purchasesCount; i++) { + purchases.push({ + name: faker.commerce.productName(), + category: faker.commerce.department(), + price: faker.commerce.price({ min: '1', symbol: '$' }), + employeeId: faker.number.int({ min: 1, max: shopCount * 10 }), + boughtOnline: faker.datatype.boolean(0.2), + }) + } + + return { + updaterId: faker.string.uuid(), + updatedAt: faker.date.recent(), + creatorId: faker.string.uuid(), + createdAt: faker.date.past(), + __STATE__: 'PUBLIC', + customerId: index, + firstName, + lastName, + gender: faker.person.gender(), + birthDate: faker.date.birthdate(), + creditCardDetail, + canBeContacted: faker.datatype.boolean(0.9), + email, + phoneNumber: faker.phone.number(), + address, + socialNetworkProfiles, + subscriptionNumber: faker.finance.creditCardNumber(), + shopID: faker.number.int({ min: 1, max: shopCount }), + purchasesCount, + purchases, + detail: faker.hacker.phrase(), + } +} + +async function generateData(options) { + const { + connectionString = 'mongodb://localhost:27017', + database = 'bench.test', + number = 100000, + shopCount = 250, + } = options + // #region constants + const customerCollectionName = 'customers' + const customerBatchSize = number / 10 + // #endregion + + const mongo = new MongoClient(connectionString) + await mongo.connect() + + const coll = mongo.db(database).collection(customerCollectionName) + + try { + let i = number + while (i > 0) { + process.stdout.write(`\rStarting the creation of ${number} documents for collection "customers".`) + const numberToGenerate = Math.min(customerBatchSize, i) + + const users = [] + for (let j = 0; j < numberToGenerate; j++) { + users.push(generateCustomers({ index: j + i, shopCount })) + } + + // eslint-disable-next-line no-await-in-loop + await coll.insertMany(users) + + i -= numberToGenerate + process.stdout.write(`\r(${number - i}/${number}) ${((number - i) / number * 100).toFixed(2)}%`) + } + } catch (error) { + console.error(`failed to generate data: ${error}`) + } finally { + await mongo.close() + } +} + +async function main() { + const program = new Command() + + program + .option('-c, --connectionString ', 'MongoDB connection string', 'mongodb://localhost:27017') + .option('-d, --database ', 'MongoDB database name', 'bench-test') + .option('-n, --number ', 'Number of documents to generate', 100000) + .option('-s, --shopCount ', 'Number of shops to be used inside the "shopID" field inside each database document', 250) + .action(generateData) + + await program.parseAsync() +} + +if (require.main === module) { + main() + .then(() => { + console.info(`\n\n 🦋 records successfully created\n`) + process.exitCode = 0 + }) + .catch(error => { + console.error(`\n ❌ failed to create records: ${error.message}\n`) + process.exitCode = 1 + }) +} diff --git a/bench/utils/healthcheck.js b/bench/utils/healthcheck.js new file mode 100755 index 00000000..487bafe4 --- /dev/null +++ b/bench/utils/healthcheck.js @@ -0,0 +1,27 @@ +/* + * Copyright 2023 Mia s.r.l. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict' + +async function main() { + try { + await fetch('http://localhost:3000/-/healthz') + } catch (error) { + process.exitCode = 1 + } +} + +main() diff --git a/package-lock.json b/package-lock.json index 36c20129..6c8a47f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@fastify/env": "^4.3.0", "@fastify/mongodb": "^8.0.0", - "@fastify/multipart": "^8.0.0", + "@fastify/multipart": "^8.1.0", "@mia-platform/lc39": "^7.0.3", "@mia-platform/mongodb-healthchecker": "^1.0.1", "ajv-formats": "^2.1.1", @@ -36,13 +36,15 @@ "xlsx-write-stream": "^1.0.0" }, "devDependencies": { + "@faker-js/faker": "^8.3.1", "@mia-platform/eslint-config-mia": "^3.0.0", "abstract-logging": "^2.0.1", + "commander": "^11.1.0", "eslint": "^8.56.0", "fastbench": "^1.0.1", "form-data": "^4.0.0", "mock-require": "^3.0.3", - "mongodb": "^6.2.0", + "mongodb": "^6.3.0", "node-xlsx": "^0.23.0", "pre-commit": "^1.2.2", "swagger-parser": "^10.0.3", @@ -236,6 +238,22 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@faker-js/faker": { + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-8.3.1.tgz", + "integrity": "sha512-FdgpFxY6V6rLZE9mmIBb9hM0xpfvQOSNOLnzolzKwsE1DH+gC7lEKV1p1IbR0lAYyvYd5a4u3qWJzowUkw1bIw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/fakerjs" + } + ], + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0", + "npm": ">=6.14.13" + } + }, "node_modules/@fastify/accept-negotiator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@fastify/accept-negotiator/-/accept-negotiator-1.1.0.tgz", diff --git a/package.json b/package.json index f4d9bd8a..d5ce0ac1 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,8 @@ "url": "https://github.com/mia-platform/crud-service" }, "scripts": { - "bench": "node bench/queryParser.bench.js", + "bench:queryParser": "node bench/queryParser.bench.js", + "bench:init": "docker compose -f bench/docker-compose.yml up -d --force-recreate && node bench/utils/generate-customer-data.js", "checkonly": "! grep -r '\\.only' tests/", "coverage": "npm run unit -- --jobs=4", "postcoverage": "tap report --coverage-report=lcovonly --coverage-report=text-summary", @@ -59,8 +60,10 @@ "ajv": "^8.12.0" }, "devDependencies": { + "@faker-js/faker": "^8.3.1", "@mia-platform/eslint-config-mia": "^3.0.0", "abstract-logging": "^2.0.1", + "commander": "^11.1.0", "eslint": "^8.56.0", "fastbench": "^1.0.1", "form-data": "^4.0.0", @@ -89,6 +92,7 @@ "jsx": true } }, + "ignorePatterns": ["bench"], "overrides": [ { "files": [