Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add a case for load testing historical events #54

Closed
wants to merge 1 commit into from
Closed

add a case for load testing historical events #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions quest_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,34 @@ func TestLoadStreamBatchWithK6(t *testing.T) {
}
}

func TestLoadHistoricalStreamBatchWithK6(t *testing.T) {
if NewGlob.Mode == "load" {
historicalStream := NewGlob.Stream + "historical"
timeHeader := map[string]string{"X-P-Time-Partition": "source_time"}
CreateStreamWithHeader(t, NewGlob.Client, historicalStream, timeHeader)

cmd := exec.Command("k6",
"run",
"-e", fmt.Sprintf("P_URL=%s", NewGlob.Url.String()),
"-e", fmt.Sprintf("P_USERNAME=%s", NewGlob.Username),
"-e", fmt.Sprintf("P_PASSWORD=%s", NewGlob.Password),
"-e", fmt.Sprintf("P_STREAM=%s", historicalStream),
"-e", fmt.Sprintf("P_SCHEMA_COUNT=%s", schema_count),
"-e", fmt.Sprintf("P_EVENTS_COUNT=%s", schema_count),
"./scripts/load_historical_batch_events.js",
"--vus=", vus,
"--duration=", duration)

cmd.Run()
op, err := cmd.Output()
if err != nil {
t.Log(err)
}
t.Log(string(op))
DeleteStream(t, NewGlob.Client, historicalStream)
}
}

func TestLoadStreamNoBatchWithK6(t *testing.T) {
if NewGlob.Mode == "load" {
cmd := exec.Command("k6",
Expand Down
195 changes: 195 additions & 0 deletions scripts/load_historical_batch_events.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
import http from 'k6/http';
import { check, sleep } from 'k6';
import exec from 'k6/execution';
import encoding from 'k6/encoding';
import { randomString, randomItem, randomIntBetween, uuidv4 } from 'https://jslib.k6.io/k6-utils/1.4.0/index.js'

export const options = {
discardResponseBodies: true,
scenarios: {
contacts: {
executor: 'constant-vus',
vus: 10,
duration: "5m",
},
},
};

function current_time() {
const startDate = new Date('2022-01-01');
const endDate = new Date('2022-01-31');
const timeDiff = endDate.getTime() - startDate.getTime();
const randomTime = Math.random() * timeDiff;
const randomDate = new Date(startDate.getTime() + randomTime);
return(randomDate.toISOString());
}

function schemas() {
return Number(__ENV.P_SCHEMA_COUNT)
}

function events_per_call() {
return Number(__ENV.P_EVENTS_COUNT)
}

const common_schema = [
{ "name": "source_time", "gen": current_time, "arg": null },
{ "name": "level", "gen": randomItem, "arg": ["info", "warn", "error"] },
{ "name": "message", "gen": randomItem, "arg": ["Application started", "Application is failing", "Logging a request"] },
{ "name": "version", "gen": randomItem, "arg": ["1.0.0", "1.1.0", "1.2.0"] },
{ "name": "user_id", "gen": randomIntBetween, "arg": [10000, 100000] },
{ "name": "device_id", "gen": randomIntBetween, "arg": [0, 5000] },
{ "name": "session_id", "gen": randomItem, "arg": ["abc", "pqr", "xyz"] },
{ "name": "os", "gen": randomItem, "arg": ["macOS", "Linux", "Windows"] },
{ "name": "host", "gen": randomItem, "arg": ["192.168.1.100", "112.168.1.110", "172.162.1.120"] },
{ "name": "uuid", "gen": uuidv4, "arg": null },
]

const add_fields = {
"location": { "gen": randomString, "arg": 16 },
"timezone": { "gen": randomString, "arg": 3 },
"user_agent": { "gen": randomItem, "arg": ["Banana", "PineApple", "PearOS", "OrangeOS", "Kiwi"] },
"runtime": { "gen": randomString, "arg": 3 },
"request_body": { "gen": randomString, "arg": 100 },
"status_code": { "gen": randomItem, "arg": [200, 300, 400, 500] },
"response_time": { "gen": randomItem, "arg": [12, 22, 34, 56, 70, 112] },
"process_id": { "gen": randomIntBetween, "arg": [100, 1000] },
"app_meta": { "gen": randomString, "arg": 24 }
}

const addFields_permutation = [
['location', 'request_body', 'status_code', 'app_meta'],
['timezone', 'user_agent', 'runtime', 'app_meta'],
['timezone', 'request_body', 'response_time', 'process_id'],
['timezone', 'user_agent', 'request_body', 'process_id'],
['runtime', 'status_code', 'response_time', 'process_id'],
['location', 'user_agent', 'runtime', 'process_id'],
['location', 'timezone', 'request_body', 'response_time'],
['timezone', 'user_agent', 'status_code', 'process_id'],
['timezone', 'runtime', 'request_body', 'response_time'],
['timezone', 'status_code', 'response_time', 'process_id'],
['timezone', 'runtime', 'status_code', 'response_time'],
['location', 'timezone', 'response_time', 'process_id'],
['location', 'timezone', 'runtime', 'process_id'],
['user_agent', 'runtime', 'status_code', 'process_id'],
['timezone', 'response_time', 'process_id', 'app_meta'],
['location', 'user_agent', 'status_code', 'response_time'],
['timezone', 'user_agent', 'runtime', 'status_code'],
['request_body', 'status_code', 'process_id', 'app_meta'],
['location', 'user_agent', 'runtime', 'request_body'],
['location', 'timezone', 'status_code', 'response_time'],
['location', 'user_agent', 'response_time', 'process_id'],
['timezone', 'runtime', 'response_time', 'process_id'],
['location', 'timezone', 'user_agent', 'runtime'],
['user_agent', 'request_body', 'status_code', 'process_id'],
['runtime', 'request_body', 'response_time', 'process_id'],
['location', 'runtime', 'request_body', 'app_meta'],
['runtime', 'response_time', 'process_id', 'app_meta'],
['location', 'runtime', 'status_code', 'app_meta'],
['location', 'runtime', 'process_id', 'app_meta'],
['location', 'request_body', 'process_id', 'app_meta'],
['location', 'timezone', 'runtime', 'request_body'],
['timezone', 'user_agent', 'response_time', 'app_meta'],
['runtime', 'request_body', 'status_code', 'response_time'],
['location', 'timezone', 'user_agent', 'response_time'],
['location', 'runtime', 'request_body', 'status_code'],
['location', 'user_agent', 'request_body', 'response_time'],
['location', 'status_code', 'process_id', 'app_meta'],
['user_agent', 'status_code', 'response_time', 'app_meta'],
['timezone', 'request_body', 'status_code', 'response_time'],
['user_agent', 'runtime', 'request_body', 'process_id'],
['user_agent', 'runtime', 'response_time', 'app_meta'],
['user_agent', 'request_body', 'response_time', 'app_meta']
];

function generateOverlappingSchemas(number = 5) {
const schemas = [];
addFields_permutation.slice(0, number).forEach((listOfFields) => {
const new_schema = [...common_schema];
listOfFields.forEach((field) => {
let gen_value = add_fields[field];
let obj = { "name": field };
Object.assign(obj, gen_value);
new_schema.push(obj)
})
schemas.push(new_schema)
})

return schemas;
}

function generateJSON(schema) {
let json = {};
schema.forEach(item => {
let { name, gen, arg } = item;
var value;
if ((gen === current_time) || (gen === uuidv4)) {
value = gen();
}
else if (gen === randomIntBetween) {
value = gen(...arg);
}
else {
value = gen(arg)
}
json[name] = value;
});
return json;
}

function generateEvents(numberOfEvents = 3) {
const events = [];
let numberOfSchemas = schemas();

if (!numberOfSchemas) {
numberOfSchemas = 5
}

let listOfSchema = generateOverlappingSchemas(numberOfSchemas);

for (let i = 0; i < numberOfEvents; i++) {
for (let j = 0; j < listOfSchema.length; j++) {
events.push(generateJSON(listOfSchema[j]))
}
}

return events
}

export default function () {
sleep(0.1);
const url = `${__ENV.P_URL}/api/v1/ingest`;
const credentials = `${__ENV.P_USERNAME}:${__ENV.P_PASSWORD}`;
const encodedCredentials = encoding.b64encode(credentials);

const params = {
headers: {
'Content-Type': 'application/json',
'Authorization': 'Basic ' + `${encodedCredentials}`,
'X-P-STREAM': `${__ENV.P_STREAM}`,
'X-P-META-Host': '10.116.0.3',
'X-P-META-Source': 'quest-test',
'X-P-META-ContainerName': 'log-generator',
'X-P-META-ContainerImage': 'ghcr.io/parseablehq/quest',
'X-P-META-Namespace': 'go-apasdp',
'X-P-META-PodLabels': 'app=go-app,pod-template-hash=6c87bc9cc9',
}
}

let events = events_per_call();

if (!events) {
events = 10
}

let batch_requests = JSON.stringify(generateEvents(events));
let response = http.post(url, batch_requests, params);

if (
!check(response, {
'status code MUST be 200': (res) => res.status == 200,
})
) {
exec.test.abort("Failed to send event.. status != 200");
}
}
10 changes: 10 additions & 0 deletions test_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,16 @@ func CreateStream(t *testing.T, client HTTPClient, stream string) {
require.Equalf(t, 200, response.StatusCode, "Server returned http code: %s", response.Status)
}

func CreateStreamWithHeader(t *testing.T, client HTTPClient, stream string, header map[string]string) {
req, _ := client.NewRequest("PUT", "logstream/"+stream, nil)
for k, v := range header {
req.Header.Add(k, v)
}
response, err := client.Do(req)
require.NoErrorf(t, err, "Request failed: %s", err)
require.Equalf(t, 200, response.StatusCode, "Server returned http code: %s", response.Status)
}

func DeleteStream(t *testing.T, client HTTPClient, stream string) {
req, _ := client.NewRequest("DELETE", "logstream/"+stream, nil)
response, err := client.Do(req)
Expand Down
Loading