From 24925be660acbbb824b130b6cce32e8ed614ecd2 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 29 Oct 2024 17:12:22 +0000 Subject: [PATCH 1/9] feat: retry and uniqid --- app/http.php | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/app/http.php b/app/http.php index 584a7f7..314669b 100644 --- a/app/http.php +++ b/app/http.php @@ -490,7 +490,7 @@ function cleanUp(Orchestration $orchestration, Table $activeRuntimes, array $net ->action(function (string $runtimeId, string $image, string $entrypoint, string $source, string $destination, string $outputDirectory, array $variables, string $runtimeEntrypoint, string $command, int $timeout, bool $remove, float $cpus, int $memory, string $version, string $restartPolicy, array $networks, Orchestration $orchestration, Table $activeRuntimes, Response $response, Log $log) { $runtimeName = System::getHostname() . '-' . $runtimeId; - $runtimeHostname = \uniqid(); + $runtimeHostname = \bin2hex(\random_bytes(16)); $log->addTag('image', $image); $log->addTag('version', $version); @@ -1272,23 +1272,34 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix // Execute function $executionRequest = $version === 'v4' ? $executeV4 : $executeV2; - $executionResponse = \call_user_func($executionRequest); - // Error occured - if ($executionResponse['errNo'] !== 0) { - // Intended timeout error for v2 functions - if ($executionResponse['errNo'] === 110 && $version === 'v2') { - throw new Exception($executionResponse['error'], 400); + while (\microtime(true) - $startTime < $timeout) { + $executionResponse = \call_user_func($executionRequest); + if ($executionResponse['errNo'] === CURLE_OK) { + break; + } + + // Retryable errors, runtime not ready + if (in_array($executionResponse['errNo'], [CURLE_COULDNT_RESOLVE_HOST, CURLE_COULDNT_CONNECT])) { + continue; } + + break; + } + if ($executionResponse['errNo'] !== CURLE_OK) { + $log->addExtra('activeRuntime', $activeRuntimes->get($runtimeName)); $log->addExtra('error', $executionResponse['error']); $log->addTag('hostname', $hostname); - - throw new Exception('Internal curl errors has occurred within the executor! Error Number: ' . $executionResponse['errNo'], 500); + + if ($version === 'v2' && $executionResponse['errNo'] === SOCKET_ETIMEDOUT) { + throw new Exception($executionResponse['error'], 400); + } + + throw new Exception('Internal curl error has occurred within the executor! Error Number: ' . $executionResponse['errNo'], 500); } // Successful execution - ['statusCode' => $statusCode, 'body' => $body, 'logs' => $logs, 'errors' => $errors, 'headers' => $headers] = $executionResponse; $endTime = \microtime(true); From da6e41ce4029e1e9da127ba9d0eff945410c702d Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 29 Oct 2024 17:13:19 +0000 Subject: [PATCH 2/9] chore: fmt --- app/http.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/http.php b/app/http.php index 314669b..7c6071c 100644 --- a/app/http.php +++ b/app/http.php @@ -1278,12 +1278,12 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix if ($executionResponse['errNo'] === CURLE_OK) { break; } - + // Retryable errors, runtime not ready if (in_array($executionResponse['errNo'], [CURLE_COULDNT_RESOLVE_HOST, CURLE_COULDNT_CONNECT])) { continue; } - + break; } @@ -1291,11 +1291,11 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix $log->addExtra('activeRuntime', $activeRuntimes->get($runtimeName)); $log->addExtra('error', $executionResponse['error']); $log->addTag('hostname', $hostname); - + if ($version === 'v2' && $executionResponse['errNo'] === SOCKET_ETIMEDOUT) { throw new Exception($executionResponse['error'], 400); } - + throw new Exception('Internal curl error has occurred within the executor! Error Number: ' . $executionResponse['errNo'], 500); } From c1a2e53366e8eadc5a7ed26b8757b021bcda2151 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 29 Oct 2024 17:16:11 +0000 Subject: [PATCH 3/9] chore: fmt --- app/http.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/http.php b/app/http.php index 7c6071c..b18fca4 100644 --- a/app/http.php +++ b/app/http.php @@ -1273,7 +1273,7 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix // Execute function $executionRequest = $version === 'v4' ? $executeV4 : $executeV2; - while (\microtime(true) - $startTime < $timeout) { + do { $executionResponse = \call_user_func($executionRequest); if ($executionResponse['errNo'] === CURLE_OK) { break; @@ -1285,7 +1285,7 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix } break; - } + } while (\microtime(true) - $startTime < $timeout); if ($executionResponse['errNo'] !== CURLE_OK) { $log->addExtra('activeRuntime', $activeRuntimes->get($runtimeName)); From bd70ea8a00c6f4431984f73287db52142fc88b19 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 29 Oct 2024 17:27:06 +0000 Subject: [PATCH 4/9] chore: refactor --- app/http.php | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/app/http.php b/app/http.php index b18fca4..6df6b42 100644 --- a/app/http.php +++ b/app/http.php @@ -1272,26 +1272,20 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix // Execute function $executionRequest = $version === 'v4' ? $executeV4 : $executeV2; - do { $executionResponse = \call_user_func($executionRequest); - if ($executionResponse['errNo'] === CURLE_OK) { - break; - } - - // Retryable errors, runtime not ready - if (in_array($executionResponse['errNo'], [CURLE_COULDNT_RESOLVE_HOST, CURLE_COULDNT_CONNECT])) { - continue; - } - - break; - } while (\microtime(true) - $startTime < $timeout); + } while ( + \in_array($executionResponse['errNo'], [CURLE_COULDNT_RESOLVE_HOST, CURLE_COULDNT_CONNECT]) && + (\microtime(true) - $startTime < $timeout) + ); + // Error occurred if ($executionResponse['errNo'] !== CURLE_OK) { $log->addExtra('activeRuntime', $activeRuntimes->get($runtimeName)); $log->addExtra('error', $executionResponse['error']); $log->addTag('hostname', $hostname); + // Intended timeout error for v2 functions if ($version === 'v2' && $executionResponse['errNo'] === SOCKET_ETIMEDOUT) { throw new Exception($executionResponse['error'], 400); } From 4886c5c8504529de5373ec074a5ba7e3a45c407e Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 29 Oct 2024 17:50:48 +0000 Subject: [PATCH 5/9] chore: add sleep --- app/http.php | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/app/http.php b/app/http.php index 6df6b42..5941e3f 100644 --- a/app/http.php +++ b/app/http.php @@ -1274,10 +1274,18 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix $executionRequest = $version === 'v4' ? $executeV4 : $executeV2; do { $executionResponse = \call_user_func($executionRequest); - } while ( - \in_array($executionResponse['errNo'], [CURLE_COULDNT_RESOLVE_HOST, CURLE_COULDNT_CONNECT]) && - (\microtime(true) - $startTime < $timeout) - ); + if ($executionResponse['errNo'] === CURLE_OK) { + break; + } + + // Retryable errors, runtime not ready + if (in_array($executionResponse['errNo'], [CURLE_COULDNT_RESOLVE_HOST, CURLE_COULDNT_CONNECT])) { + usleep(100000); + continue; + } + + break; + } while (\microtime(true) - $startTime < $timeout); // Error occurred if ($executionResponse['errNo'] !== CURLE_OK) { From ee130d5a4d43ea3964825e64317a4f35488df039 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Tue, 29 Oct 2024 17:56:19 +0000 Subject: [PATCH 6/9] chore: fmt --- app/http.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/http.php b/app/http.php index 5941e3f..e125c7c 100644 --- a/app/http.php +++ b/app/http.php @@ -1277,13 +1277,13 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix if ($executionResponse['errNo'] === CURLE_OK) { break; } - + // Retryable errors, runtime not ready if (in_array($executionResponse['errNo'], [CURLE_COULDNT_RESOLVE_HOST, CURLE_COULDNT_CONNECT])) { usleep(100000); continue; } - + break; } while (\microtime(true) - $startTime < $timeout); From b87e9d2d3b6345c01603c26771fe1dfb3814d145 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:26:24 +0000 Subject: [PATCH 7/9] feat: max retry attempts --- .env | 2 ++ app/http.php | 20 ++++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.env b/.env index be72ee7..706c156 100644 --- a/.env +++ b/.env @@ -11,3 +11,5 @@ OPR_EXECUTOR_LOGGING_CONFIG= OPR_EXECUTOR_DOCKER_HUB_USERNAME= OPR_EXECUTOR_DOCKER_HUB_PASSWORD= OPR_EXECUTOR_RUNTIME_VERSIONS=v4 +OPR_EXECUTOR_RETRY_ATTEMPTS=4 +OPR_EXECUTOR_RETRY_DELAY_MS=500 \ No newline at end of file diff --git a/app/http.php b/app/http.php index e125c7c..b445e26 100644 --- a/app/http.php +++ b/app/http.php @@ -1272,20 +1272,28 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix // Execute function $executionRequest = $version === 'v4' ? $executeV4 : $executeV2; + + $retryDelayMs = Http::getEnv('OPR_EXECUTOR_RETRY_DELAY_MS', 500); + $retryAttempts = Http::getEnv('OPR_EXECUTOR_RETRY_ATTEMPTS', 4); + + $attempts = 0; do { $executionResponse = \call_user_func($executionRequest); if ($executionResponse['errNo'] === CURLE_OK) { break; } - // Retryable errors, runtime not ready - if (in_array($executionResponse['errNo'], [CURLE_COULDNT_RESOLVE_HOST, CURLE_COULDNT_CONNECT])) { - usleep(100000); - continue; + // Not retryable, return error immediately + if (!in_array($executionResponse['errNo'], [ + CURLE_COULDNT_RESOLVE_HOST, // 6 + CURLE_COULDNT_CONNECT, // 7 + CURLE_GOT_NOTHING, // 52 + ])) { + break; } - break; - } while (\microtime(true) - $startTime < $timeout); + usleep($retryDelayMs * 1000); + } while (++$attempts < $retryAttempts); // Error occurred if ($executionResponse['errNo'] !== CURLE_OK) { From 41f03ab03dc75b57f283327ef095c748c3fe0873 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:06:17 +0000 Subject: [PATCH 8/9] doc: new env vars --- .env | 2 +- README.md | 6 ++++++ app/http.php | 6 +++--- docker-compose.yml | 2 ++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 706c156..6491ac7 100644 --- a/.env +++ b/.env @@ -11,5 +11,5 @@ OPR_EXECUTOR_LOGGING_CONFIG= OPR_EXECUTOR_DOCKER_HUB_USERNAME= OPR_EXECUTOR_DOCKER_HUB_PASSWORD= OPR_EXECUTOR_RUNTIME_VERSIONS=v4 -OPR_EXECUTOR_RETRY_ATTEMPTS=4 +OPR_EXECUTOR_RETRY_ATTEMPTS=5 OPR_EXECUTOR_RETRY_DELAY_MS=500 \ No newline at end of file diff --git a/README.md b/README.md index 6858e1a..0d7b038 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ services: - OPR_EXECUTOR_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD - OPR_EXECUTOR_RUNTIME_VERSIONS + - OPR_EXECUTOR_RETRY_ATTEMPTS + - OPR_EXECUTOR_RETRY_DELAY_MS networks: openruntimes-runtimes: @@ -87,6 +89,8 @@ OPR_EXECUTOR_LOGGING_CONFIG= OPR_EXECUTOR_DOCKER_HUB_USERNAME= OPR_EXECUTOR_DOCKER_HUB_PASSWORD= OPR_EXECUTOR_RUNTIME_VERSIONS=v4 +OPR_EXECUTOR_RETRY_ATTEMPTS=5 +OPR_EXECUTOR_RETRY_DELAY_MS=500 ``` > `OPR_EXECUTOR_CONNECTION_STORAGE` takes a DSN string that represents a connection to your storage device. If you would like to use your local filesystem, you can use `file://localhost`. If using S3 or any other provider for storage, use a DSN of the following format `s3://access_key:access_secret@host:port/bucket_name?region=us-east-1` @@ -196,6 +200,8 @@ docker compose down | OPR_EXECUTOR_DOCKER_HUB_USERNAME | Username for Docker Hub authentication (if applicable) | | OPR_EXECUTOR_DOCKER_HUB_PASSWORD | Password for Docker Hub authentication (if applicable) | | OPR_EXECUTOR_RUNTIME_VERSIONS | Version tag for runtime environments, ex: `v4` | +| OPR_EXECUTOR_RETRY_ATTEMPTS | Number of retry attempts for failed executions, ex: `5` | +| OPR_EXECUTOR_RETRY_DELAY_MS | Delay (in milliseconds) between retry attempts, ex: `500` | ## Contributing diff --git a/app/http.php b/app/http.php index b445e26..0be3596 100644 --- a/app/http.php +++ b/app/http.php @@ -1273,8 +1273,8 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix // Execute function $executionRequest = $version === 'v4' ? $executeV4 : $executeV2; - $retryDelayMs = Http::getEnv('OPR_EXECUTOR_RETRY_DELAY_MS', 500); - $retryAttempts = Http::getEnv('OPR_EXECUTOR_RETRY_ATTEMPTS', 4); + $retryDelayMs = \intval(Http::getEnv('OPR_EXECUTOR_RETRY_DELAY_MS', '500')); + $retryAttempts = \intval(Http::getEnv('OPR_EXECUTOR_RETRY_ATTEMPTS', '5')); $attempts = 0; do { @@ -1293,7 +1293,7 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix } usleep($retryDelayMs * 1000); - } while (++$attempts < $retryAttempts); + } while ((++$attempts < $retryAttempts) || (\microtime(true) - $startTime < $timeout)); // Error occurred if ($executionResponse['errNo'] !== CURLE_OK) { diff --git a/docker-compose.yml b/docker-compose.yml index da616b0..63ce22e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,6 +44,8 @@ services: - OPR_EXECUTOR_DOCKER_HUB_USERNAME - OPR_EXECUTOR_DOCKER_HUB_PASSWORD - OPR_EXECUTOR_RUNTIME_VERSIONS + - OPR_EXECUTOR_RETRY_ATTEMPTS + - OPR_EXECUTOR_RETRY_DELAY_MS volumes: openruntimes-builds: From 593dc78d8723682654a1137717c8610098c7f486 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:23:23 +0000 Subject: [PATCH 9/9] fix: restart policy --- app/http.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/http.php b/app/http.php index 0be3596..47f2b92 100644 --- a/app/http.php +++ b/app/http.php @@ -1287,7 +1287,6 @@ function (string $runtimeId, ?string $payload, string $path, string $method, mix if (!in_array($executionResponse['errNo'], [ CURLE_COULDNT_RESOLVE_HOST, // 6 CURLE_COULDNT_CONNECT, // 7 - CURLE_GOT_NOTHING, // 52 ])) { break; }