Skip to content

Commit

Permalink
fix: requests error handling (#82)
Browse files Browse the repository at this point in the history
- added locally built docker build-image
- fixed retries for 5xx errors
- fixed various leftover build issues
LOG-11814
  • Loading branch information
dkhokhlov authored May 10, 2022
1 parent 6d5d432 commit e412859
Show file tree
Hide file tree
Showing 7 changed files with 653 additions and 477 deletions.
7 changes: 7 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
FROM condaforge/miniforge3:4.12.0-0

RUN conda install -y gcc pip poetry=1.1.7 git
RUN mkdir /workdir && chmod 777 /workdir
RUN git config --global --add safe.directory /workdir
WORKDIR /workdir

19 changes: 6 additions & 13 deletions Jenkinsfile
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,12 @@ def DEFAULT_BRANCH = 'master'
def CURRENT_BRANCH = [env.CHANGE_BRANCH, env.BRANCH_NAME]?.find{branch -> branch != null}

pipeline {
agent none
agent {
node {
label 'ec2-fleet'
customWorkspace "${PROJECT_NAME}-${BUILD_NUMBER}"
}
}

options {
timestamps()
Expand All @@ -18,12 +23,6 @@ pipeline {

stages {
stage('Test') {
agent {
node {
label 'ec2-fleet'
customWorkspace "${PROJECT_NAME}-${BUILD_NUMBER}"
}
}

steps {
sh 'make install lint test'
Expand All @@ -45,12 +44,6 @@ pipeline {
}

stage('Release') {
agent {
node {
label 'ec2-fleet'
customWorkspace "${PROJECT_NAME}-${BUILD_NUMBER}"
}
}

stages {
stage('dry run') {
Expand Down
29 changes: 17 additions & 12 deletions Makefile
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ include .config.mk
DOCKER = docker
DOCKER_RUN := $(DOCKER) run --rm -i
WORKDIR :=/workdir
DOCKER_COMMAND := $(DOCKER_RUN) -v $(PWD):$(WORKDIR):Z -w $(WORKDIR) \
DOCKER_COMMAND := $(DOCKER_RUN) -u "$(shell id -u)":"$(shell id -g)" -v $(PWD):$(WORKDIR):Z -w $(WORKDIR) \
-e XDG_CONFIG_HOME=$(WORKDIR) \
-e XDG_CACHE_HOME=$(WORKDIR) \
-e POETRY_CACHE_DIR=$(WORKDIR)/.cache \
Expand All @@ -21,13 +21,19 @@ DOCKER_COMMAND := $(DOCKER_RUN) -v $(PWD):$(WORKDIR):Z -w $(WORKDIR) \
-e GIT_AUTHOR_EMAIL \
-e GIT_COMMITTER_NAME \
-e GIT_COMMITTER_EMAIL \
us.gcr.io/logdna-k8s/python:3.7-ci
logdna-poetry:local


POETRY_COMMAND := $(DOCKER_COMMAND) poetry

# Exports the variables for shell use
export

# build image
.PHONY:build-image
build-image:
DOCKER_BUILDKIT=1 $(DOCKER) build -t logdna-poetry:local .

# This helper function makes debugging much easier.
.PHONY:debug-%
debug-%: ## Debug a variable by calling `make debug-VARIABLE`
Expand All @@ -39,30 +45,31 @@ help: ## Show this help, includes list of all actions.
@awk 'BEGIN {FS = ":.*?## "}; /^.+: .*?## / && !/awk/ {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' ${MAKEFILE_LIST}

.PHONY:run
run: ## purge build time artifacts
run: install ## purge build time artifacts
$(DOCKER_COMMAND) bash

.PHONY:clean
clean: ## purge build time artifacts
rm -rf dist/ build/ coverage/ pypoetry/ pip/ **/__pycache__/ .pytest_cache/ .cache .coverage

.PHONY:changelog
changelog: ## print the next version of the change log to stdout
changelog: install ## print the next version of the change log to stdout
$(POETRY_COMMAND) run semantic-release changelog --unreleased

.PHONY:install
install: ## install development and build time dependencies
$(POETRY_COMMAND) install --no-interaction -vvv
install: build-image ## install development and build time dependencies
$(POETRY_COMMAND) install --no-interaction

.PHONY:lint
lint: ## run lint rules and print error report
lint: install ## run lint rules and print error report
$(POETRY_COMMAND) run task lint

.PHONY:lint-fix
lint-fix:## attempt to auto fix linting error and report remaining errors
lint-fix: install ## attempt to auto fix linting error and report remaining errors
$(POETRY_COMMAND) run task lint:fix

.PHONY:package
package: ## Generate a python sdist and wheel
package: install ## Generate a python sdist and wheel
$(POETRY_COMMAND) build

.PHONY:release
Expand All @@ -71,7 +78,6 @@ release: clean install fetch-tags ## run semantic release build and publish resu

.PHONY: fetch-tags
fetch-tags: ## workaround for jenkins repo cloning behavior
git config remote.origin.url "https://logdnabot:${GH_TOKEN}@github.com/logdna/python"
git fetch origin --tags

.PHONY:release-dry
Expand All @@ -91,6 +97,5 @@ release-major: clean install ## run semantic release build an
$(POETRY_COMMAND) run semantic-release publish --major

.PHONY:test
test: ## run project test suite
test: install ## run project test suite
$(POETRY_COMMAND) run task test

86 changes: 69 additions & 17 deletions logdna/logdna.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,13 @@ def try_request(self):
self.close_flusher()
self.exception_flag = True

def send_request(self, data):
def send_request(self, data): # noqa: max-complexity: 13
"""
Send log data to LogDNA server
Returns:
True - discard flush buffer
False - retry, keep flush buffer
"""
try:
response = requests.post(url=self.url,
json=data,
Expand All @@ -190,38 +196,84 @@ def send_request(self, data):
'now': int(time.time() * 1000)
},
stream=True,
allow_redirects=True,
timeout=self.request_timeout,
headers={'user-agent': self.user_agent})

response.raise_for_status()
status_code = response.status_code
if status_code in [401, 403]:
self.internalLogger.debug(
'Please provide a valid ingestion key.' +
' Discarding flush buffer')
return True

'''
response code:
1XX unexpected status
200 expected status, OK
2XX unexpected status
301 302 303 unexpected status,
per "allow_redirects=True"
3XX unexpected status
401, 403 expected client error,
invalid ingestion key
4XX unexpected client error
500 502 503 507 expected server error, transient
5XX unexpected server error
handling:
expected status discard flush buffer
unexpected status log + discard flush buffer
expected client error log + discard flush buffer
unexpected client error log + discard flush buffer
expected server error log + retry
unexpected server error log + discard flush buffer
'''
if status_code == 200:
return True
return True # discard

if status_code in [400, 500, 504]:
self.internalLogger.debug('The request failed %s. Retrying...',
response.reason)
return True
if isinstance(response.reason, bytes):
# We attempt to decode utf-8 first because some servers
# choose to localize their reason strings. If the string
# isn't utf-8, we fall back to iso-8859-1 for all other
# encodings. (See PR #3538)
try:
reason = response.reason.decode('utf-8')
except UnicodeDecodeError:
reason = response.reason.decode('iso-8859-1')
else:
reason = response.reason

if 200 < status_code <= 399:
self.internalLogger.debug('Unexpected response: %s. ' +
'Discarding flush buffer',
reason)
return True # discard

if status_code in [401, 403]:
self.internalLogger.debug(
'The request failed: %s. Retrying...', response.reason)
'Please provide a valid ingestion key. ' +
'Discarding flush buffer')
return True # discard

if 400 <= status_code <= 499:
self.internalLogger.debug('Client Error: %s. ' +
'Discarding flush buffer',
reason)
return True # discard

if status_code in [500, 502, 503, 507]:
self.internalLogger.debug('Server Error: %s. Retrying...',
reason)
return False # retry

self.internalLogger.debug('The request failed: %s.' +
'Discarding flush buffer',
reason)

except requests.exceptions.Timeout as timeout:
self.internalLogger.debug('Timeout error occurred %s. Retrying...',
self.internalLogger.debug('Timeout Error: %s. Retrying...',
timeout)
return False # retry

except requests.exceptions.RequestException as exception:
self.internalLogger.debug(
'Error sending logs %s. Discarding flush buffer', exception)
return True

return False
return True # discard

def emit(self, record):
msg = self.format(record)
Expand Down
Loading

0 comments on commit e412859

Please sign in to comment.