Skip to content

Commit 7c3311d

Browse files
ykoerykoer
committed
wip
1 parent f1160d2 commit 7c3311d

File tree

6 files changed

+254
-0
lines changed

6 files changed

+254
-0
lines changed

‎.github/actions/log_to_splunk/Dockerfile‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Container image that runs your code
2+
FROM python:3-slim AS builder
3+
4+
# Copies your code file from your action repository to the filesystem path `/` of the container
5+
ADD . /app
6+
WORKDIR /app
7+
8+
RUN pip install --target=/app requests
9+
10+
# Code file to execute when the docker container starts up (`entrypoint.sh`)
11+
FROM gcr.io/distroless/python3-debian10
12+
COPY --from=builder /app /app
13+
ENV PYTHONPATH /app
14+
CMD ["/app/main.py"]

‎.github/actions/log_to_splunk/LICENSE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2022 Splunk GitHub
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎.github/actions/log_to_splunk/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# POST GitHub Workflow Logs to Splunk HTTP Event Collector
2+
test

‎.github/actions/log_to_splunk/action.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# action.yml
2+
name: 'Send Workflow Logs to Splunk'
3+
description: 'Upload GitHub Workflow logs to Splunk HEC'
4+
inputs:
5+
splunk-url:
6+
description: 'Full URL for Splunk HEC endpoint'
7+
required: true
8+
hec-token:
9+
description: 'Splunk HEC Token'
10+
required: true
11+
github-token:
12+
description: 'Github PAT'
13+
required: true
14+
sourcetype:
15+
description: 'Splunk Sourcetype'
16+
default: 'github_workflow_log_action'
17+
source:
18+
description: 'GitHub Workflow name'
19+
default: ${{ github.workflow }}
20+
workflowID:
21+
description: 'The Workflow Run number'
22+
default: ${{ github.run_number}}
23+
outputs:
24+
status:
25+
description: 'value is success/fail based on POST result'
26+
runs:
27+
using: 'docker'
28+
image: 'Dockerfile'

‎.github/actions/log_to_splunk/main.py‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import os
2+
import requests
3+
import json
4+
import zipfile
5+
import io
6+
import glob
7+
import re
8+
from datetime import datetime
9+
10+
def main():
11+
12+
GITHUB_REF=os.environ["GITHUB_REF"]
13+
GITHUB_REPOSITORY=os.environ["GITHUB_REPOSITORY"]
14+
GITHUB_RUN_ID=os.environ["GITHUB_RUN_ID"]
15+
GITHUB_API_URL=os.environ["GITHUB_API_URL"]
16+
GITHUB_WORKFLOWID=os.environ["INPUT_WORKFLOWID"]
17+
GITHUB_TOKEN = os.environ.get("INPUT_GITHUB-TOKEN")
18+
19+
SPLUNK_HEC_URL=os.environ["INPUT_SPLUNK-URL"]+"services/collector/event"
20+
SPLUNK_HEC_TOKEN=os.environ["INPUT_HEC-TOKEN"]
21+
SPLUNK_SOURCE=os.environ["INPUT_SOURCE"]
22+
SPLUNK_SOURCETYPE=os.environ["INPUT_SOURCETYPE"]
23+
24+
batch = count = 0
25+
eventBatch = ""
26+
headers = {"Authorization": "Splunk "+SPLUNK_HEC_TOKEN}
27+
host=os.uname()[1]
28+
29+
summary_url = "{url}/repos/{repo}/actions/runs/{run_id}".format(url=GITHUB_API_URL,repo=GITHUB_REPOSITORY,run_id=GITHUB_WORKFLOWID)
30+
31+
try:
32+
x = requests.get(summary_url, stream=True, auth=('token',GITHUB_TOKEN))
33+
x.raise_for_status()
34+
except requests.exceptions.HTTPError as errh:
35+
output = "GITHUB API Http Error:" + str(errh)
36+
print(f"Error: {output}")
37+
print(f"::set-output name=result::{output}")
38+
return x.status_code
39+
except requests.exceptions.ConnectionError as errc:
40+
output = "GITHUB API Error Connecting:" + str(errc)
41+
print(f"Error: {output}")
42+
print(f"::set-output name=result::{output}")
43+
return x.status_code
44+
except requests.exceptions.Timeout as errt:
45+
output = "Timeout Error:" + str(errt)
46+
print(f"Error: {output}")
47+
print(f"::set-output name=result::{output}")
48+
return x.status_code
49+
except requests.exceptions.RequestException as err:
50+
output = "GITHUB API Non catched error conecting:" + str(err)
51+
print(f"Error: {output}")
52+
print(f"::set-output name=result::{output}")
53+
return x.status_code
54+
except Exception as e:
55+
print("Internal error", e)
56+
return x.status_code
57+
58+
summary = x.json()
59+
60+
summary.pop('repository')
61+
62+
summary["repository"]=summary["head_repository"]["name"]
63+
summary["repository_full"]=summary["head_repository"]["full_name"]
64+
65+
summary.pop('head_repository')
66+
67+
utc_time = datetime.strptime(summary["updated_at"], "%Y-%m-%dT%H:%M:%SZ")
68+
epoch_time = (utc_time - datetime(1970, 1, 1)).total_seconds()
69+
70+
event={'event':json.dumps(summary),'sourcetype':SPLUNK_SOURCETYPE,'source':'workflow_summary','host':host,'time':epoch_time}
71+
event=json.dumps(event)
72+
73+
x=requests.post(SPLUNK_HEC_URL, data=event, headers=headers)
74+
75+
76+
url = "{url}/repos/{repo}/actions/runs/{run_id}/logs".format(url=GITHUB_API_URL,repo=GITHUB_REPOSITORY,run_id=GITHUB_WORKFLOWID)
77+
print(url)
78+
79+
try:
80+
x = requests.get(url, stream=True, auth=('token',GITHUB_TOKEN))
81+
82+
except requests.exceptions.HTTPError as errh:
83+
output = "GITHUB API Http Error:" + str(errh)
84+
print(f"Error: {output}")
85+
print(f"::set-output name=result::{output}")
86+
return
87+
except requests.exceptions.ConnectionError as errc:
88+
output = "GITHUB API Error Connecting:" + str(errc)
89+
print(f"Error: {output}")
90+
print(f"::set-output name=result::{output}")
91+
return
92+
except requests.exceptions.Timeout as errt:
93+
output = "Timeout Error:" + str(errt)
94+
print(f"Error: {output}")
95+
print(f"::set-output name=result::{output}")
96+
return
97+
except requests.exceptions.RequestException as err:
98+
output = "GITHUB API Non catched error conecting:" + str(err)
99+
print(f"Error: {output}")
100+
print(f"::set-output name=result::{output}")
101+
return
102+
103+
z = zipfile.ZipFile(io.BytesIO(x.content))
104+
z.extractall('/app')
105+
106+
timestamp = batch = count = 0
107+
108+
for name in glob.glob('/app/*.txt'):
109+
logfile = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), name.replace('./','')),'r')
110+
Lines = logfile.readlines()
111+
for line in Lines:
112+
113+
if line:
114+
count+=1
115+
if timestamp:
116+
t2=timestamp
117+
timestamp = re.search("\d{4}-\d{2}-\d{2}T\d+:\d+:\d+.\d+Z",line.strip())
118+
119+
if timestamp:
120+
timestamp = re.sub("\dZ","",timestamp.group())
121+
timestamp = datetime.strptime(timestamp,"%Y-%m-%dT%H:%M:%S.%f")
122+
timestamp = (timestamp - datetime(1970,1,1)).total_seconds()
123+
else:
124+
timestamp=t2
125+
126+
x = re.sub("\d{4}-\d{2}-\d{2}T\d+:\d+:\d+.\d+Z","",line.strip())
127+
x=x.strip()
128+
job_name=re.search("\/\d+\_(?P<job>.*)\.txt",name)
129+
job_name=job_name.group('job')
130+
fields = {'lineNumber':count,'workflowID':GITHUB_WORKFLOWID,'job':job_name}
131+
if x:
132+
batch+=1
133+
event={'event':x,'sourcetype':SPLUNK_SOURCETYPE,'source':SPLUNK_SOURCE,'host':host,'time':timestamp,'fields':fields}
134+
eventBatch=eventBatch+json.dumps(event)
135+
else:
136+
print("skipped line "+str(count))
137+
138+
if batch>=1000:
139+
batch=0
140+
x=requests.post(SPLUNK_HEC_URL, data=eventBatch, headers=headers)
141+
eventBatch=""
142+
143+
x=requests.post(SPLUNK_HEC_URL, data=eventBatch, headers=headers)
144+
145+
if __name__ == '__main__':
146+
main()

‎.github/workflows/log_to_splunk.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: Send Workflow Logs to Splunk
2+
3+
# Controls when the action will run.
4+
on:
5+
workflow_dispatch:
6+
workflow_run:
7+
workflows: ["*"]
8+
types:
9+
- completed
10+
11+
env:
12+
triggerID: ${{ github.event.workflow_run.id }}
13+
triggerJob: ${{ github.event.workflow_run.name }}
14+
15+
jobs:
16+
WriteLogs:
17+
runs-on: ubuntu-latest
18+
# if: ${{ github.event.workflow_run.name!='WriteLogs'}}
19+
20+
steps:
21+
- name: Debug Workflow Information
22+
run: |
23+
echo "Trigger ID: ${{ env.triggerID }}"
24+
echo "Trigger Job: ${{ env.triggerJob }}"
25+
echo "Event type: ${{ github.event_name }}"
26+
echo "Workflow ID that triggered this: ${{ github.event.workflow_run.id }}"
27+
echo "Workflow Name that triggered this: ${{ github.event.workflow_run.name }}"
28+
echo "Workflow file: ${{ github.event.workflow_run.path }}"
29+
30+
- uses: actions/checkout@v2
31+
32+
- name: Output Job ID
33+
run: echo ${{ github.event.workflow_run.id }}
34+
35+
- name: Send Workflow logs to Splunk
36+
if: ${{ always() }}
37+
uses: ./.github/actions/log_to_splunk
38+
with:
39+
splunk-url: ${{ secrets.HEC_URL }}
40+
hec-token: ${{ secrets.HEC_TOKEN }}
41+
github-token: ${{ secrets.GITHUB_TOKEN }}
42+
workflowID: ${{ env.triggerID }}
43+
source: ${{ env.triggerJob }}

0 commit comments

Comments
 (0)