wip

Author: ykoer

.github/actions/log_to_splunk/main.py

@@ -117,7 +117,8 @@ def main():
 
     z = zipfile.ZipFile(io.BytesIO(x.content))
     # z.extractall('/app')
-    log_folder = '/Users/ykoer/Workspace/ykoer/github-actions-example-workflows/.github/actions/log_to_splunk/tmp'
+    #log_folder = '/Users/ykoer/Workspace/ykoer/github-actions-example-workflows/.github/actions/log_to_splunk/tmp'
+    log_folder = '/tmp'
     z.extractall(log_folder)
 
     timestamp = batch = count = 0
@@ -134,19 +135,19 @@ def main():
                 count+=1
                 if timestamp:
                     t2=timestamp
-                timestamp = re.search("\d{4}-\d{2}-\d{2}T\d+:\d+:\d+.\d+Z",line.strip())
+                timestamp = re.search(r"\d{4}-\d{2}-\d{2}T\d+:\d+:\d+\.\d+Z", line.strip())
 
                 if timestamp:
-                    timestamp = re.sub("\dZ","",timestamp.group())
+                    timestamp = re.sub(r"\dZ","",timestamp.group())
                     timestamp = datetime.strptime(timestamp,"%Y-%m-%dT%H:%M:%S.%f")
                     timestamp = (timestamp - datetime(1970,1,1)).total_seconds()
                 else:
                     timestamp=t2
 
                 # find empty lines and skip them
-                x = re.sub("\d{4}-\d{2}-\d{2}T\d+:\d+:\d+.\d+Z","",line.strip())
+                x = re.sub(r"\d{4}-\d{2}-\d{2}T\d+:\d+:\d+.\d+Z","",line.strip())
                 x=x.strip()
-                job_name=re.search("\/\d+\_(?P<job>.*)\.txt",name)
+                job_name=re.search(r"\/\d+\_(?P<job>.*)\.txt",name)
                 job_name=job_name.group('job')
                 fields = {'github_run_id':GITHUB_RUN_ID,'github_workflow_id':GITHUB_WORKFLOWID,'github_job_name':job_name,'line_number':count}
                 if x:
@@ -158,18 +159,20 @@ def main():
 
                 # push every 1000 log lines to splunk as a batch
                 if batch>=1000:
+                    print(f'log_file={name}, batch_number={batch_number}, line_number={count}')
                     batch=0
 
-                    x=requests.post(SPLUNK_HEC_URL, data=eventBatch, headers=headers)
-                    print(f'log_file={name}, batch_number={batch_number}, line_number={count}, request_status_code:{x.status_code}')
+                    # x=requests.post(SPLUNK_HEC_URL, data=eventBatch, headers=headers)
+                    # print(f'log_file={name}, batch_number={batch_number}, line_number={count}, request_status_code:{x.status_code}')
                     eventBatch=""
                     batch_number+=1
                 break
 
         # push the last batch
         if batch>0:
-            x=requests.post(SPLUNK_HEC_URL, data=eventBatch, headers=headers)
-            print(f'log_file={name}, batch_number={batch_number}, line_number={count}, request_status_code:{x.status_code}')
+            print(f'log_file={name}, batch_number={batch_number}, line_number={count}')
+            # x=requests.post(SPLUNK_HEC_URL, data=eventBatch, headers=headers)
+            # print(f'log_file={name}, batch_number={batch_number}, line_number={count}, request_status_code:{x.status_code}')
             eventBatch=""
             batch_number+=1