Skip to content

Commit

Permalink
small changes
Browse files Browse the repository at this point in the history
  • Loading branch information
sh-rp committed Nov 1, 2023
1 parent e5dd4d6 commit cffa87d
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 6 deletions.
3 changes: 1 addition & 2 deletions dlt/common/configuration/specs/run_configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,7 @@ class RunConfiguration(BaseConfiguration):
"""Maximum delay between http request retries"""
config_files_storage_path: str = "/run/config/"
"""Platform connection"""
beacon_url: Optional[str] = None
beacon_token: Optional[str] = None
beacon_dsn: Optional[str] = None

__section__ = "runtime"

Expand Down
2 changes: 1 addition & 1 deletion dlt/common/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def asdict(self) -> DictStrAny:
"""A dictionary representation of NormalizeInfo that can be loaded with `dlt`"""
d = self._asdict()
# list representation creates a nice table
d["row_counts"] = [(k, v) for k, v in self.row_counts.items()]
d["row_counts"] = [{"table_name": k, "count": v} for k, v in self.row_counts.items()]
return d

def asstr(self, verbosity: int = 0) -> str:
Expand Down
15 changes: 12 additions & 3 deletions dlt/pipeline/platform.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,19 @@
count = 0

def _send_to_beacon(trace: PipelineTrace, step: PipelineStepTrace, pipeline: SupportsPipeline, some):
if pipeline.runtime_config.beacon_token and pipeline.runtime_config.beacon_url:
if pipeline.runtime_config.beacon_dsn:
trace_dump = json.dumps(trace.asdict())
url = f"{pipeline.runtime_config.beacon_url}/pipeline/{pipeline.runtime_config.beacon_token}/traces"
requests.put(url, json=trace_dump)
requests.put(pipeline.runtime_config.beacon_dsn, data=trace_dump)

print("runtime_config")
print(pipeline.runtime_config)
print(pipeline.runtime_config.beacon_token)
print(pipeline.runtime_config.beacon_url)

trace_dump = json.dumps(trace.asdict(), pretty=True)
with open(f"trace-{count}.json", "w") as f:
f.write(trace_dump)


def on_start_trace(trace: PipelineTrace, step: TPipelineStep, pipeline: SupportsPipeline) -> None:
# _send_to_beacon(trace, step, pipeline, None)
Expand Down

0 comments on commit cffa87d

Please sign in to comment.