From 3ed73fc99e82faa76c927828f46c8df6613e9142 Mon Sep 17 00:00:00 2001 From: DistributedDoge <49067954+DistributedDoge@users.noreply.github.com> Date: Tue, 12 Dec 2023 10:17:06 +0100 Subject: [PATCH] =?UTF-8?q?fix:=20=E2=99=BB=20=20ensure=20'metadata'=20col?= =?UTF-8?q?umn=20is=20saved=20as=20JSON?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ggdp/assets.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ggdp/assets.py b/ggdp/assets.py index ccdbe33..ad70b7d 100644 --- a/ggdp/assets.py +++ b/ggdp/assets.py @@ -1,3 +1,5 @@ +import json + import pandas as pd from dagster import asset from fsspec.implementations.http import HTTPFileSystem @@ -76,7 +78,9 @@ def raw_passport_scores() -> pd.DataFrame: @asset def raw_projects() -> pd.DataFrame: - return chain_file_aggregator("projects.json") + projects = chain_file_aggregator("projects.json") + projects['metadata'] = projects['metadata'].apply(json.dumps) + return projects @asset @@ -86,7 +90,9 @@ def raw_prices() -> pd.DataFrame: @asset def raw_rounds() -> pd.DataFrame: - return chain_file_aggregator("rounds.json") + rounds = chain_file_aggregator("rounds.json") + rounds['metadata'] = rounds['metadata'].apply(json.dumps) + return rounds @asset @@ -96,7 +102,9 @@ def raw_round_votes() -> pd.DataFrame: @asset def raw_round_applications() -> pd.DataFrame: - return round_file_aggregator("applications.json") + applications = chain_file_aggregator("applications.json") + applications['metadata'] = applications['metadata'].apply(json.dumps) + return applications @asset