Skip to content

Commit

Permalink
Update bigquery.py
Browse files Browse the repository at this point in the history
  • Loading branch information
MaxHalford committed Oct 26, 2023
1 parent 4ede8e3 commit 5c467e4
Showing 1 changed file with 10 additions and 12 deletions.
22 changes: 10 additions & 12 deletions lea/clients/bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@
import os

import pandas as pd
import sqlglot

from lea import views
import lea

from .base import Client

Expand Down Expand Up @@ -51,7 +50,7 @@ def teardown(self):
dataset.location = self.location
self.client.delete_dataset(dataset, delete_contents=True, not_found_ok=True)

def _make_job(self, view: views.SQLView):
def _make_job(self, view: lea.views.SQLView):
query = view.query.replace(f"{self._dataset_name}.", f"{self.dataset_name}.")

return self.client.create_job(
Expand All @@ -76,11 +75,11 @@ def _make_job(self, view: views.SQLView):
}
)

def _create_sql(self, view: views.SQLView):
def _create_sql(self, view: lea.views.SQLView):
job = self._make_job(view)
job.result()

def _create_python(self, view: views.PythonView):
def _create_python(self, view: lea.views.PythonView):
from google.cloud import bigquery

dataframe = self._load_python(view)
Expand All @@ -97,19 +96,18 @@ def _create_python(self, view: views.PythonView):
)
job.result()

def _load_sql(self, view: views.SQLView) -> pd.DataFrame:
def _load_sql(self, view: lea.views.SQLView) -> pd.DataFrame:
query = view.query
if self.username:
query = query.replace(f"{self._dataset_name}_{self.username}.", f"{self.dataset_name}.")
return pd.read_gbq(query, credentials=self.client._credentials)

def list_existing_view_names(self):
return [
table.table_id.split(lea._SEP, 1)
for table in self.client.list_tables(self.dataset_name)
table.table_id.split("__", 1) for table in self.client.list_tables(self.dataset_name)
]

def delete_view(self, view: views.View):
def delete_view(self, view: lea.views.View):
self.client.delete_table(f"{self.project_id}.{self._make_view_path(view)}")

def get_columns(self, schema=None) -> pd.DataFrame:
Expand All @@ -122,15 +120,15 @@ def get_columns(self, schema=None) -> pd.DataFrame:
FROM {schema}.INFORMATION_SCHEMA.COLUMNS
"""
return self._load_sql(
views.GenericSQLView(
lea.views.GenericSQLView(
schema=None, name=None, query=query, sqlglot_dialect=self.sqlglot_dialect
)
)

def _make_view_path(self, view: views.View) -> str:
def _make_view_path(self, view: lea.views.View) -> str:
return f"{self.dataset_name}.{view.schema}{lea._SEP}{view.name}"

def make_test_unique_column(self, view: views.View, column: str) -> str:
def make_test_unique_column(self, view: lea.views.View, column: str) -> str:
return f"""
SELECT {column}, COUNT(*) AS n
FROM {self._make_view_path(view)}
Expand Down

0 comments on commit 5c467e4

Please sign in to comment.