Skip to content

Commit

Permalink
Merge pull request #7 from voi-oss/DATAPS-1479-change-default-value-f…
Browse files Browse the repository at this point in the history
…or-page-size-in-dbt-toolkit

feat(dbt cloud): Changing page size
  • Loading branch information
gabby-dol authored Nov 23, 2023
2 parents f604042 + 228475a commit d4624ab
Showing 1 changed file with 2 additions and 5 deletions.
7 changes: 2 additions & 5 deletions src/dbttoolkit/dbt_cloud/clients/dbt_cloud_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,7 @@

logger = get_logger()

LARGE_PAGE_SIZE = 2000
STANDARD_PAGE_SIZE = 500
STANDARD_PAGE_SIZE = 100


@dataclass
Expand All @@ -35,8 +34,7 @@ def retrieve_runs_finished_between(self, start_time: datetime, end_time: datetim
"""
logger.info(f"Retrieving all production runs (between {start_time} and {end_time})")

# As of 2021-08, a page size of 2000 will give us around 1 month worth of runs
completed_runs = self.retrieve_completed_runs(page_size=LARGE_PAGE_SIZE, created_after=start_time)
completed_runs = self.retrieve_completed_runs(created_after=start_time)
runs = [
run
for run in completed_runs
Expand Down Expand Up @@ -97,7 +95,6 @@ def retrieve_artifact_from_run(self, run_id: int, artifact_name: str, *, step: i
headers=self._default_headers(),
)
response.raise_for_status()

return response.json()

def retrieve_completed_runs(
Expand Down

0 comments on commit d4624ab

Please sign in to comment.