diff --git a/spectacles/client.py b/spectacles/client.py index 323f7bcc..06149e26 100644 --- a/spectacles/client.py +++ b/spectacles/client.py @@ -66,17 +66,26 @@ def expired(self) -> bool: return False if time.time() < self.expires_at else True +def log_backoff(details: dict) -> None: + logger.debug( + f"Backing off {details['wait']:0.1f} seconds after {details['tries']} tries. " + f"Error: {details['exception'].__class__.__name__}" + ) + + def backoff_with_exceptions(func: Callable[..., Any]) -> Callable[..., Any]: @backoff.on_exception( backoff.expo, STATUS_EXCEPTIONS, giveup=giveup_unless_bad_gateway, max_tries=DEFAULT_RETRIES, + on_backoff=log_backoff, ) @backoff.on_exception( backoff.expo, NETWORK_EXCEPTIONS, max_tries=DEFAULT_NETWORK_RETRIES, + on_backoff=log_backoff, ) async def wrapper(*args: Any, **kwargs: Any) -> Any: if asyncio.iscoroutinefunction(func): @@ -708,6 +717,9 @@ async def get_lookml_dimensions( try: response.raise_for_status() except httpx.HTTPStatusError as error: + logger.info( + f"Error code: {response.status_code} for url {url} response: {response.text}" + ) raise LookerApiError( name="unable-to-get-dimension-lookml", title="Couldn't retrieve dimensions.", diff --git a/spectacles/lookml.py b/spectacles/lookml.py index 3b40d8e6..dc9e9d92 100644 --- a/spectacles/lookml.py +++ b/spectacles/lookml.py @@ -307,6 +307,7 @@ class Project(LookMlObject): def __init__(self, name: str, models: Sequence[Model]) -> None: self.name = name self.models = models + self._is_complete = False def __eq__(self, other: Any) -> bool: if not isinstance(other, Project): @@ -344,6 +345,14 @@ def iter_dimensions(self, errored: bool = False) -> Iterable[Dimension]: else: yield dimension + @property + def is_complete_project(self) -> bool: + return self._is_complete + + @is_complete_project.setter + def is_complete_project(self, value: bool) -> None: + self._is_complete = value + @property def errored(self) -> Optional[bool]: if self.queried: @@ -507,28 +516,45 @@ async def build_project( include_dimensions: bool = False, ignore_hidden_fields: bool = False, include_all_explores: bool = False, + get_full_project: bool = True, ) -> Project: """Creates an object (tree) representation of a LookML project.""" if filters is None: filters = ["*/*"] + is_complete_project = False + + if get_full_project: + models = [] + fields = ["name", "project_name", "explores"] + for lookmlmodel in await client.get_lookml_models(fields=fields): + model = Model.from_json(lookmlmodel) + if model.project_name == name: + models.append(model) + + if not models: + raise LookMlNotFound( + name="project-models-not-found", + title="No configured models found for the specified project.", + detail=( + f"Go to {client.base_url}/projects and confirm " + "a) at least one model exists for the project and " + "b) it has an active configuration." + ), + ) + is_complete_project = True - models = [] - fields = ["name", "project_name", "explores"] - for lookmlmodel in await client.get_lookml_models(fields=fields): - model = Model.from_json(lookmlmodel) - if model.project_name == name: - models.append(model) - - if not models: - raise LookMlNotFound( - name="project-models-not-found", - title="No configured models found for the specified project.", - detail=( - f"Go to {client.base_url}/projects and confirm " - "a) at least one model exists for the project and " - "b) it has an active configuration." - ), - ) + else: + # Create a project with only the models specified in the filters + logger.debug("Building project with only the filtered models") + models: Dict[str, Model] = {} + for filter in filters: + model, explore = filter.split("/") + if model not in models: + models[model] = Model(name=model, project_name=name, explores=[]) + if explore not in models[model].explores: + models[model].explores.append(Explore(name=explore, model_name=model)) + project = Project(name=name, models=models.values()) + models = project.models # Prune to selected explores for non-content validators if not include_all_explores: @@ -555,4 +581,6 @@ async def build_project( else: project = Project(name, [m for m in models if len(m.explores) > 0]) + # Indicates whether the project has all of the models/explores or just the selected ones + project.is_complete_project = is_complete_project return project diff --git a/spectacles/runner.py b/spectacles/runner.py index 161ec94c..951044ff 100644 --- a/spectacles/runner.py +++ b/spectacles/runner.py @@ -7,7 +7,7 @@ from spectacles.client import LOOKML_VALIDATION_TIMEOUT, LookerClient from spectacles.exceptions import LookerApiError, SpectaclesException, SqlError from spectacles.logger import GLOBAL_LOGGER as logger -from spectacles.lookml import CompiledSql, Explore, build_project +from spectacles.lookml import CompiledSql, Explore, build_project, Project from spectacles.models import JsonDict, SkipReason from spectacles.printer import print_header from spectacles.utils import time_hash @@ -355,6 +355,10 @@ async def validate_sql( ) -> JsonDict: if filters is None: filters = ["*/*"] + else: + # Only build the full project from the API if we're using a wildcard filter and not in incremental mode + get_full_project = any("*" in f for f in filters) or incremental + validator = SqlValidator(self.client, concurrency, runtime_threshold) ephemeral = True if incremental else None # Create explore-level tests for the desired ref @@ -367,6 +371,7 @@ async def validate_sql( filters=filters, include_dimensions=True, ignore_hidden_fields=ignore_hidden_fields, + get_full_project=get_full_project, ) base_explores: Set[CompiledSql] = set() if incremental: @@ -532,8 +537,16 @@ async def validate_content( exclude_personal: bool = False, folders: Optional[List[str]] = None, ) -> JsonDict: - if filters is None: - filters = ["*/*"] + logger.debug( + f"Validating content. ref={ref}, filters={filters}, incremental={incremental}", + ) + if filters is not None or filters == ["*/*"]: + # Only build the full project from the API if we're using a wildcard filter and not in incremental mode + get_full_project = any("*" in f for f in filters if f != "*/*") or ( + not incremental + ) + logger.debug(f"get_full_project = {get_full_project}") + if folders is None: folders = []