Skip building the project for the SQL validation if it isn't needed

spectacles/client.py

@@ -66,17 +66,26 @@ def expired(self) -> bool:
         return False if time.time() < self.expires_at else True
 
 
+def log_backoff(details: dict) -> None:
+    logger.debug(
+        f"Backing off {details['wait']:0.1f} seconds after {details['tries']} tries. "
+        f"Error: {details['exception'].__class__.__name__}"
+    )
+
+
 def backoff_with_exceptions(func: Callable[..., Any]) -> Callable[..., Any]:
     @backoff.on_exception(
         backoff.expo,
         STATUS_EXCEPTIONS,
         giveup=giveup_unless_bad_gateway,
         max_tries=DEFAULT_RETRIES,
+        on_backoff=log_backoff,
     )
     @backoff.on_exception(
         backoff.expo,
         NETWORK_EXCEPTIONS,
         max_tries=DEFAULT_NETWORK_RETRIES,
+        on_backoff=log_backoff,
     )
     async def wrapper(*args: Any, **kwargs: Any) -> Any:
         if asyncio.iscoroutinefunction(func):

spectacles/lookml.py

@@ -307,6 +307,7 @@ class Project(LookMlObject):
     def __init__(self, name: str, models: Sequence[Model]) -> None:
         self.name = name
         self.models = models
+        self._is_complete = False
 
     def __eq__(self, other: Any) -> bool:
         if not isinstance(other, Project):
@@ -344,6 +345,14 @@ def iter_dimensions(self, errored: bool = False) -> Iterable[Dimension]:
                 else:
                     yield dimension
 
+    @property
+    def is_complete_project(self) -> bool:
+        return self._is_complete
+
+    @is_complete_project.setter
+    def is_complete_project(self, value: bool) -> None:
+        self._is_complete = value
+
     @property
     def errored(self) -> Optional[bool]:
         if self.queried:
@@ -507,28 +516,45 @@ async def build_project(
     include_dimensions: bool = False,
     ignore_hidden_fields: bool = False,
     include_all_explores: bool = False,
+    get_full_project: bool = True,
 ) -> Project:
     """Creates an object (tree) representation of a LookML project."""
     if filters is None:
         filters = ["*/*"]
+    is_complete_project = False
+
+    if get_full_project:
+        models = []
+        fields = ["name", "project_name", "explores"]
+        for lookmlmodel in await client.get_lookml_models(fields=fields):
+            model = Model.from_json(lookmlmodel)
+            if model.project_name == name:
+                models.append(model)
+
+        if not models:
+            raise LookMlNotFound(
+                name="project-models-not-found",
+                title="No configured models found for the specified project.",
+                detail=(
+                    f"Go to {client.base_url}/projects and confirm "
+                    "a) at least one model exists for the project and "
+                    "b) it has an active configuration."
+                ),
+            )
+        is_complete_project = True
 
-    models = []
-    fields = ["name", "project_name", "explores"]
-    for lookmlmodel in await client.get_lookml_models(fields=fields):
-        model = Model.from_json(lookmlmodel)
-        if model.project_name == name:
-            models.append(model)
-
-    if not models:
-        raise LookMlNotFound(
-            name="project-models-not-found",
-            title="No configured models found for the specified project.",
-            detail=(
-                f"Go to {client.base_url}/projects and confirm "
-                "a) at least one model exists for the project and "
-                "b) it has an active configuration."
-            ),
-        )
+    else:
+        # Create a project with only the models specified in the filters
+        logger.debug("Building project with only the filtered models")
+        models: Dict[str, Model] = {}
+        for filter in filters:
+            model, explore = filter.split("/")
+            if model not in models:
+                models[model] = Model(name=model, project_name=name, explores=[])
+            if explore not in models[model].explores:
+                models[model].explores.append(Explore(name=explore, model_name=model))
+        project = Project(name=name, models=models.values())
+        models = project.models
 
     # Prune to selected explores for non-content validators
     if not include_all_explores:
@@ -555,4 +581,6 @@ async def build_project(
     else:
         project = Project(name, [m for m in models if len(m.explores) > 0])
 
+    # Indicates whether the project has all of the models/explores or just the selected ones
+    project.is_complete_project = is_complete_project
     return project

spectacles/runner.py

@@ -7,7 +7,7 @@
 from spectacles.client import LOOKML_VALIDATION_TIMEOUT, LookerClient
 from spectacles.exceptions import LookerApiError, SpectaclesException, SqlError
 from spectacles.logger import GLOBAL_LOGGER as logger
-from spectacles.lookml import CompiledSql, Explore, build_project
+from spectacles.lookml import CompiledSql, Explore, build_project, Project
 from spectacles.models import JsonDict, SkipReason
 from spectacles.printer import print_header
 from spectacles.utils import time_hash
@@ -355,6 +355,10 @@ async def validate_sql(
     ) -> JsonDict:
         if filters is None:
             filters = ["*/*"]
+        else:
+            # Only build the full project from the API if we're using a wildcard filter and not in incremental mode
+            get_full_project = any("*" in f for f in filters) or incremental
+
         validator = SqlValidator(self.client, concurrency, runtime_threshold)
         ephemeral = True if incremental else None
         # Create explore-level tests for the desired ref
@@ -367,6 +371,7 @@ async def validate_sql(
                 filters=filters,
                 include_dimensions=True,
                 ignore_hidden_fields=ignore_hidden_fields,
+                get_full_project=get_full_project,
             )
             base_explores: Set[CompiledSql] = set()
             if incremental:
@@ -532,8 +537,16 @@ async def validate_content(
         exclude_personal: bool = False,
         folders: Optional[List[str]] = None,
     ) -> JsonDict:
-        if filters is None:
-            filters = ["*/*"]
+        logger.debug(
+            f"Validating content. ref={ref}, filters={filters}, incremental={incremental}",
+        )
+        if filters is not None or filters == ["*/*"]:
+            # Only build the full project from the API if we're using a wildcard filter and not in incremental mode
+            get_full_project = any("*" in f for f in filters if f != "*/*") or (
+                not incremental
+            )
+            logger.debug(f"get_full_project = {get_full_project}")
+
         if folders is None:
             folders = []