From 911e618a3162877aea1c13fb0e2632c264c8e724 Mon Sep 17 00:00:00 2001 From: peitili <62725319+peitili@users.noreply.github.com> Date: Fri, 15 Oct 2021 15:39:03 -0700 Subject: [PATCH] Make buffer_cfg work for meta tile build (#401) --- tilequeue/process.py | 26 ++++++++++++-------------- tilequeue/query/rawr.py | 24 ++++++++++++++++-------- tilequeue/transform.py | 25 +++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 22 deletions(-) diff --git a/tilequeue/process.py b/tilequeue/process.py index 39a2e5ee..182af26b 100644 --- a/tilequeue/process.py +++ b/tilequeue/process.py @@ -12,7 +12,7 @@ from tilequeue.tile import calc_meters_per_pixel_dim from tilequeue.tile import coord_to_mercator_bounds from tilequeue.tile import normalize_geometry_type -from tilequeue.transform import mercator_point_to_lnglat +from tilequeue.transform import mercator_point_to_lnglat, calc_max_padded_bounds from tilequeue.transform import transform_feature_layers_shape from tilequeue import utils from zope.dottedname.resolve import resolve @@ -518,7 +518,7 @@ def process_coord(coord, nominal_zoom, feature_layers, post_process_data, return all_formatted_tiles, extra_data -def convert_source_data_to_feature_layers(rows, layer_data, bounds, zoom): +def convert_source_data_to_feature_layers(rows, layer_data, unpadded_bounds, zoom): # TODO we might want to fold in the other processing into this # step at some point. This will prevent us from having to iterate # through all the features again. @@ -581,12 +581,9 @@ def convert_source_data_to_feature_layers(rows, layer_data, bounds, zoom): for layer_datum in layer_data: layer_name = layer_datum['name'] features = features_by_layer[layer_name] - # TODO padded bounds - padded_bounds = dict( - polygon=bounds, - line=bounds, - point=bounds, - ) + query_bounds_pad_fn = layer_datum['query_bounds_pad_fn'] + + padded_bounds = query_bounds_pad_fn(unpadded_bounds, calc_meters_per_pixel_dim(zoom)) feature_layer = dict( name=layer_name, features=features, @@ -748,24 +745,25 @@ def __init__(self, coord, metatile_zoom, fetch_fn, layer_data, self.cfg_tile_sizes = cfg_tile_sizes self.log_fn = None - def fetch(self): - unpadded_bounds = coord_to_mercator_bounds(self.coord) + self.unpadded_bounds = coord_to_mercator_bounds(self.coord) + meters_per_pixel_dim = calc_meters_per_pixel_dim(self.coord.zoom) + self.max_padded_bounds = calc_max_padded_bounds(self.unpadded_bounds, meters_per_pixel_dim, self.buffer_cfg) + def fetch(self): cut_coords_by_zoom = calculate_cut_coords_by_zoom( self.coord, self.metatile_zoom, self.cfg_tile_sizes, self.max_zoom) feature_layers_by_zoom = {} for nominal_zoom, _ in cut_coords_by_zoom.items(): - source_rows = self.fetch_fn(nominal_zoom, unpadded_bounds) + source_rows = self.fetch_fn(nominal_zoom, self.max_padded_bounds) feature_layers = convert_source_data_to_feature_layers( - source_rows, self.layer_data, unpadded_bounds, self.coord.zoom) + source_rows, self.layer_data, self.unpadded_bounds, self.coord.zoom) feature_layers_by_zoom[nominal_zoom] = feature_layers self.cut_coords_by_zoom = cut_coords_by_zoom self.feature_layers_by_zoom = feature_layers_by_zoom def process_tiles(self): - unpadded_bounds = coord_to_mercator_bounds(self.coord) all_formatted_tiles = [] all_extra_data = {} @@ -782,7 +780,7 @@ def log_fn(data): feature_layers = self.feature_layers_by_zoom[nominal_zoom] formatted_tiles, extra_data = process_coord( self.coord, nominal_zoom, feature_layers, - self.post_process_data, self.formats, unpadded_bounds, + self.post_process_data, self.formats, self.unpadded_bounds, cut_coords, self.buffer_cfg, self.output_calc_mapping, log_fn=log_fn, ) diff --git a/tilequeue/query/rawr.py b/tilequeue/query/rawr.py index 013720b8..6f677054 100644 --- a/tilequeue/query/rawr.py +++ b/tilequeue/query/rawr.py @@ -724,9 +724,11 @@ def _add_feature(source, feature): return source_features.iteritems() - def __call__(self, zoom, unpadded_bounds): + def __call__(self, zoom, bounds): + """ The bounds is either an unpadded bounds if buffer_cfg is not set + or a padded bounds if buffer_cfg is set upstream """ read_rows = [] - bbox = box(*unpadded_bounds) + bbox = box(*bounds) # check that the call is fetching data which is actually within the # bounds of the tile pyramid. we don't have data outside of that, so @@ -735,25 +737,27 @@ def __call__(self, zoom, unpadded_bounds): # loaded? assert zoom <= self.tile_pyramid.max_z assert zoom >= self.tile_pyramid.z - assert bbox.within(self.tile_pyramid.bbox()) + # assert bbox.within(self.tile_pyramid.bbox()) - for source, features in self._lookup(zoom, unpadded_bounds): + for source, features in self._lookup(zoom, bounds): for (fid, shape, props, layer_min_zooms) in features: read_row = self._parse_row( - zoom, unpadded_bounds, bbox, source, fid, shape, props, + zoom, bounds, bbox, source, fid, shape, props, layer_min_zooms) if read_row: read_rows.append(read_row) return read_rows - def _parse_row(self, zoom, unpadded_bounds, bbox, source, fid, shape, + def _parse_row(self, zoom, bounds, bbox, source, fid, shape, props, layer_min_zooms): + """ The bounds is either an unpadded bounds if buffer_cfg is not set + or a padded bounds if buffer_cfg is set upstream""" # reject any feature which doesn't intersect the given bounds if bbox.disjoint(shape): return None - # place for assembing the read row as if from postgres + # place for assembling the read row as if from postgres read_row = {} generate_label_placement = False @@ -826,11 +830,15 @@ def _parse_row(self, zoom, unpadded_bounds, bbox, source, fid, shape, # if this is a water layer feature, then clip to an expanded # bounding box to avoid tile-edge artefacts. + # Note: As of Oct 2021 we added support for buffer_cfg to + # tilequeue such that this extra buffer for water can actually be + # configured as a water layer buffer_cfg. But we leave as is for + # now. clip_box = bbox if layer_name == 'water': pad_factor = 1.1 clip_box = calculate_padded_bounds( - pad_factor, unpadded_bounds) + pad_factor, bounds) # don't need to clip if geom is fully within the clipping box if box(*shape.bounds).within(clip_box): clip_shape = shape diff --git a/tilequeue/transform.py b/tilequeue/transform.py index a3c76c25..2d32d354 100644 --- a/tilequeue/transform.py +++ b/tilequeue/transform.py @@ -97,6 +97,31 @@ def calc_buffered_bounds( return bounds +def calc_max_padded_bounds(bounds, meters_per_pixel_dim, buffer_cfg): + """ + :return: The bounds expanded by the maximum value in buffer_cfg, default = 0 + """ + max_buffer = 0 + + if buffer_cfg is None: + return bounds + + for _, format_cfg in buffer_cfg.items(): + layer_cfg = format_cfg.get('layer', {}) + if layer_cfg is not None: + for _, value in layer_cfg.items(): + assert isinstance(value, Number) + max_buffer = max(max_buffer, value) + + geometry_cfg = format_cfg.get('geometry', {}) + if geometry_cfg is not None: + for _, value in geometry_cfg.items(): + assert isinstance(value, Number) + max_buffer = max(max_buffer, value) + + return bounds_buffer(bounds, meters_per_pixel_dim * max_buffer) + + def _intersect_multipolygon(shape, tile_bounds, clip_bounds): """ Return the parts of the MultiPolygon shape which overlap the tile_bounds,