From 441bd9bc1c9cb99c00574cc324f39f1d1a1a7b92 Mon Sep 17 00:00:00 2001 From: jjrom Date: Thu, 10 Oct 2024 15:38:56 +0200 Subject: [PATCH 1/4] Still error with invalid bbox in JSONUtil. SHould be corrected now --- app/resto/core/RestoConstants.php | 2 +- app/resto/core/RestoModel.php | 2 +- app/resto/core/utils/JSONLDUtil.php | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/resto/core/RestoConstants.php b/app/resto/core/RestoConstants.php index 747afad2..d4be8e4a 100644 --- a/app/resto/core/RestoConstants.php +++ b/app/resto/core/RestoConstants.php @@ -20,7 +20,7 @@ class RestoConstants // [IMPORTANT] Starting resto 7.x, default routes are defined in RestoRouter class // resto version - const VERSION = '9.0.0-RC13'; + const VERSION = '9.0.0-RC14'; /* ============================================================ * NEVER EVER TOUCH THESE VALUES diff --git a/app/resto/core/RestoModel.php b/app/resto/core/RestoModel.php index 5781bb40..74b89b46 100755 --- a/app/resto/core/RestoModel.php +++ b/app/resto/core/RestoModel.php @@ -852,7 +852,7 @@ private function storeFeature($collection, $data, $params) * (do this before getKeywords to avoid iTag process) */ if (isset($productIdentifier) && (new FeaturesFunctions($collection->context->dbDriver))->featureExists($featureId, $collection->context->dbDriver->targetSchema . '.feature')) { - RestoLogUtil::httpError(409, 'Feature ' . $featureId . ' (with productIdentifier=' . $productIdentifier . ') already in database'); + return RestoLogUtil::httpError(409, 'Feature ' . $featureId . ' (with productIdentifier=' . $productIdentifier . ') already in database'); } /* diff --git a/app/resto/core/utils/JSONLDUtil.php b/app/resto/core/utils/JSONLDUtil.php index 83db513a..ce8c756d 100755 --- a/app/resto/core/utils/JSONLDUtil.php +++ b/app/resto/core/utils/JSONLDUtil.php @@ -80,7 +80,7 @@ public static function addDataCatalogMetadata($catalog) if ( isset($catalog['extent'])) { - if ( isset($catalog['extent']['spatial']['bbox']) && is_array($catalog['extent']['spatial']['bbox']) ) { + if ( isset($catalog['extent']['spatial']['bbox']) && is_array($catalog['extent']['spatial']['bbox']) && is_array($catalog['extent']['spatial']['bbox'][0]) ) { $jsonld['spatialCoverage'] = array( '@type' => 'Place', 'geo' => array( @@ -90,7 +90,7 @@ public static function addDataCatalogMetadata($catalog) ); } - if ( isset($catalog['extent']['temporal']['interval']) && is_array($catalog['extent']['temporal']['interval']) ) { + if ( isset($catalog['extent']['temporal']['interval']) && is_array($catalog['extent']['temporal']['interval']) && is_array($catalog['extent']['temporal']['interval'][0]) ) { $jsonld['temporalCoverage'] = join('/', array($catalog['extent']['temporal']['interval'][0][0] ?? '..', $catalog['extent']['temporal']['interval'][0][1] ?? '..')); } From 1ea4fef481dddc87ebb641ef7b2bd8981c8fefa6 Mon Sep 17 00:00:00 2001 From: jjrom Date: Fri, 11 Oct 2024 10:45:20 +0200 Subject: [PATCH 2/4] Correctly handle PUT in catalog counters --- .../core/dbfunctions/CatalogsFunctions.php | 39 ++++++++++++++--- resto-database-model/01_resto_functions.sql | 43 ++++++++----------- 2 files changed, 51 insertions(+), 31 deletions(-) diff --git a/app/resto/core/dbfunctions/CatalogsFunctions.php b/app/resto/core/dbfunctions/CatalogsFunctions.php index de0c8135..0f7f8587 100755 --- a/app/resto/core/dbfunctions/CatalogsFunctions.php +++ b/app/resto/core/dbfunctions/CatalogsFunctions.php @@ -246,7 +246,7 @@ public function storeCatalog($catalog, $userid, $baseUrl, $collectionId, $featur $catalog['description'] ?? null, isset($catalog['id']) ? count(explode('/', $catalog['id'])) : 0, // If no input counter is specified - set to 1 - json_encode($counters, JSON_UNESCAPED_SLASHES), + str_replace('[]', '{}', json_encode($counters, JSON_UNESCAPED_SLASHES)), $catalog['owner'] ?? $userid, json_encode($cleanLinks['links'], JSON_UNESCAPED_SLASHES), RestoConstants::GROUP_DEFAULT_ID, @@ -426,8 +426,10 @@ public function updateCatalog($catalog, $userid, $baseUrl) $path = RestoUtil::path2ltree($catalog['id']); /* - * Add an entry in catalog_feature for each interalItems + * Add an entry in catalog_feature for each interalItems but first remove all items ! */ + $this->removeCatalogFeatures($catalog['id']); + for ($i = 0, $ii = count($cleanLinks['internalItems']); $i < $ii; $i++) { $this->insertIntoCatalogFeature($cleanLinks['internalItems'][$i]['id'], $path, $catalog['id'], $cleanLinks['internalItems'][$i]['collection']); } @@ -453,10 +455,11 @@ public function updateFeatureCatalogsCounters($featureId, $collectionId, $increm { $query = join(' ', array( - 'WITH path_hierarchy AS (SELECT distinct featureid, subpath(path, 0, generate_series(1, nlevel(path))) AS p FROM ' . $this->dbDriver->targetSchema . '.catalog_feature', + 'WITH path_hierarchy AS (SELECT distinct featureid, collection, catalogid, subpath(path, 0, generate_series(1, nlevel(path))) AS p FROM ' . $this->dbDriver->targetSchema . '.catalog_feature', 'WHERE featureid = \'' . pg_escape_string($this->dbDriver->getConnection(), $featureId) . '\')', 'UPDATE ' . $this->dbDriver->targetSchema . '.catalog SET counters=public.increment_counters(counters,' . $increment . ',' . (isset($collectionId) ? '\'' . $collectionId . '\'': 'NULL') . ')', - 'WHERE lower(id) IN (SELECT LOWER(REPLACE(REPLACE(path_hierarchy.p::text, \'_\', \'.\'), \'.\', \'/\')) FROM path_hierarchy)' + // 'WHERE lower(id) IN (SELECT LOWER(REPLACE(REPLACE(path_hierarchy.p::text, \'_\', \'.\'), \'.\', \'/\')) FROM path_hierarchy)' + 'WHERE lower(id) IN (SELECT LOWER(path_hierarchy.catalogid) FROM path_hierarchy)' )); $results = $this->dbDriver->fetch($this->dbDriver->query($query)); @@ -524,6 +527,28 @@ public function removeCatalog($catalogId, $inTransaction = true) } + /** + * Remove features from a catalog i.e. unassociate feature from a catalog + * + * [WARNING] This DOES NOT REMOVE FEATURE IN TABLE feature + * + * @param string $catalogId + */ + private function removeCatalogFeatures($catalogId) + { + + $query = join(' ', array( + 'WITH path_hierarchy AS (SELECT distinct featureid, collection, catalogid, subpath(path, 0, generate_series(1, nlevel(path))) AS p FROM ' . $this->dbDriver->targetSchema . '.catalog_feature', + 'WHERE path = \'' . pg_escape_string($this->dbDriver->getConnection(), RestoUtil::path2ltree($catalogId)) . '\')', + 'UPDATE ' . $this->dbDriver->targetSchema . '.catalog SET counters=public.increment_counters(counters, -13, (SELECT path_hierarchy.collection FROM path_hierarchy LIMIT 1))', + 'WHERE lower(id) IN (SELECT LOWER(path_hierarchy.catalogid) FROM path_hierarchy)' + )); + $this->dbDriver->query($query); + + $results = $this->dbDriver->fetch($this->dbDriver->pQuery('DELETE FROM ' . $this->dbDriver->targetSchema . '.catalog_feature WHERE path ~ $1 RETURNING featureid, collection' , array(RestoUtil::path2ltree($catalogId) . '.*'), 500, 'Cannot delete catalog_feature association for catalog ' . $catalogId)); + + } + /** * Return STAC Summaries from catalogs elements from a type for a given collection * @@ -665,15 +690,17 @@ private function insertIntoCatalogFeature($featureId, $path, $catalogId, $collec $path, $catalogId, $collectionId - ), 500, 'Cannot create association for ' . $featureId . ' intp catalog ' . $catalogId); + ), 500, 'Cannot create association for ' . $featureId . ' in catalog ' . $catalogId); $this->dbDriver->pQuery('INSERT INTO ' . $this->dbDriver->targetSchema . '.catalog_feature (featureid, path, catalogid, collection) SELECT $1, $2::ltree, $3, $4 WHERE NOT EXISTS (SELECT 1 FROM ' . $this->dbDriver->targetSchema . '.catalog_feature WHERE featureid = $1 AND (path <@ $2::ltree OR path @> $2::ltree))', array( $featureId, $path, $catalogId, $collectionId - ), 500, 'Cannot create association for ' . $featureId . ' intp catalog ' . $catalogId); + ), 500, 'Cannot create association for ' . $featureId . ' in catalog ' . $catalogId); + $this->updateFeatureCatalogsCounters($featureId, $collectionId, 1); + } /** diff --git a/resto-database-model/01_resto_functions.sql b/resto-database-model/01_resto_functions.sql index f9a61018..b2c6a044 100644 --- a/resto-database-model/01_resto_functions.sql +++ b/resto-database-model/01_resto_functions.sql @@ -303,45 +303,38 @@ RETURNS JSON AS $$ DECLARE -- Variable to store the total property from JSON total INTEGER; - -- Variable to store individual keys and values from the collections array - collection_key TEXT; - collection_value INTEGER; -- Variable to store the updated collections array updated_collections JSONB; -- Variable to store the updated JSON object updated_counters JSONB; - -- Boolean to check if collection_id exists - collection_exists BOOLEAN := FALSE; + -- Variable to store current value of the specific collection + collection_value INTEGER; BEGIN -- Extract the total property from the JSON object total := (counters->>'total')::INTEGER; - -- Increment the total by the value + -- Increment the total by the input value total := GREATEST(0, total + increment); -- Initialize the updated collections as the original collections updated_collections := counters->'collections'; - -- Check if collection_id is NULL - IF collection_id IS NOT NULL THEN - -- Loop through the collections array and update the collection_id value - FOR collection_key, collection_value IN - SELECT key, value::INTEGER - FROM json_each_text(counters->'collections') - LOOP - IF collection_key = collection_id THEN - -- Increment the collection value by the input value - collection_value := GREATEST(0, collection_value + increment); - -- Update the collections JSONB with the new value - updated_collections := jsonb_set(updated_collections, ARRAY[collection_key], to_jsonb(collection_value)); - collection_exists := TRUE; - END IF; - END LOOP; + -- Be sure to have collections:{} and not collections:[] + IF jsonb_typeof(updated_collections) = 'array' THEN + updated_collections := '{}'; + END IF; - -- If the collection_id does not exist, add it with the input value - IF NOT collection_exists THEN - updated_collections := jsonb_set(updated_collections, ARRAY[collection_id], to_jsonb(increment)); - END IF; + -- Check if collection_id is NOT NULL and update the specific collection + IF collection_id IS NOT NULL THEN + -- Get the current value of the collection_id or default to 0 if it doesn't exist + collection_value := COALESCE((counters->'collections'->>collection_id)::INTEGER, 0); + + -- Increment the collection value + collection_value := GREATEST(0, collection_value + increment); + + -- Update the collections JSONB with the new value or insert a new one + updated_collections := jsonb_set(updated_collections, ARRAY[collection_id], to_jsonb(collection_value)); + END IF; -- Update the JSON object with the new total and collections From 4ce8862fa5a097c6f1fe13cc0f78fee358b257ea Mon Sep 17 00:00:00 2001 From: jjrom Date: Fri, 11 Oct 2024 13:44:19 +0200 Subject: [PATCH 3/4] Correctly compute counts for catalog --- .../core/dbfunctions/CatalogsFunctions.php | 40 ++++++++----------- .../core/dbfunctions/FeaturesFunctions.php | 4 +- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/app/resto/core/dbfunctions/CatalogsFunctions.php b/app/resto/core/dbfunctions/CatalogsFunctions.php index 0f7f8587..27563553 100755 --- a/app/resto/core/dbfunctions/CatalogsFunctions.php +++ b/app/resto/core/dbfunctions/CatalogsFunctions.php @@ -445,20 +445,18 @@ public function updateCatalog($catalog, $userid, $baseUrl) } /** - * Increment all catalogs relied to feature + * Increment catalog counters * * @param string $featureId - * @param string $collectionId * @param integer $increment */ - public function updateFeatureCatalogsCounters($featureId, $collectionId, $increment) + public function updateFeatureCatalogsCounters($featureId, $increment) { $query = join(' ', array( - 'WITH path_hierarchy AS (SELECT distinct featureid, collection, catalogid, subpath(path, 0, generate_series(1, nlevel(path))) AS p FROM ' . $this->dbDriver->targetSchema . '.catalog_feature', + 'WITH path_hierarchy AS (SELECT collection, catalogid FROM ' . $this->dbDriver->targetSchema . '.catalog_feature', 'WHERE featureid = \'' . pg_escape_string($this->dbDriver->getConnection(), $featureId) . '\')', - 'UPDATE ' . $this->dbDriver->targetSchema . '.catalog SET counters=public.increment_counters(counters,' . $increment . ',' . (isset($collectionId) ? '\'' . $collectionId . '\'': 'NULL') . ')', - // 'WHERE lower(id) IN (SELECT LOWER(REPLACE(REPLACE(path_hierarchy.p::text, \'_\', \'.\'), \'.\', \'/\')) FROM path_hierarchy)' + 'UPDATE ' . $this->dbDriver->targetSchema . '.catalog SET counters=public.increment_counters(counters,' . $increment . ', (SELECT path_hierarchy.collection FROM path_hierarchy LIMIT 1))', 'WHERE lower(id) IN (SELECT LOWER(path_hierarchy.catalogid) FROM path_hierarchy)' )); @@ -536,17 +534,8 @@ public function removeCatalog($catalogId, $inTransaction = true) */ private function removeCatalogFeatures($catalogId) { - - $query = join(' ', array( - 'WITH path_hierarchy AS (SELECT distinct featureid, collection, catalogid, subpath(path, 0, generate_series(1, nlevel(path))) AS p FROM ' . $this->dbDriver->targetSchema . '.catalog_feature', - 'WHERE path = \'' . pg_escape_string($this->dbDriver->getConnection(), RestoUtil::path2ltree($catalogId)) . '\')', - 'UPDATE ' . $this->dbDriver->targetSchema . '.catalog SET counters=public.increment_counters(counters, -13, (SELECT path_hierarchy.collection FROM path_hierarchy LIMIT 1))', - 'WHERE lower(id) IN (SELECT LOWER(path_hierarchy.catalogid) FROM path_hierarchy)' - )); - $this->dbDriver->query($query); - - $results = $this->dbDriver->fetch($this->dbDriver->pQuery('DELETE FROM ' . $this->dbDriver->targetSchema . '.catalog_feature WHERE path ~ $1 RETURNING featureid, collection' , array(RestoUtil::path2ltree($catalogId) . '.*'), 500, 'Cannot delete catalog_feature association for catalog ' . $catalogId)); - + $this->dbDriver->query('UPDATE ' . $this->dbDriver->targetSchema . '.catalog SET counters=\'{"total":0, "collections":{}}\' WHERE lower(id) = lower(\'' . pg_escape_string($this->dbDriver->getConnection(), $catalogId) . '\')'); + $this->dbDriver->fetch($this->dbDriver->pQuery('DELETE FROM ' . $this->dbDriver->targetSchema . '.catalog_feature WHERE path = $1' , array(RestoUtil::path2ltree($catalogId)), 500, 'Cannot delete catalog_feature association for catalog ' . $catalogId)); } /** @@ -699,7 +688,7 @@ private function insertIntoCatalogFeature($featureId, $path, $catalogId, $collec $collectionId ), 500, 'Cannot create association for ' . $featureId . ' in catalog ' . $catalogId); - $this->updateFeatureCatalogsCounters($featureId, $collectionId, 1); + $this->updateFeatureCatalogsCounters($featureId, 1); } @@ -808,12 +797,17 @@ private function onTheFlyUpdateCountersWithCollection($catalog, $baseUrl) $originalLinks = json_decode($catalog['links'], true); $collections = array(); - + $results = $this->dbDriver->pQuery('SELECT id, counters, links FROM ' . $this->dbDriver->targetSchema . '.catalog WHERE lower(id) = lower($1) OR lower(id) LIKE lower($2) ORDER BY id ASC', array( $catalog['id'], $catalog['id'] . '/%' )); while ($result = pg_fetch_assoc($results)) { + + if ($catalog['id'] !== $result['id']) { + $catalogCounters = json_decode($result['counters'], true); + $counters['total'] = $counters['total'] + $catalogCounters['total']; + } // Process collection if ( isset($result['links']) ) { @@ -835,14 +829,14 @@ private function onTheFlyUpdateCountersWithCollection($catalog, $baseUrl) 'collections/' . $collectionId )); while ($result = pg_fetch_assoc($results)) { - $collectionCounter = json_decode($result['counters'], true); - $counters['total'] = $counters['total'] + $collectionCounter['total']; - $counters['collections'][$collectionId] = $collectionCounter['total']; + $collectionCounters = json_decode($result['counters'], true); + $counters['total'] = $counters['total'] + $collectionCounters['total']; + $counters['collections'][$collectionId] = $collectionCounters['total']; for ($i = 0, $ii = count($originalLinks); $i < $ii; $i++) { if ($originalLinks[$i]['rel'] === 'child') { $exploded = explode('/', substr($originalLinks[$i]['href'], strlen($baseUrl . RestoRouter::ROUTE_TO_COLLECTIONS) + 1)); if (count($exploded) === 1 && $exploded[0] === $collectionId) { - $originalLinks[$i]['matched'] = $collectionCounter['total']; + $originalLinks[$i]['matched'] = $collectionCounters['total']; if ( isset($result['title']) ) { $originalLinks[$i]['title'] = $result['title']; } diff --git a/app/resto/core/dbfunctions/FeaturesFunctions.php b/app/resto/core/dbfunctions/FeaturesFunctions.php index d8f808a0..f89d7526 100755 --- a/app/resto/core/dbfunctions/FeaturesFunctions.php +++ b/app/resto/core/dbfunctions/FeaturesFunctions.php @@ -395,7 +395,7 @@ public function removeFeature($feature) /* * Update statistics counter for featureId - i.e. remove 1 per catalogs containing this feature */ - $catalogsUpdated = (new CatalogsFunctions($this->dbDriver))->updateFeatureCatalogsCounters($feature->id, $feature->collection->id, -1); + $catalogsUpdated = (new CatalogsFunctions($this->dbDriver))->updateFeatureCatalogsCounters($feature->id, -1); /* * Next remove @@ -500,7 +500,7 @@ public function updateFeature($feature, $collection, $newFeatureArray) * 2. Then delete resto.catalog_feature rows * 3. Then add resto.catalog_feature rows */ - (new CatalogsFunctions($this->dbDriver))->updateFeatureCatalogsCounters($feature->id, $collection->id, -1); + (new CatalogsFunctions($this->dbDriver))->updateFeatureCatalogsCounters($feature->id, -1); $this->dbDriver->pQuery( 'DELETE FROM ' . $this->dbDriver->targetSchema . '.catalog_feature WHERE featureid=$1', array( From 7779c4d1c595ba155a3f0d07f60233002b162858 Mon Sep 17 00:00:00 2001 From: jjrom Date: Fri, 11 Oct 2024 13:50:17 +0200 Subject: [PATCH 4/4] Correct search on '_' --- app/resto/core/api/STACAPI.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/resto/core/api/STACAPI.php b/app/resto/core/api/STACAPI.php index 5869e4b2..e7832e20 100644 --- a/app/resto/core/api/STACAPI.php +++ b/app/resto/core/api/STACAPI.php @@ -1321,7 +1321,7 @@ private function processPath($segments, $params = array()) } $searchParams = array( - 'q' => RestoUtil::path2ltree($catalogs[0]['id']) + 'q' => $catalogs[0]['id'] ); foreach (array_keys($params) as $key) {