bug(bq,sf,rs|clustering):ST_CLUSTERKMEANS remove duplicated coords (#491

)

clouds/bigquery/modules/doc/clustering/ST_CLUSTERKMEANS.md

@@ -11,6 +11,10 @@ Takes a set of points as input and partitions them into clusters using the k-mea
 * `geog`: `ARRAY<GEOGRAPHY>` points to be clustered.
 * `numberOfClusters`: `INT64`|`NULL` numberOfClusters that will be generated. If `NULL` the default value `Math.sqrt(<NUMBER OF POINTS>/2)` is used.
 
+````hint:info
+The resulting geometries are unique. So duplicated points will be removed from the input array
+````
+
 **Return type**
 
 `ARRAY<STRUCT<cluster INT64, geom GEOGRAPHY>>`

clouds/bigquery/modules/sql/clustering/ST_CLUSTERKMEANS.sql

@@ -1,6 +1,6 @@
-----------------------------
--- Copyright (C) 2021 CARTO
-----------------------------
+--------------------------------
+-- Copyright (C) 2021-2024 CARTO
+--------------------------------
 
 CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.__CLUSTERKMEANS`
 (geojson ARRAY<STRING>, numberOfClusters INT64)
@@ -15,8 +15,11 @@ AS """
     const options = {};
     if (numberOfClusters != null) {
         options.numberOfClusters = Number(numberOfClusters);
+    } else {
+        options.numberOfClusters = parseInt(Math.sqrt(geojson.length/2))
     }
     options.mutate = true;
+    geojson = Array.from(new Set(geojson));
     const featuresCollection = lib.clustering.featureCollection(geojson.map(x => lib.clustering.feature(JSON.parse(x))));
     lib.clustering.clustersKmeans(featuresCollection, options);
     const cluster = [];

clouds/redshift/libraries/python/lib/clustering/__init__.py

@@ -10,6 +10,13 @@ def load_geom(geom):
     geom = json.dumps(_geom)
     return loads(geom)
 
+def remove_duplicated_coords(arr):
+    import numpy as np
+    unique_rows = []
+    for row in arr:
+        if not any(np.array_equal(row, unique_row) for unique_row in unique_rows):
+            unique_rows.append(row)
+    return np.array(unique_rows)
 
 def clusterkmeanstable(geom, k):
     from .kmeans import KMeans
@@ -39,14 +46,14 @@ def clusterkmeans(geom, k):
     if geom.type != 'MultiPoint':
         raise Exception('Invalid operation: Input points parameter must be MultiPoint.')
     else:
-        coords = np.array(list(geojson.utils.coords(geom)))
+        coords = remove_duplicated_coords(np.array(list(geojson.utils.coords(geom))))
     cluster_idxs, centers, loss = KMeans()(coords, k)
     return geojson.dumps(
         [
             {
                 'cluster': cluster_idxs[idx],
-                'geom': {'coordinates': point, 'type': 'Point'},
+                'geom': {'coordinates': point.tolist(), 'type': 'Point'},
             }
-            for idx, point in enumerate(geom['coordinates'])
+            for idx, point in enumerate(coords)
         ]
     )

clouds/redshift/modules/doc/clustering/ST_CLUSTERKMEANS.md

@@ -11,6 +11,10 @@ Takes a set of points as input and partitions them into clusters using the k-mea
 * `geog`: `GEOMETRY` points to be clustered.
 * `numberOfClusters` (optional): `INT` number of clusters that will be generated. It defaults to the square root of half the number of points (`sqrt(<NUMBER OF POINTS>/2)`).
 
+````hint:info
+The resulting geometries are unique. So duplicated points will be removed from the input multipoint
+````
+
 **Return type**
 
 `SUPER`: containing objects with `cluster` as the cluster id and `geom` as the geometry in GeoJSON format.

clouds/redshift/modules/sql/clustering/ST_CLUSTERKMEANS.sql

@@ -1,6 +1,6 @@
-----------------------------
--- Copyright (C) 2021 CARTO
-----------------------------
+--------------------------------
+-- Copyright (C) 2021-2024 CARTO
+--------------------------------
 
 CREATE OR REPLACE FUNCTION @@RS_SCHEMA@@.__CLUSTERKMEANS
 (geom VARCHAR(MAX), numberofClusters INT)

clouds/redshift/modules/test/clustering/fixtures/st_clusterkmeans_out.txt

@@ -1,3 +1,3 @@
 [{"cluster":0,"geom":{"type":"Point","coordinates":[0.0,0.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[0.0,1.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[5.0,0.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[1.0,0.0]}}]
-[{"cluster":0,"geom":{"type":"Point","coordinates":[0.0,0.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[0.0,1.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[5.0,0.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[1.0,0.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[0.0,1.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[5.0,0.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[1.0,19.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[12.0,1.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[9.0,2.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[1.0,10.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[-3.0,1.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[5.0,5.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[8.0,6.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[10.0,10.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[-3.0,-5.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[6.0,5.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[-8.0,9.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[1.0,-10.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[2.0,-2.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[0.0,0.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[3.0,10.0]}}]
-[{"cluster":0,"geom":{"type":"Point","coordinates":[0.0,0.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[0.0,1.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[5.0,0.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[1.0,0.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[0.0,1.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[5.0,0.0]}},{"cluster":4,"geom":{"type":"Point","coordinates":[1.0,19.0]}},{"cluster":3,"geom":{"type":"Point","coordinates":[12.0,1.0]}},{"cluster":3,"geom":{"type":"Point","coordinates":[9.0,2.0]}},{"cluster":4,"geom":{"type":"Point","coordinates":[1.0,10.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[-3.0,1.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[5.0,5.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[8.0,6.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[10.0,10.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[-3.0,-5.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[6.0,5.0]}},{"cluster":4,"geom":{"type":"Point","coordinates":[-8.0,9.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[1.0,-10.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[2.0,-2.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[0.0,0.0]}},{"cluster":4,"geom":{"type":"Point","coordinates":[3.0,10.0]}}]
+[{"cluster":2,"geom":{"type":"Point","coordinates":[0.0,0.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[0.0,1.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[5.0,0.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[1.0,0.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[1.0,19.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[12.0,1.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[9.0,2.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[1.0,10.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[-3.0,1.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[5.0,5.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[8.0,6.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[10.0,10.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[-3.0,-5.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[6.0,5.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[-8.0,9.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[1.0,-10.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[2.0,-2.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[3.0,10.0]}}]
+[{"cluster":3,"geom":{"type":"Point","coordinates":[0.0,0.0]}},{"cluster":3,"geom":{"type":"Point","coordinates":[0.0,1.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[5.0,0.0]}},{"cluster":3,"geom":{"type":"Point","coordinates":[1.0,0.0]}},{"cluster":1,"geom":{"type":"Point","coordinates":[1.0,19.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[12.0,1.0]}},{"cluster":0,"geom":{"type":"Point","coordinates":[9.0,2.0]}},{"cluster":4,"geom":{"type":"Point","coordinates":[1.0,10.0]}},{"cluster":3,"geom":{"type":"Point","coordinates":[-3.0,1.0]}},{"cluster":4,"geom":{"type":"Point","coordinates":[5.0,5.0]}},{"cluster":4,"geom":{"type":"Point","coordinates":[8.0,6.0]}},{"cluster":4,"geom":{"type":"Point","coordinates":[10.0,10.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[-3.0,-5.0]}},{"cluster":4,"geom":{"type":"Point","coordinates":[6.0,5.0]}},{"cluster":3,"geom":{"type":"Point","coordinates":[-8.0,9.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[1.0,-10.0]}},{"cluster":2,"geom":{"type":"Point","coordinates":[2.0,-2.0]}},{"cluster":4,"geom":{"type":"Point","coordinates":[3.0,10.0]}}]

clouds/snowflake/modules/doc/clustering/ST_CLUSTERKMEANS.md

@@ -11,6 +11,10 @@ Takes a set of points as input and partitions them into clusters using the k-mea
 * `geojsons`: `ARRAY` points to be clustered.
 * `numberOfClusters` (optional): `INT` numberOfClusters that will be generated. By default `numberOfClusters` is `Math.sqrt(<NUMBER OF POINTS>/2)`.
 
+````hint:info
+The resulting geometries are unique. So duplicated points will be removed from the input array
+````
+
 **Return type**
 
 `ARRAY`: containing objects with `cluster`, as the cluster id, and `geom`, as the geometry geojson.

clouds/snowflake/modules/sql/clustering/ST_CLUSTERKMEANS.sql

@@ -1,6 +1,6 @@
-----------------------------
--- Copyright (C) 2021 CARTO
-----------------------------
+--------------------------------
+-- Copyright (C) 2021-2024 CARTO
+--------------------------------
 
 CREATE OR REPLACE FUNCTION @@SF_SCHEMA@@._CLUSTERKMEANS
 (geojsons ARRAY, numberOfClusters DOUBLE)
@@ -17,6 +17,7 @@ AS $$
     const options = {};
     options.numberOfClusters = Number(NUMBEROFCLUSTERS);
     options.mutate = true;
+    GEOJSONS = Array.from(new Set(GEOJSONS));
     const featuresCollection = clusteringLib.featureCollection(GEOJSONS.map(x => clusteringLib.feature(JSON.parse(x))));
     clusteringLib.clustersKmeans(featuresCollection, options);
     const cluster = [];