release: 2024-05-21

.github/workflows/bigquery-ded.yml

@@ -27,15 +27,11 @@ jobs:
       - name: Set BQ_PREFIX for releases
         if: startsWith(github.event.pull_request.head.ref, 'release/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#release/}
-          echo "BQ_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
+          echo "BQ_PREFIX=dedicated_release_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Set BQ_PREFIX for hotfixes
         if: startsWith(github.event.pull_request.head.ref, 'hotfix/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#hotfix/}
-          echo "BQ_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
+          echo "BQ_PREFIX=dedicated_hotfix_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Set BQ_PREFIX for regular deploys
         if: |
           !(startsWith(github.event.pull_request.head.ref, 'hotfix/')) &&

.github/workflows/postgres-ded.yml

@@ -29,15 +29,11 @@ jobs:
       - name: Set PG_PREFIX for releases
         if: startsWith(github.event.pull_request.head.ref, 'release/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#release/}
-          echo "PG_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
+          echo "PG_PREFIX=dedicated_release_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Set PG_PREFIX for hotfixes
         if: startsWith(github.event.pull_request.head.ref, 'hotfix/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#hotfix/}
-          echo "PG_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
+          echo "PG_PREFIX=dedicated_hotfix_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Set PG_PREFIX for regular deploys
         if: |
           !(startsWith(github.event.pull_request.head.ref, 'hotfix/')) &&

.github/workflows/redshift-ded.yml

@@ -18,6 +18,7 @@ jobs:
       (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'dedicated_redshift')) ||
       (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'dedicated_redshift'))
     runs-on: ubuntu-20.04
+    container: python:2.7.18-buster
     timeout-minutes: 20
     env:
       RS_HOST: ${{ secrets.RS_HOST_CD }}
@@ -30,19 +31,17 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
+      - name: Check diff
+        uses: technote-space/get-diff-action@v4
       - name: Set RS_PREFIX for releases
         if: startsWith(github.event.pull_request.head.ref, 'release/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#release/}
-          echo "RS_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
+          echo "RS_PREFIX=dedicated_release_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Set RS_PREFIX for hotfixes
         if: startsWith(github.event.pull_request.head.ref, 'hotfix/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#hotfix/}
-          echo "RS_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
-      - name: Set RS_PREFIX for regular deploys
+          echo "RS_PREFIX=dedicated_hotfix_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
+      - name: Set RS_PREFIX for the rest
         if: |
           !(startsWith(github.event.pull_request.head.ref, 'hotfix/')) &&
           !(startsWith(github.event.pull_request.head.ref, 'release/'))
@@ -62,6 +61,8 @@ jobs:
           curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
           unzip awscliv2.zip
           ./aws/install
+      - name: Setup virtualenv
+        run: pip install virtualenv==${{ env.VIRTUALENV_VERSION }}
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -75,7 +76,7 @@ jobs:
         if: github.event.action == 'synchronize' || github.event.action == 'labeled'
         run: |
           cd clouds/redshift
-          make deploy
+          make deploy dropfirst=1
       - name: Run remove
         id: remove
         if: github.event.action == 'unlabeled' || github.event.action == 'closed'

.github/workflows/snowflake-ded.yml

@@ -5,7 +5,7 @@ on:
     types: [closed, unlabeled, labeled, synchronize]
 
 env:
-  NODE_VERSION: 14
+  NODE_VERSION: 18.16
 
 jobs:
 
@@ -28,15 +28,11 @@ jobs:
       - name: Set SF_PREFIX for releases
         if: startsWith(github.event.pull_request.head.ref, 'release/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#release/}
-          echo "SF_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
+          echo "SF_PREFIX=dedicated_release_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Set SF_PREFIX for hotfixes
         if: startsWith(github.event.pull_request.head.ref, 'hotfix/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#hotfix/}
-          echo "SF_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
+          echo "SF_PREFIX=dedicated_hotfix_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Set SF_PREFIX for regular deploys
         if: |
           !(startsWith(github.event.pull_request.head.ref, 'hotfix/')) &&

.github/workflows/snowflake.yml

@@ -15,8 +15,8 @@ on:
   workflow_call:
 
 env:
-  NODE_VERSION: 14
-  PYTHON3_VERSION: 3.8.10
+  NODE_VERSION: 18.16
+  PYTHON3_VERSION: 3.8.18
   VIRTUALENV_VERSION: 20.21.1
   GCLOUD_VERSION: 290.0.1
 
@@ -88,6 +88,37 @@ jobs:
           cd clouds/snowflake
           make deploy diff="$GIT_DIFF" production=1
 
+  deploy-internal-app:
+    if: github.ref_name == 'main'
+    needs: test
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
+    env:
+      APP_PACKAGE_NAME: ${{ secrets.SF_NATIVE_APP_PACKAGE_NAME_CD }}
+      APP_NAME: ${{ secrets.SF_NATIVE_APP_NAME_CD }}
+      SF_ACCOUNT: ${{ secrets.SF_ACCOUNT_NATIVE_APP }}
+      SF_DATABASE: ${{ secrets.SF_DATABASE_NATIVE_APP }}
+      SF_USER: ${{ secrets.SF_USER_NATIVE_APP }}
+      SF_PASSWORD: ${{ secrets.SF_PASSWORD_NATIVE_APP }}
+      SF_ROLE: ${{ secrets.SF_ROLE_NATIVE_APP }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+      - name: Check diff
+        uses: technote-space/get-diff-action@v4
+      - name: Setup node
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+      - name: Deploy native app package
+        run: |
+          cd clouds/snowflake
+          make deploy-native-app-package production=1
+      - name: Deploy native app locally
+        run: |
+          cd clouds/snowflake
+          make deploy-native-app production=1
+
   deploy-share:
     if: github.ref_name == 'stable'
     needs: test
@@ -123,7 +154,6 @@ jobs:
     timeout-minutes: 10
     env:
       PACKAGE_BUCKET: gs://carto-analytics-toolbox-core/snowflake
-      SF_DATABASE: 'carto'
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2

.gitignore

@@ -22,6 +22,10 @@ package-lock.json
 MANIFEST
 *.ipynb
 .pgpass
+.~lock.*
 
 # IntelliJ
 .idea/
+
+# Vim
+*.swp

CHANGELOG.md

@@ -4,6 +4,52 @@ CARTO Analytics Toolbox Core.
 
 All notable commits to this project will be documented in this file.
 
+## 2024-05-21
+
+- refactor(sf|h3): avoid memory limit exceeded in H3_POLYFILL_TABLE (#501)
+- docs(bq,sf,rs|constructors): fix ST_MAKEELLIPSE angle parameter explanation (#505)
+- chore(sf): update create-package and licenses year (#507)
+- chore(pg): DB Connection Error in tests when % character in PG_PASSWORD (#509)
+
+## 2024-04-18
+
+- chore(sf|h3): reimplement basic h3 functions (#489)
+- docs(bq,sf,rs|processing): update voronoi doc (#492)
+- chore(sf|h3): reimplement polyfill h3 functions (#490)
+- fix(bq,sf,rs|clustering): improve how ST_CLUSTERKMEANS deals with duplicates (#491, #495)
+- chore(deps): bump sqlparse from 0.4.4 to 0.5.0 in /clouds/redshift/common (#494)
+- chore(deps): bump sqlparse from 0.4.4 to 0.5.0 in /clouds/postgres/common (#493)
+- chore(deps): fix CI crashing because native-apps timeout and sql-parse version (#496)
+
+## 2024-03-18
+
+- fix(sf): CI and CD not working because of snowflake driver breaking changes (#484)
+- fix(bq,sf,rs,pg): drop schemas when dedicateds gets released (#485)
+- fix(bq|random): ST_GENERATEPOINTS returning exact name of points (#486)
+- chore(rs,pg): remove installer tool (#483)
+- chore(deps): bump jinja2 from 3.1.2 to 3.1.3 in /clouds/databricks/common (#468)
+
+## 2024-02-15
+
+- chore(sf): add additional tables to native apps (#473)
+- docs(bs,sf,rs|transformations): fix ST_DESTINATION bearing parameter description (#475)
+- fix(sf|quadbin): QUADBIN_TOPARENT not working with views (#476)
+- docs(bq,sf,rs|constructors): fix angle parameter description in ST_MAKEELLIPSE (#477)
+- fix(sf|random): ST_GENERATEPOINTS was not accepting column names (#480)
+
+## 2024-01-17
+
+- chore(bq): increase tests timeout to 200000 (#455)
+- feat(bq,sf,rs,pg|quadbin): add function QUADBIN_DISTANCE (#457)
+- fix(bq|h3): fix broken reference in H3_POLYFILL_TABLE (#458, #460)
+- chore(bq,sf,rs,pg): fix naming dedicated deployments for releases (#462)
+- fix(sf|quadbin): QUADBIN_FROMLONGLAT not clamping latitudes and return some quadbin functions return NULL when NULL parameters (#456)
+- fix(rs|constructors,transformations): adjust SRID and use native ST_GEOMFROMGEOJSON to return geometries instead of VARCHAR (#463)
+- chore(pg): fix typo naming dedicated deployments for releases (#464)
+- chore(bq,sf,rs,pg): make remove drop functions instead of whole schema (#466)
+- fix(bq,sf,rs,pg|quadbin): improve precision of long lat conversion near the latitude limits (#461)
+- feat(bq,sf|transformations): add function ST_POINTONSURFACE (#469, #470)
+
 ## 2023-08-04
 
 - chore(bq|quadbin,h3): optimize quadbin/h3 polyfill performance (#421)

CONTRIBUTING.md

@@ -162,3 +162,68 @@ Images must be stored in `common/assets/`, and included with the following synta
 ```md
 ![My image](my_image.png)
 ```
+
+## Naming requirements
+
+The format of function and procedure names in the code must follow some conventions, so that the scripts that build the SQL code can determine the dependencies between them.
+
+### BigQuery
+
+Function/procedure invocations: the name must be quoted with backticks, including the project/dataset name. The opening bracket must follow the closing quote with no space between them:
+
+```sql
+SELECT `@@BQ_DATASET@@.A_FUNCTION`();
+CALL `@@BQ_DATASET@@.A_PROCEDURE`();
+```
+
+Function/procedure definitions: must be quoted as invocations, but parentheses should be on a separate line:
+
+```sql
+CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.A_FUNCTION`
+(
+    an_argument STRING
+)
+...
+```
+
+### Other clouds (Snowflake, Redshift, Postgres)
+
+Function/procedure invocations: The opening bracket must follow name with no space between them:
+
+
+```sql
+SELECT @@XX_SCHEMA@@.A_FUNCTION();
+CALL @@XX_SCHEMA@@.A_PROCEDURE();
+```
+
+Function/procedure definitions: must be quoted as invocations, but parentheses should be on a separate line:
+
+```sql
+CREATE OR REPLACE FUNCTION @@XX_SCHEMA@@.A_FUNCTION
+(
+  an_argument STRING
+)
+RETURNS STRING
+IMMUTABLE
+...
+```
+
+### Extra dependencies
+
+Sometimes the name of an invoked function or procedure is dynamically generated in the SQL code;
+in those cases it's necessary to add a coment in the file referencing all the functions/procedures that can potentially be called.
+
+In this comment, the names must appear as in actual invocations, following the rules of each cloud, i.e. including the argument parentheses, and
+in the case of BigQuery, the backtick quotes. For example:
+
+```sql
+/*
+Extra dependencies:
+`@@BQ_DATASET@@.SOME_FUNCTION`()
+*/
+```
+
+## Known Limitations
+### Snowflake
+
+Due to Snowflake Native Apps limitations at this moment TEMPORARY tables cannot be used within procedures. For the time being create normal tables and ensure that they are dropped.

LICENSE

@@ -1,6 +1,6 @@
 SPDX short identifier: BSD-3-Clause
 
-Copyright (c) 2021-2022, CARTO
+Copyright (c) 2021-2024, CARTO
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

README.md

@@ -62,6 +62,23 @@ Right now the only way to get access the Analytics toolbox is by installing it d
 | Postgres | [README.md](./clouds/postgres/README.md) |
 | Databricks | [README.md](./clouds/databricks/README.md) |
 
+### Useful make commands
+
+To run tests, switch to a specific cloud directory. For example, Showflake: `cd clouds/snowflake`.  
+
+```
+# All tests
+make test
+
+# Specific module(s)
+make test modules=h3
+make test modules=h3,transformations
+
+# Specific function(s)
+make test functions=H3_POLYFILL
+make test functions=H3_POLYFILL,ST_BUFFER
+```
+
 ## Contribute
 
 This project is public. We are more than happy of receiving feedback and contributions. Feel free to open a ticket with a bug, a doubt or a discussion, or open a pull request with a fix or a new feature.

clouds/bigquery/CHANGELOG.md

@@ -4,6 +4,29 @@ CARTO Analytics Toolbox Core for BigQuery.
 
 All notable commits to this project will be documented in this file.
 
+## [1.2.3] - 2024-05-21
+
+- docs(constructors): fix ST_MAKEELLIPSE angle parameter explanation (#505)
+
+## [1.2.2] - 2024-04-18
+
+- docs(processing): update voronoi doc (#492)
+- fix(clustering): improve how ST_CLUSTERKMEANS deals with duplicates (#491, #495)
+
+## [1.2.1] - 2024-03-18
+
+- fix(random): ST_GENERATEPOINTS returning exact name of points (#486)
+
+## [1.2.0] - 2024-01-17
+
+- chore: increase tests timeout to 200000 (#455)
+- feat(quadbin): add function QUADBIN_DISTANCE (#457)
+- fix(h3): fix broken reference in H3_POLYFILL_TABLE (#458, #460)
+- chore: fix naming dedicated deployments for releases (#462)
+- chore: make remove drop functions instead of whole schema (#466)
+- fix(quadbin): improve precision of long lat conversion near the latitude limits (#461)
+- feat(transformations): add function ST_POINTONSURFACE (#469)
+
 ## [1.1.0] - 2023-08-04
 
 - chore(quadbin,h3): optimize quadbin/h3 polyfill performance (#421)

clouds/bigquery/common/DROP_FUNCTIONS.sql

@@ -1,5 +1,5 @@
 ---------------------------------
--- Copyright (C) 2021-2023 CARTO
+-- Copyright (C) 2021-2024 CARTO
 ---------------------------------
 
 CREATE OR REPLACE PROCEDURE `@@BQ_DATASET@@.DROP_FUNCTIONS`()

clouds/bigquery/common/build_modules.js

@@ -99,7 +99,7 @@ functionsFilter.forEach(f => {
 if (!nodeps) {
     functions.forEach(mainFunction => {
         functions.forEach(depFunction => {
-            if (mainFunction.name != depFunction.name && depFunction.name !== 'SETUP') {
+            if (mainFunction.name != depFunction.name) {
                 const depFunctionMatches = [];
                 depFunctionMatches.push(...depFunction.content.replace(/(\r\n|\n|\r)/gm,' ').matchAll(new RegExp('(?<=(?<!TEMP )FUNCTION)(.*?)(?=AS |RETURNS)','g')));
                 depFunctionMatches.push(...depFunction.content.replace(/(\r\n|\n|\r)/gm,' ').matchAll(new RegExp('(?<=PROCEDURE)(.*?)(?=BEGIN)','g')));
@@ -109,7 +109,7 @@ if (!nodeps) {
                     qualifiedDepFunctName = qualifiedDepFunctName.split('.');
                     depFunctionNames.push(qualifiedDepFunctName[qualifiedDepFunctName.length - 1]);
                 })
-                if (depFunctionNames.some((depFunctionName) => mainFunction.content.includes(`DATASET@@.${depFunctionName}`))) {
+                if (depFunctionNames.some((depFunctionName) => mainFunction.content.includes(`DATASET@@.${depFunctionName}\`(`))) {
                     mainFunction.dependencies.push(depFunction.name);
                 }
             }