From d4ecc282adafe5d1e7651329724af6d35ed0ad20 Mon Sep 17 00:00:00 2001
From: Xing Guo <higuoxing@gmail.com>
Date: Thu, 25 May 2023 17:30:00 +0800
Subject: [PATCH 01/19] Fix compiler warnings. (#339)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Note: `TextDatumGetCString()` returns a null-terminated CString.

```
[ 12%] Building C object CMakeFiles/diskquota.dir/src/diskquota.c.o
In file included from /home/v/workspace/diskquota/src/diskquota.h:16,
                 from /home/v/workspace/diskquota/src/diskquota.c:18:
/home/v/workspace/diskquota/src/diskquota.c: In function ‘diskquota_status_schema_version’:
/home/v/.local/gpdb7/include/postgresql/server/c.h:957:25: warning: ‘strncpy’ specified bound 64 equals destinatio
n size [-Wstringop-truncation]
  957 |                         strncpy(_dst, (src), _len); \
      |                         ^~~~~~~~~~~~~~~~~~~~~~~~~~
/home/v/workspace/diskquota/src/diskquota.c:1548:9: note: in expansion of macro ‘StrNCpy’
 1548 |         StrNCpy(version, vv, sizeof(version));
      |         ^~~~~~~
```

Co-authored-by: Hao Zhang <zhrt1446384557@gmail.com>
---
 src/diskquota.c | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/src/diskquota.c b/src/diskquota.c
index d5630700..04f27f5d 100644
--- a/src/diskquota.c
+++ b/src/diskquota.c
@@ -1514,9 +1514,6 @@ diskquota_status_binary_version()
 static const char *
 diskquota_status_schema_version()
 {
-	static char version[64] = {0};
-	memset(version, 0, sizeof(version));
-
 	int ret = SPI_connect();
 	Assert(ret = SPI_OK_CONNECT);
 
@@ -1526,30 +1523,31 @@ diskquota_status_schema_version()
 	{
 		ereport(WARNING,
 		        (errmsg("[diskquota] when reading installed version lines %ld code = %d", SPI_processed, ret)));
-		goto out;
+		goto fail;
 	}
 
 	if (SPI_processed == 0)
 	{
-		goto out;
+		goto fail;
 	}
 
-	bool  is_null = false;
-	Datum v       = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &is_null);
+	bool  is_null       = false;
+	Datum version_datum = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &is_null);
 	Assert(is_null == false);
 
-	char *vv = TextDatumGetCString(v);
-	if (vv == NULL)
+	char *version = TextDatumGetCString(version_datum);
+	if (version == NULL || *version == '\0')
 	{
 		ereport(WARNING, (errmsg("[diskquota] 'extversion' is empty in pg_class.pg_extension. may catalog corrupted")));
-		goto out;
+		goto fail;
 	}
 
-	StrNCpy(version, vv, sizeof(version));
-
-out:
 	SPI_finish();
 	return version;
+
+fail:
+	SPI_finish();
+	return "";
 }
 
 PG_FUNCTION_INFO_V1(diskquota_status);

From 1111f71109957d493a903c8ed9b7b7071a6bf6b2 Mon Sep 17 00:00:00 2001
From: Zhang Hao <hzhang2@vmware.com>
Date: Mon, 26 Jun 2023 13:31:06 +0800
Subject: [PATCH 02/19] Fix bug: diskquota.status() (#344)

Fix bug produced by #339

Co-authored-by: Xing Guo admin@higuoxing.com
---
 src/diskquota.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/diskquota.c b/src/diskquota.c
index 04f27f5d..0d15d9fc 100644
--- a/src/diskquota.c
+++ b/src/diskquota.c
@@ -1514,7 +1514,8 @@ diskquota_status_binary_version()
 static const char *
 diskquota_status_schema_version()
 {
-	int ret = SPI_connect();
+	static char ret_version[64];
+	int         ret = SPI_connect();
 	Assert(ret = SPI_OK_CONNECT);
 
 	ret = SPI_execute("select extversion from pg_extension where extname = 'diskquota'", true, 0);
@@ -1542,8 +1543,10 @@ diskquota_status_schema_version()
 		goto fail;
 	}
 
+	StrNCpy(ret_version, version, sizeof(ret_version) - 1);
+
 	SPI_finish();
-	return version;
+	return ret_version;
 
 fail:
 	SPI_finish();

From 58bb9e25f40695d4faf9e8763db3b20f9e823251 Mon Sep 17 00:00:00 2001
From: Zhang Hao <hzhang2@vmware.com>
Date: Mon, 26 Jun 2023 14:33:38 +0800
Subject: [PATCH 03/19] fix pipeline (#345)

fix pipeline resource
---
 concourse/pipeline/res_def.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/concourse/pipeline/res_def.yml b/concourse/pipeline/res_def.yml
index 4e6578d8..9920270e 100644
--- a/concourse/pipeline/res_def.yml
+++ b/concourse/pipeline/res_def.yml
@@ -172,25 +172,25 @@ resources:
   source:
     bucket: pivotal-gpdb-concourse-resources-prod
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-centos6.tar.gz
+    regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev\.\d+)-.*-centos6.tar.gz
 - name: bin_gpdb6_centos7
   type: gcs
   source:
     bucket: pivotal-gpdb-concourse-resources-prod
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-centos7.tar.gz
+    regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev\.\d+)-.*-centos7.tar.gz
 - name: bin_gpdb6_rhel8
   type: gcs
   source:
     bucket: pivotal-gpdb-concourse-resources-prod
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-rhel8.tar.gz
+    regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev\.\d+)-.*-rhel8.tar.gz
 - name: bin_gpdb6_ubuntu18
   type: gcs
   source:
     bucket: pivotal-gpdb-concourse-resources-prod
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-ubuntu18.04.tar.gz
+    regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev\.\d+)-.*-ubuntu18.04.tar.gz
 - name: bin_gpdb7_el8
   type: gcs
   source:

From c2686c900367ff84af84cc741b31107812976ee5 Mon Sep 17 00:00:00 2001
From: Zhang Hao <hzhang2@vmware.com>
Date: Tue, 27 Jun 2023 17:46:14 +0800
Subject: [PATCH 04/19] Reduce the number of the log in bgworker. (#346)

---
 src/diskquota.c | 17 +++++++++++------
 src/diskquota.h |  2 ++
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/diskquota.c b/src/diskquota.c
index 0d15d9fc..5099b713 100644
--- a/src/diskquota.c
+++ b/src/diskquota.c
@@ -338,7 +338,13 @@ disk_quota_worker_main(Datum main_arg)
 	pqsignal(SIGUSR1, disk_quota_sigusr1);
 
 	if (!MyWorkerInfo->dbEntry->inited)
+	{
+		MyWorkerInfo->dbEntry->last_log_time = GetCurrentTimestamp();
 		ereport(LOG, (errmsg("[diskquota] start disk quota worker process to monitor database:%s", dbname)));
+	}
+	/* To avoid last_log_time from being uninitialized. */
+	if (MyWorkerInfo->dbEntry->last_log_time > GetCurrentTimestamp())
+		MyWorkerInfo->dbEntry->last_log_time = GetCurrentTimestamp();
 	/*
 	 * The shmem exit hook is registered after registering disk_quota_sigterm.
 	 * So if the SIGTERM arrives before this statement, the shmem exit hook
@@ -482,11 +488,10 @@ disk_quota_worker_main(Datum main_arg)
 
 	if (!MyWorkerInfo->dbEntry->inited) update_monitordb_status(MyWorkerInfo->dbEntry->dbid, DB_RUNNING);
 
-	bool        is_gang_destroyed   = false;
-	TimestampTz log_start_timestamp = GetCurrentTimestamp();
-	TimestampTz log_end_timestamp;
+	bool        is_gang_destroyed    = false;
 	TimestampTz loop_start_timestamp = 0;
 	TimestampTz loop_end_timestamp;
+	TimestampTz log_time;
 	long        sleep_time = diskquota_naptime * 1000;
 	long        secs;
 	int         usecs;
@@ -502,11 +507,11 @@ disk_quota_worker_main(Datum main_arg)
 		 * every BGWORKER_LOG_TIME to ensure that we can find the database name
 		 * by the bgworker's pid in the log file.
 		 */
-		log_end_timestamp = GetCurrentTimestamp();
-		if (TimestampDifferenceExceeds(log_start_timestamp, log_end_timestamp, BGWORKER_LOG_TIME))
+		log_time = GetCurrentTimestamp();
+		if (TimestampDifferenceExceeds(log_time, MyWorkerInfo->dbEntry->last_log_time, BGWORKER_LOG_TIME))
 		{
 			ereport(LOG, (errmsg("[diskquota] disk quota worker process is monitoring database:%s", dbname)));
-			log_start_timestamp = log_end_timestamp;
+			MyWorkerInfo->dbEntry->last_log_time = log_time;
 		}
 
 		/*
diff --git a/src/diskquota.h b/src/diskquota.h
index f044773b..7baeaf1b 100644
--- a/src/diskquota.h
+++ b/src/diskquota.h
@@ -217,6 +217,8 @@ struct DiskquotaDBEntry
 
 	bool inited; // this entry is inited, will set to true after the worker finish the frist run.
 	bool in_use; // this slot is in using. AKA dbid != 0
+
+	TimestampTz last_log_time; // the last time log current database info.
 };
 
 typedef enum MonitorDBStatus

From 0a837c89e131c88a2b05bc626ac35f22f358e577 Mon Sep 17 00:00:00 2001
From: Zhang Hao <hzhang2@vmware.com>
Date: Tue, 27 Jun 2023 17:54:08 +0800
Subject: [PATCH 05/19] Fix bug: lose monitored_dbid_cache after switching
 mirror (#342)

# Problem
Recording active tables and permission checking on segments rely on `monitored_dbid_cache`. But after mirror switching, the data in shared memory is lost, and the above behaviors will be blocked.

# Solution
Segments update their `monitored_dbid_cache` after receiving pull_active_tables_oid requests every naptime.
---
 src/diskquota.h                               |  1 +
 src/gp_activetable.c                          |  2 +
 src/monitored_db.c                            | 29 +++++++++++
 .../regress/expected/test_primary_failure.out | 51 +++++++++++++++----
 tests/regress/sql/test_primary_failure.in.sql | 27 +++++++---
 5 files changed, 95 insertions(+), 15 deletions(-)

diff --git a/src/diskquota.h b/src/diskquota.h
index 7baeaf1b..8e46d317 100644
--- a/src/diskquota.h
+++ b/src/diskquota.h
@@ -291,4 +291,5 @@ extern HTAB        *diskquota_hash_create(const char *tabname, long nelem, HASHC
                                           DiskquotaHashFunction hashFunction);
 extern HTAB *DiskquotaShmemInitHash(const char *name, long init_size, long max_size, HASHCTL *infoP, int hash_flags,
                                     DiskquotaHashFunction hash_function);
+extern void  refresh_monitored_dbid_cache(void);
 #endif
diff --git a/src/gp_activetable.c b/src/gp_activetable.c
index cf3178b3..a0d1c524 100644
--- a/src/gp_activetable.c
+++ b/src/gp_activetable.c
@@ -749,6 +749,8 @@ get_active_tables_oid(void)
 
 	Oid relOid;
 
+	refresh_monitored_dbid_cache();
+
 	memset(&ctl, 0, sizeof(ctl));
 	ctl.keysize                 = sizeof(DiskQuotaActiveTableFileEntry);
 	ctl.entrysize               = sizeof(DiskQuotaActiveTableFileEntry);
diff --git a/src/monitored_db.c b/src/monitored_db.c
index f23cb9a9..c2b29a1b 100644
--- a/src/monitored_db.c
+++ b/src/monitored_db.c
@@ -336,3 +336,32 @@ dump_monitored_dbid_cache(long *nitems)
 	Assert(count == 0);
 	return entries;
 }
+
+/*
+ * After primary failure and mirror switching, the monitored_dbid_cache
+ * is lost on segments. We should refresh the monitored_dbid_cache during
+ * every diskquota refresh procedure.
+ */
+void
+refresh_monitored_dbid_cache(void)
+{
+	bool found;
+	Oid  dbid = MyDatabaseId;
+	LWLockAcquire(diskquota_locks.monitored_dbid_cache_lock, LW_EXCLUSIVE);
+	MonitorDBEntry entry = hash_search(monitored_dbid_cache, &dbid, HASH_ENTER_NULL, &found);
+	if (entry == NULL)
+	{
+		ereport(WARNING, (errmsg("can't alloc memory on dbid cache, there are too many databases to monitor")));
+	}
+	else if (!found)
+	{
+		entry->paused = false;
+		pg_atomic_init_u32(&(entry->epoch), 0);
+		pg_atomic_init_u32(&(entry->status), DB_RUNNING);
+		ereport(LOG, (errmsg("the entry in monitored_dbid_cache is lost due to mirror switching and is added back now, "
+		                     "dbid: %d",
+		                     dbid)));
+	}
+
+	LWLockRelease(diskquota_locks.monitored_dbid_cache_lock);
+}
diff --git a/tests/regress/expected/test_primary_failure.out b/tests/regress/expected/test_primary_failure.out
index 5f5c18a2..48160e24 100644
--- a/tests/regress/expected/test_primary_failure.out
+++ b/tests/regress/expected/test_primary_failure.out
@@ -36,19 +36,27 @@ returns text as $$
     else:
         return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True, encoding='utf8').replace('.', '')
 $$ language plpython2u;
-CREATE TABLE a(i int) DISTRIBUTED BY (i);
-NOTICE:  Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'i' as the Greenplum Database data distribution key for this table.
-HINT:  The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
-INSERT INTO a SELECT generate_series(1,100);
-INSERT INTO a SELECT generate_series(1,100000);
+CREATE TABLE a(i int, j int) DISTRIBUTED BY (i);
+-- the entries will be inserted into seg0
+INSERT INTO a SELECT 2, generate_series(1,100);
+INSERT INTO a SELECT 2, generate_series(1,100000);
 SELECT diskquota.wait_for_worker_new_epoch();
  wait_for_worker_new_epoch 
 ---------------------------
  t
 (1 row)
 
+SELECT tableid::regclass, size, segid FROM diskquota.table_size WHERE tableid = 'a'::regclass ORDER BY segid;
+ tableid |  size   | segid 
+---------+---------+-------
+ a       | 3735552 |    -1
+ a       | 3735552 |     0
+ a       |       0 |     1
+ a       |       0 |     2
+(4 rows)
+
 -- expect insert fail
-INSERT INTO a SELECT generate_series(1,100);
+INSERT INTO a SELECT 2, generate_series(1,100);
 ERROR:  schema's disk space quota exceeded with name: ftsr
 -- now one of primary is down
 select pg_ctl((select datadir from gp_segment_configuration c where c.role='p' and c.content=0), 'stop');
@@ -74,7 +82,7 @@ select content, preferred_role, role, status, mode from gp_segment_configuration
 (2 rows)
 
 -- expect insert fail
-INSERT INTO a SELECT generate_series(1,100);
+INSERT INTO a SELECT 2, generate_series(1,100);
 ERROR:  schema's disk space quota exceeded with name: ftsr
 -- increase quota
 SELECT diskquota.set_schema_quota('ftsr', '200 MB');
@@ -83,6 +91,31 @@ SELECT diskquota.set_schema_quota('ftsr', '200 MB');
  
 (1 row)
 
+SELECT diskquota.wait_for_worker_new_epoch();
+ wait_for_worker_new_epoch 
+---------------------------
+ t
+(1 row)
+
+-- expect insert success
+INSERT INTO a SELECT 2, generate_series(1,10000);
+SELECT diskquota.wait_for_worker_new_epoch();
+ wait_for_worker_new_epoch 
+---------------------------
+ t
+(1 row)
+
+-- check whether monitored_dbid_cache is refreshed in mirror
+-- diskquota.table_size should be updated
+SELECT tableid::regclass, size, segid FROM diskquota.table_size WHERE tableid = 'a'::regclass ORDER BY segid;
+ tableid |  size   | segid 
+---------+---------+-------
+ a       | 4096000 |    -1
+ a       | 4096000 |     0
+ a       |       0 |     1
+ a       |       0 |     2
+(4 rows)
+
 -- pull up failed primary
 -- start_ignore
 select pg_recoverseg((select datadir from gp_segment_configuration c where c.role='p' and c.content=-1), 'a');
@@ -256,10 +289,10 @@ SELECT diskquota.wait_for_worker_new_epoch();
 SELECT quota_in_mb, nspsize_in_bytes from diskquota.show_fast_schema_quota_view where schema_name='ftsr';
  quota_in_mb | nspsize_in_bytes 
 -------------+------------------
-         200 |          3932160
+         200 |          4096000
 (1 row)
 
-INSERT INTO a SELECT generate_series(1,100);
+INSERT INTO a SELECT 2, generate_series(1,100);
 DROP TABLE a;
 DROP SCHEMA ftsr CASCADE;
 NOTICE:  drop cascades to 2 other objects
diff --git a/tests/regress/sql/test_primary_failure.in.sql b/tests/regress/sql/test_primary_failure.in.sql
index cbac6e4c..2dd2689b 100644
--- a/tests/regress/sql/test_primary_failure.in.sql
+++ b/tests/regress/sql/test_primary_failure.in.sql
@@ -33,12 +33,16 @@ returns text as $$
         return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True, encoding='utf8').replace('.', '')
 $$ language @PLPYTHON_LANG_STR@;
 
-CREATE TABLE a(i int) DISTRIBUTED BY (i);
-INSERT INTO a SELECT generate_series(1,100);
-INSERT INTO a SELECT generate_series(1,100000);
+CREATE TABLE a(i int, j int) DISTRIBUTED BY (i);
+-- the entries will be inserted into seg0
+INSERT INTO a SELECT 2, generate_series(1,100);
+INSERT INTO a SELECT 2, generate_series(1,100000);
 SELECT diskquota.wait_for_worker_new_epoch();
+
+SELECT tableid::regclass, size, segid FROM diskquota.table_size WHERE tableid = 'a'::regclass ORDER BY segid;
+
 -- expect insert fail
-INSERT INTO a SELECT generate_series(1,100);
+INSERT INTO a SELECT 2, generate_series(1,100);
 
 -- now one of primary is down
 select pg_ctl((select datadir from gp_segment_configuration c where c.role='p' and c.content=0), 'stop');
@@ -50,11 +54,22 @@ select gp_request_fts_probe_scan();
 select content, preferred_role, role, status, mode from gp_segment_configuration where content = 0;
 
 -- expect insert fail
-INSERT INTO a SELECT generate_series(1,100);
+INSERT INTO a SELECT 2, generate_series(1,100);
 
 -- increase quota
 SELECT diskquota.set_schema_quota('ftsr', '200 MB');
 
+SELECT diskquota.wait_for_worker_new_epoch();
+
+-- expect insert success
+INSERT INTO a SELECT 2, generate_series(1,10000);
+
+SELECT diskquota.wait_for_worker_new_epoch();
+
+-- check whether monitored_dbid_cache is refreshed in mirror
+-- diskquota.table_size should be updated
+SELECT tableid::regclass, size, segid FROM diskquota.table_size WHERE tableid = 'a'::regclass ORDER BY segid;
+
 -- pull up failed primary
 -- start_ignore
 select pg_recoverseg((select datadir from gp_segment_configuration c where c.role='p' and c.content=-1), 'a');
@@ -67,7 +82,7 @@ select content, preferred_role, role, status, mode from gp_segment_configuration
 
 SELECT diskquota.wait_for_worker_new_epoch();
 SELECT quota_in_mb, nspsize_in_bytes from diskquota.show_fast_schema_quota_view where schema_name='ftsr';
-INSERT INTO a SELECT generate_series(1,100);
+INSERT INTO a SELECT 2, generate_series(1,100);
 
 DROP TABLE a;
 DROP SCHEMA ftsr CASCADE;

From 05da9d4dbb78373ab2c38ad576b22f637cc159ec Mon Sep 17 00:00:00 2001
From: Zhang Hao <hzhang2@vmware.com>
Date: Tue, 27 Jun 2023 17:55:14 +0800
Subject: [PATCH 06/19] Enable continuous upgrade. (#340)

To achieve upgrading directly from 2.0 to 2.2, we should do the following things:
- cherry-pick this PR in diskquota-2.0, diskquota-2.1 and diskquota-2.2.
- set the shared_preload_libraries as `diskquota-2.2`: `gpconfig -c shared_preload_libraries -v 'diskquota-2.2'`
- restart cluster: `gpstop -ar`
- execute the following SQLs:
```
ALTER extension diskquota update to '2.2';
```
---
 src/diskquota.c | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/src/diskquota.c b/src/diskquota.c
index 5099b713..3c4fa634 100644
--- a/src/diskquota.c
+++ b/src/diskquota.c
@@ -42,6 +42,7 @@
 #include "utils/snapmgr.h"
 #include "utils/syscache.h"
 #include "utils/timestamp.h"
+#include "tcop/pquery.h"
 
 PG_MODULE_MAGIC;
 
@@ -175,6 +176,28 @@ _PG_init(void)
 	/* diskquota.so must be in shared_preload_libraries to init SHM. */
 	if (!process_shared_preload_libraries_in_progress)
 	{
+		/*
+		 * To support the continuous upgrade/downgrade, we should skip the library
+		 * check in _PG_init() during upgrade/downgrade. If the POSTGRES backend
+		 * process is in normal mode and meets one of the following conditions, we
+		 * skip the library check:
+		 * - The backend is not a QD. We only need to check the library on QD.
+		 * - The current command is `ALTER EXTENSION`.
+		 */
+		if (IsNormalProcessingMode())
+		{
+			if (Gp_role != GP_ROLE_DISPATCH)
+			{
+				ereport(WARNING, (errmsg("[diskquota] booting " DISKQUOTA_VERSION ", but " DISKQUOTA_BINARY_NAME
+				                         " not in shared_preload_libraries.")));
+				return;
+			}
+			if (ActivePortal && ActivePortal->sourceTag == T_AlterExtensionStmt)
+			{
+				ereport(LOG, (errmsg("[diskquota] altering diskquota version to " DISKQUOTA_VERSION ".")));
+				return;
+			}
+		}
 		ereport(ERROR, (errmsg("[diskquota] booting " DISKQUOTA_VERSION ", but " DISKQUOTA_BINARY_NAME
 		                       " not in shared_preload_libraries. abort.")));
 	}

From c81e69600fa60bad2423e219e0ce7179f6f6edaa Mon Sep 17 00:00:00 2001
From: Zhang Hao <hzhang2@vmware.com>
Date: Wed, 28 Jun 2023 11:35:21 +0800
Subject: [PATCH 07/19] Fix pipeline. (#349)

Get the latest package version for release resource.
---
 concourse/pipeline/res_def.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/concourse/pipeline/res_def.yml b/concourse/pipeline/res_def.yml
index 9920270e..f53bb0a3 100644
--- a/concourse/pipeline/res_def.yml
+++ b/concourse/pipeline/res_def.yml
@@ -196,7 +196,7 @@ resources:
   source:
     bucket: pivotal-gpdb-concourse-resources-prod
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb7/greenplum-db-server-7\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-el8.tar.gz
+    regexp: server/release-candidates/gpdb7/greenplum-db-server-(7\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev\.\d+)-.*-el8.tar.gz
 
 # Diskquota releases
 - name: bin_diskquota_gpdb6_rhel6

From e3622e186c360dcd2b9f9e6ef4a399f3a54f8b7a Mon Sep 17 00:00:00 2001
From: Chen Mulong <chenmulong@gmail.com>
Date: Wed, 28 Jun 2023 14:39:58 +0800
Subject: [PATCH 08/19] Fix upgrade version check (#347)

- Because of 97f1f9b46b, the sql file directory has been changed. The
  sql version check wouldn't work since it cannot find the sql files
  anymore. Change it to the correct ddl directory.
- 'exec_program' is deprecated, use 'execute_process' instead.
- 'git tag | sort' returns the latest version among all branches, but
  not the closest tag to the current commit. Use 'git describe --tags'
  instead. So the upgrade version check will work for the 2.1.x patch
  release.
---
 upgrade_test/CMakeLists.txt | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/upgrade_test/CMakeLists.txt b/upgrade_test/CMakeLists.txt
index 1e3a2562..f151bd66 100644
--- a/upgrade_test/CMakeLists.txt
+++ b/upgrade_test/CMakeLists.txt
@@ -30,21 +30,24 @@ regresstarget_add(
   REGRESS_OPTS
   --dbname=contrib_regression)
 
-exec_program(
-  git ${CMAKE_SOURCE_DIR} ARGS
-  tag | sort --version-sort -r | head -n 1
+execute_process(
+  COMMAND git describe --tags --abbrev=0
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
   OUTPUT_VARIABLE latest_tag
+  OUTPUT_STRIP_TRAILING_WHITESPACE
 )
 
 # check whether DDL file (*.sql) is modified
-file(GLOB ddl_files ${CMAKE_SOURCE_DIR}/*.sql)
+file(GLOB ddl_files ${DISKQUOTA_DDL_DIR}/*.sql)
 foreach(ddl IN LISTS ddl_files)
   cmake_path(GET ddl FILENAME ddl)
-  exec_program(
-    git ${CMAKE_SOURCE_DIR} ARGS
-    diff ${latest_tag} --exit-code ${ddl}
-    OUTPUT_VARIABLE NULL
-    RETURN_VALUE "${ddl}_modified")
+  execute_process(
+    COMMAND
+    git diff ${latest_tag} --exit-code ${ddl}
+    OUTPUT_QUIET
+    WORKING_DIRECTORY ${DISKQUOTA_DDL_DIR}
+    RESULT_VARIABLE "${ddl}_modified"
+  )
 
   if("${${ddl}_modified}")
     message(

From 974876dc6e867391d6316aa4981f3e18cda4fca9 Mon Sep 17 00:00:00 2001
From: Zhang Hao <hzhang2@vmware.com>
Date: Wed, 28 Jun 2023 15:12:10 +0800
Subject: [PATCH 09/19] Remove gp7 from pipeline (#350)

After creating extension, the `diskquota.state` is always `unready` due to the change https://github.com/greenplum-db/gpdb/pull/15239. It makes the test timeout. We disable the job of gp7 in release/pr/merge pipeline until the problem is fixed.
---
 concourse/pipeline/commit.yml  | 2 +-
 concourse/pipeline/pr.yml      | 2 +-
 concourse/pipeline/release.yml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/concourse/pipeline/commit.yml b/concourse/pipeline/commit.yml
index 823dfcc2..1891caeb 100644
--- a/concourse/pipeline/commit.yml
+++ b/concourse/pipeline/commit.yml
@@ -19,7 +19,7 @@
 #@   centos7_gpdb6_conf(),
 #@   rhel8_gpdb6_conf(),
 #@   ubuntu18_gpdb6_conf(),
-#@   rhel8_gpdb7_conf(),
+#! #@   rhel8_gpdb7_conf(),
 #@ ]
 jobs:
 #@ param = {
diff --git a/concourse/pipeline/pr.yml b/concourse/pipeline/pr.yml
index 4a715120..3578131a 100644
--- a/concourse/pipeline/pr.yml
+++ b/concourse/pipeline/pr.yml
@@ -22,7 +22,7 @@
 #@   centos7_gpdb6_conf(),
 #@   rhel8_gpdb6_conf(),
 #@   ubuntu18_gpdb6_conf(),
-#@   rhel8_gpdb7_conf(),
+#! #@   rhel8_gpdb7_conf(),
 #@ ]
 jobs:
 #@ param = {
diff --git a/concourse/pipeline/release.yml b/concourse/pipeline/release.yml
index 023e86bd..6b12b935 100644
--- a/concourse/pipeline/release.yml
+++ b/concourse/pipeline/release.yml
@@ -20,7 +20,7 @@
 #@   centos7_gpdb6_conf(release_build=True),
 #@   rhel8_gpdb6_conf(release_build=True),
 #@   ubuntu18_gpdb6_conf(release_build=True),
-#@   rhel8_gpdb7_conf(release_build=True)
+#! #@   rhel8_gpdb7_conf(release_build=True)
 #@ ]
 jobs:
 #@ param = {

From 6001a059054bc0ab8dd91a1a08d2e040623b7a62 Mon Sep 17 00:00:00 2001
From: Chen Mulong <chenmulong@gmail.com>
Date: Wed, 28 Jun 2023 17:12:15 +0800
Subject: [PATCH 10/19] Add alter extension upgrade test (#348)

- By #340, diskquota should be able to upgraded directly from any previous
  version. A script is added to test this.
- Modify the cmakefile so before installing/packaing, only previous so
  files will copied. This would help us to make patch release for
  2.0/2.1.

---------

Co-authored-by: Zhang Hao <hzhang2@vmware.com>
---
 CMakeLists.txt                      | 15 ++++++-
 concourse/scripts/test_diskquota.sh |  2 +
 upgrade_test/alter_test.sh          | 61 +++++++++++++++++++++++++++++
 3 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100755 upgrade_test/alter_test.sh

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 38c64a98..d12fa448 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -91,8 +91,21 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Distro.cmake)
 if(DEFINED DISKQUOTA_LAST_RELEASE_PATH)
   message(STATUS "Copy pervious installer from ${DISKQUOTA_LAST_RELEASE_PATH}")
   file(ARCHIVE_EXTRACT INPUT ${DISKQUOTA_LAST_RELEASE_PATH} PATTERNS "*.so")
-  file(GLOB DISKQUOTA_PREVIOUS_LIBRARY
+  file(GLOB so_files_list
        "${CMAKE_BINARY_DIR}/lib/postgresql/*.so")
+
+  foreach(so_path IN LISTS so_files_list)
+    get_filename_component(so_name ${so_path} NAME_WLE)
+    # Replace 'diskquota-x.y' with 'x.y'. 'diskquota' won't be replaced, which belongs to 1.x release.
+    string(REPLACE "diskquota-" "" so_ver ${so_name})
+    # Install the previous so files. Those so files have versions less than current version.
+    # diskqutoa.so doesn't have version string in the file name. It belongs to 1.x release.
+    if((${so_ver} STREQUAL "diskquota") OR
+      ${so_ver} VERSION_LESS ${DISKQUOTA_MAJOR_VERSION}.${DISKQUOTA_MINOR_VERSION})
+      list(APPEND DISKQUOTA_PREVIOUS_LIBRARY ${so_path})
+    endif()
+  endforeach()
+
   install(PROGRAMS ${DISKQUOTA_PREVIOUS_LIBRARY} DESTINATION "lib/postgresql/")
 
   get_filename_component(
diff --git a/concourse/scripts/test_diskquota.sh b/concourse/scripts/test_diskquota.sh
index 85b2bce1..6abe35d0 100755
--- a/concourse/scripts/test_diskquota.sh
+++ b/concourse/scripts/test_diskquota.sh
@@ -41,6 +41,8 @@ function _main() {
     # Run upgrade test (with standby master)
     time cmake --build . --target upgradecheck
     popd
+
+    time /home/gpadmin/diskquota_src/upgrade_test/alter_test.sh
 }
 
 _main
diff --git a/upgrade_test/alter_test.sh b/upgrade_test/alter_test.sh
new file mode 100755
index 00000000..11c7afd7
--- /dev/null
+++ b/upgrade_test/alter_test.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+# Test if all the previous diskquota minor versions can be directly upgraded
+# to the current version.
+
+set -ex
+
+SCRIPT_PATH="${BASH_SOURCE[0]}"
+SRC_DIR="$(cd "$(dirname "${SCRIPT_PATH}")"/.. >/dev/null 2>&1 && pwd)"
+
+# Versions like major.minor
+CUR_VERSION=$(cut --delimiter="." --fields=1-2 "${SRC_DIR}/VERSION")
+ALL_VERSIONS=$(cd "${SRC_DIR}" && git tag | cut --delimiter="." --fields=1-2 | sort -V -u)
+VERSIONS_TO_TEST=()
+
+test_alter_from() {
+    local from_ver=$1
+    local to_ver=$CUR_VERSION
+
+    gpconfig -c shared_preload_libraries -v ""
+    gpstop -rai
+    dropdb diskquota --if-exists
+    dropdb diskquota_alter_test --if-exists
+    createdb diskquota
+
+    local from_so_name="diskquota"
+    if [ "${from_ver}" != "1.0" ];then
+        from_so_name="diskquota-${from_ver}"
+    fi
+    local to_so_name="diskquota-${to_ver}"
+
+    # Preload the old diskquota so
+    gpconfig -c shared_preload_libraries -v "${from_so_name}"
+    gpstop -rai
+
+    createdb diskquota_alter_test
+
+    # Test if the extension and be upgraded directly
+    psql -d diskquota_alter_test -c "CREATE EXTENSION diskquota version '${from_ver}'"
+
+    # Preload the new diskquota so
+    gpconfig -c shared_preload_libraries -v "${to_so_name}"
+    gpstop -rai
+
+    psql -d diskquota_alter_test -c "ALTER EXTENSION diskquota update to '${to_ver}'"
+    psql -d diskquota_alter_test -c "DROP EXTENSION diskquota"
+}
+
+# Find all minor versions before current one
+while IFS= read -r ver; do
+    if [ "${ver}" = "${CUR_VERSION}" ]; then
+        break
+    fi
+    if [ "${ver}" = "0.8" ]; then
+        continue
+    fi
+    VERSIONS_TO_TEST+=("${ver}")
+done <<< "$ALL_VERSIONS"
+
+for from_ver in "${VERSIONS_TO_TEST[@]}"; do
+    test_alter_from "${from_ver}"
+done

From 2aefa8028e13c943ef1674627ed22fb29bfdbf45 Mon Sep 17 00:00:00 2001
From: Zhang Hao <hzhang2@vmware.com>
Date: Thu, 29 Jun 2023 11:31:52 +0800
Subject: [PATCH 11/19] Add a sleep in alter_test.sh (#351)

Co-authored-by: Chen Mulong chenmulong@gmail.com
---
 upgrade_test/alter_test.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/upgrade_test/alter_test.sh b/upgrade_test/alter_test.sh
index 11c7afd7..96e29af0 100755
--- a/upgrade_test/alter_test.sh
+++ b/upgrade_test/alter_test.sh
@@ -42,6 +42,9 @@ test_alter_from() {
     gpstop -rai
 
     psql -d diskquota_alter_test -c "ALTER EXTENSION diskquota update to '${to_ver}'"
+    # Sleep wait for bgworker starting, otherwise, we will get a warning
+    # 'cannot remove the database from db list, dbid not found'.
+    sleep 5
     psql -d diskquota_alter_test -c "DROP EXTENSION diskquota"
 }
 

From 86ff586b5f7018983e7348db06ad87c8d6f73359 Mon Sep 17 00:00:00 2001
From: Zhang Hao <hzhang2@vmware.com>
Date: Thu, 29 Jun 2023 16:31:13 +0800
Subject: [PATCH 12/19] Update to 2.2.2 (#352)

- bump version to 2.2.2
- Modify the check procedure of alter extension.
---
 VERSION         |  2 +-
 src/diskquota.c | 57 ++++++++++++++++++++++++++++++++++---------------
 2 files changed, 41 insertions(+), 18 deletions(-)

diff --git a/VERSION b/VERSION
index c043eea7..b1b25a5f 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.2.1
+2.2.2
diff --git a/src/diskquota.c b/src/diskquota.c
index 3c4fa634..dd681437 100644
--- a/src/diskquota.c
+++ b/src/diskquota.c
@@ -42,6 +42,7 @@
 #include "utils/snapmgr.h"
 #include "utils/syscache.h"
 #include "utils/timestamp.h"
+#include "utils/formatting.h"
 #include "tcop/pquery.h"
 
 PG_MODULE_MAGIC;
@@ -147,6 +148,7 @@ static void                    free_bgworker_handle(uint32 worker_id);
 /* WaitForBackgroundWorkerShutdown is copied from gpdb7 */
 static BgwHandleStatus WaitForBackgroundWorkerShutdown(BackgroundWorkerHandle *handle);
 #endif /* GP_VERSION_NUM */
+static bool is_altering_extension(void);
 
 /*
  * diskquota_launcher_shmem_size
@@ -163,6 +165,40 @@ diskquota_launcher_shmem_size(void)
 	size = add_size(size, mul_size(MAX_NUM_MONITORED_DB, sizeof(struct DiskquotaDBEntry))); // hidden memory for dbArray
 	return size;
 }
+
+static bool
+is_altering_extension(void)
+{
+	if (ActivePortal == NULL) return false;
+	/* QD: When the sourceTag is T_AlterExtensionStmt, then return true */
+	if (ActivePortal->sourceTag == T_AlterExtensionStmt) return true;
+
+	/*
+	 * QE: The sourceTag won't be T_AlterExtensionStmt, we should check the sourceText.
+	 * If the sourceText contains 'alter extension diskquota update', we consider it is
+	 * a alter extension query.
+	 */
+	char *query = asc_tolower(ActivePortal->sourceText, strlen(ActivePortal->sourceText));
+	char *pos;
+	bool  match = true;
+
+	pos = strstr(query, "alter");
+	if (pos)
+		pos = strstr(pos, "extension");
+	else
+		match = false;
+	if (pos)
+		pos = strstr(pos, "diskquota");
+	else
+		match = false;
+	if (pos)
+		pos = strstr(pos, "update");
+	else
+		match = false;
+	pfree(query);
+	return match;
+}
+
 /*
  * Entrypoint of diskquota module.
  *
@@ -178,25 +214,12 @@ _PG_init(void)
 	{
 		/*
 		 * To support the continuous upgrade/downgrade, we should skip the library
-		 * check in _PG_init() during upgrade/downgrade. If the POSTGRES backend
-		 * process is in normal mode and meets one of the following conditions, we
-		 * skip the library check:
-		 * - The backend is not a QD. We only need to check the library on QD.
-		 * - The current command is `ALTER EXTENSION`.
+		 * check in _PG_init() during upgrade/downgrade.
 		 */
-		if (IsNormalProcessingMode())
+		if (IsNormalProcessingMode() && is_altering_extension())
 		{
-			if (Gp_role != GP_ROLE_DISPATCH)
-			{
-				ereport(WARNING, (errmsg("[diskquota] booting " DISKQUOTA_VERSION ", but " DISKQUOTA_BINARY_NAME
-				                         " not in shared_preload_libraries.")));
-				return;
-			}
-			if (ActivePortal && ActivePortal->sourceTag == T_AlterExtensionStmt)
-			{
-				ereport(LOG, (errmsg("[diskquota] altering diskquota version to " DISKQUOTA_VERSION ".")));
-				return;
-			}
+			ereport(LOG, (errmsg("[diskquota] altering diskquota version to " DISKQUOTA_VERSION ".")));
+			return;
 		}
 		ereport(ERROR, (errmsg("[diskquota] booting " DISKQUOTA_VERSION ", but " DISKQUOTA_BINARY_NAME
 		                       " not in shared_preload_libraries. abort.")));

From f1ca0c5ad9680f780622efab4e600d8b5fd13beb Mon Sep 17 00:00:00 2001
From: Zhang Hao <hzhang2@vmware.com>
Date: Mon, 3 Jul 2023 14:53:52 +0800
Subject: [PATCH 13/19] Reduce the remain logs in bgworker. (#354)

Fix from #346
---
 src/diskquota.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/diskquota.c b/src/diskquota.c
index dd681437..7d09f70d 100644
--- a/src/diskquota.c
+++ b/src/diskquota.c
@@ -541,7 +541,6 @@ disk_quota_worker_main(Datum main_arg)
 	long        sleep_time = diskquota_naptime * 1000;
 	long        secs;
 	int         usecs;
-	ereport(LOG, (errmsg("[diskquota] disk quota worker process is monitoring database:%s", dbname)));
 
 	while (!got_sigterm)
 	{

From 2c086fc82ea2d45f301e63fe9bd7860d012adbd3 Mon Sep 17 00:00:00 2001
From: Zhang Hao <hzhang2@vmware.com>
Date: Tue, 4 Jul 2023 11:03:50 +0800
Subject: [PATCH 14/19] Fix bug: bgworkers only print log once. (#356)

---
 src/diskquota.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/diskquota.c b/src/diskquota.c
index 7d09f70d..84ac5e07 100644
--- a/src/diskquota.c
+++ b/src/diskquota.c
@@ -553,7 +553,7 @@ disk_quota_worker_main(Datum main_arg)
 		 * by the bgworker's pid in the log file.
 		 */
 		log_time = GetCurrentTimestamp();
-		if (TimestampDifferenceExceeds(log_time, MyWorkerInfo->dbEntry->last_log_time, BGWORKER_LOG_TIME))
+		if (TimestampDifferenceExceeds(MyWorkerInfo->dbEntry->last_log_time, log_time, BGWORKER_LOG_TIME))
 		{
 			ereport(LOG, (errmsg("[diskquota] disk quota worker process is monitoring database:%s", dbname)));
 			MyWorkerInfo->dbEntry->last_log_time = log_time;

From 22c35b656b6587d1a411a0d1e3a4532f9ee062d0 Mon Sep 17 00:00:00 2001
From: liuxueyang <liuxueyang457@gmail.com>
Date: Tue, 4 Jul 2023 17:54:34 +0800
Subject: [PATCH 15/19] Update resources by using gp-extensions-ci subtree

---
 concourse/PROJ_NAME                    |   1 +
 concourse/README.md                    |   8 ++
 concourse/fly.sh                       | 168 +-----------------------
 concourse/pipeline/base.lib.yml        |  41 ------
 concourse/pipeline/res_def.yml         | 172 -------------------------
 concourse/pipeline/trigger_def.lib.yml |  22 ++--
 6 files changed, 21 insertions(+), 391 deletions(-)
 create mode 100644 concourse/PROJ_NAME
 mode change 100755 => 120000 concourse/fly.sh
 delete mode 100644 concourse/pipeline/base.lib.yml

diff --git a/concourse/PROJ_NAME b/concourse/PROJ_NAME
new file mode 100644
index 00000000..67f1c089
--- /dev/null
+++ b/concourse/PROJ_NAME
@@ -0,0 +1 @@
+diskquota
diff --git a/concourse/README.md b/concourse/README.md
index 259b436b..1007f454 100644
--- a/concourse/README.md
+++ b/concourse/README.md
@@ -81,6 +81,14 @@ To test if the webhook works, use `curl` to send a `POST` request to the hook UR
 curl --data-raw "foo" <hook_url>
 ```
 
+## Update gp-extensions-ci
+
+We place some of the resources of concourse in a separate repository https://github.com/pivotal/gp-extensions-ci/. And we use that repo as a subtree with prefix ./concourse/lib. This is how to pull from the repo gp-extensions-ci:
+
+```sh
+  git subtree pull --prefix concourse/lib git@github.com:pivotal/gp-extensions-ci.git main --squash
+```
+
 # FAQ
 
 ## PR pipeline is not triggered.
diff --git a/concourse/fly.sh b/concourse/fly.sh
deleted file mode 100755
index 75404b6b..00000000
--- a/concourse/fly.sh
+++ /dev/null
@@ -1,167 +0,0 @@
-#!/bin/bash
-
-set -e
-
-fly=${FLY:-"fly"}
-echo "'fly' command: ${fly}"
-echo ""
-proj_name="diskquota"
-concourse_team="main"
-
-usage() {
-    if [ -n "$1" ]; then
-        echo "$1" 1>&2
-        echo "" 1>&2
-    fi
-
-    echo "Usage: $0 -t <concourse_target> -c <pr|rel|merge|dev> [-p <postfix>] [-b branch] [-T]"
-    echo "Options:"
-    echo "       '-T' adds '_test' suffix to the pipeline type. Useful for pipeline debugging."
-    exit 1
-}
-
-# Hacky way to find out which concourse team is being used.
-# The team name is needed to generate webhook URL
-detect_concourse_team() {
-    local target="$1"
-    local fly_rc_file="$HOME/.flyrc"
-    local found_target=false
-    while read -r line;
-    do
-        line="$(echo -e "${line}" | tr -d '[:space:]')"
-        if [ ${found_target} != true ] && [ "${line}" = "${target}:" ]; then
-            found_target=true
-        fi
-        if [ ${found_target} = true ] && [[ "${line}" == team:* ]]; then
-            concourse_team=$(echo "${line}" | cut --delimiter=":" --fields=2)
-            echo "Use concourse target: ${target}, team: ${concourse_team}"
-            return
-        fi
-    done < "${fly_rc_file}"
-}
-
-# Parse command line options
-while getopts ":c:t:p:b:T" o; do
-    case "${o}" in
-        c)
-            # pipeline type/config. pr/merge/dev/rel
-            pipeline_config=${OPTARG}
-            ;;
-        t)
-            # concourse target
-            target=${OPTARG}
-            ;;
-        p)
-            # pipeline name
-            postfix=${OPTARG}
-            ;;
-        b)
-            # branch name
-            branch=${OPTARG}
-            ;;
-        T)
-            test_suffix="_test"
-            ;;
-        *)
-            usage ""
-            ;;
-    esac
-done
-shift $((OPTIND-1))
-
-if [ -z "${target}" ] || [ -z "${pipeline_config}" ]; then
-    usage ""
-fi
-
-detect_concourse_team "${target}"
-
-pipeline_type=""
-# Decide ytt options to generate pipeline
-case ${pipeline_config} in
-  pr)
-      pipeline_type="pr"
-      config_file="pr.yml"
-      hook_res="${proj_name}_pr"
-    ;;
-  merge|commit)
-      # Default branch is 'gpdb' as it is our main branch
-      if [ -z "${branch}" ]; then
-          branch="gpdb"
-      fi
-      pipeline_type="merge"
-      config_file="commit.yml"
-      hook_res="${proj_name}_commit"
-    ;;
-  dev)
-      if [ -z "${postfix}" ]; then
-          usage "'-p' needs to be supplied to specify the pipeline name postfix for flying a 'dev' pipeline."
-      fi
-      if [ -z "${branch}" ]; then
-          usage "'-b' needs to be supplied to specify the branch for flying a 'dev' pipeline."
-      fi
-      pipeline_type="dev"
-      config_file="dev.yml"
-    ;;
-  release|rel)
-      # Default branch is 'gpdb' as it is our main branch
-      if [ -z "${branch}" ]; then
-          branch="gpdb"
-      fi
-      pipeline_type="rel"
-      config_file="release.yml"
-      hook_res="${proj_name}_commit"
-    ;;
-  *)
-      usage ""
-    ;;
-esac
-
-yml_path="/tmp/${proj_name}.yml"
-my_path=$(realpath "${BASH_SOURCE[0]}")
-ytt_base=$(dirname "${my_path}")/pipeline
-# pipeline cannot contain '/'
-pipeline_name=${pipeline_name/\//"_"}
-
-# Generate pipeline name
-if [ -n "${test_suffix}" ]; then
-    pipeline_type="${pipeline_type}_test"
-fi
-pipeline_name="${pipeline_type}.${proj_name}"
-if [ -n "${branch}" ]; then
-    pipeline_name="${pipeline_name}.${branch}"
-fi
-if [ -n "${postfix}" ]; then
-    pipeline_name="${pipeline_name}.${postfix}"
-fi
-# pipeline cannot contain '/'
-pipeline_name=${pipeline_name/\//"_"}
-
-ytt --data-values-file "${ytt_base}/res_def.yml" \
-    -f "${ytt_base}/base.lib.yml" \
-    -f "${ytt_base}/job_def.lib.yml" \
-    -f "${ytt_base}/trigger_def.lib.yml" \
-    -f "${ytt_base}/${config_file}" > "${yml_path}"
-echo "Generated pipeline yaml '${yml_path}'."
-
-echo ""
-echo "Fly the pipeline..."
-set -v
-"${fly}" \
-    -t "${target}" \
-    sp \
-    -p "${pipeline_name}" \
-    -c "${yml_path}" \
-    -v "${proj_name}-branch=${branch}"
-set +v
-
-if [ "${pipeline_config}" == "dev" ]; then
-    exit 0
-fi
-
-concourse_url=$(fly targets | awk "{if (\$1 == \"${target}\") {print \$2}}")
-echo ""
-echo "================================================================================"
-echo "Remeber to set the the webhook URL on GitHub:"
-echo "${concourse_url}/api/v1/teams/${concourse_team}/pipelines/${pipeline_name}/resources/${hook_res}/check/webhook?webhook_token=<hook_token>"
-echo "You may need to change the base URL if a differnt concourse server is used."
-echo "================================================================================"
diff --git a/concourse/fly.sh b/concourse/fly.sh
new file mode 120000
index 00000000..09b854d1
--- /dev/null
+++ b/concourse/fly.sh
@@ -0,0 +1 @@
+lib/fly.sh
\ No newline at end of file
diff --git a/concourse/pipeline/base.lib.yml b/concourse/pipeline/base.lib.yml
deleted file mode 100644
index 1a62132a..00000000
--- a/concourse/pipeline/base.lib.yml
+++ /dev/null
@@ -1,41 +0,0 @@
-#@ load("@ytt:data", "data")
-#! add_res_by_xxx is to solve the unused resources error for concourse
-#@ def add_res_by_conf(res_map, job_conf):
-#@   for key in job_conf:
-#@     if key.startswith("res_"):
-#@       val = job_conf[key]
-#@       if type(val) == "list" or type(val) == "yamlfragment":
-#@         for res_name in val:
-#@           res_map[res_name] = True
-#@         end
-#@       else:
-#@         res_name = val
-#@         res_map[res_name] = True
-#@       end
-#@     end
-#@   end
-#@ end
-#@
-#@ def add_res_by_name(res_map, res_name):
-#@   res_map[res_name] = True
-#@ end
-#@
-#@ def declare_res(res_type_map, res_map):
-#@   for val in data.values.resources:
-#@     res_name = val["name"]
-#@     res_type = val["type"]
-#@     if res_map.get(val["name"]):
-#@       res_type_map[res_type] = True
-  - #@ val
-#@     end
-#@   end
-#@ end
-#@
-#@ def declare_res_type(res_type_map):
-#@   for val in data.values.resource_types:
-#@     type_name = val["name"]
-#@     if res_type_map.get(type_name):
-  - #@ val
-#@     end
-#@   end
-#@ end
diff --git a/concourse/pipeline/res_def.yml b/concourse/pipeline/res_def.yml
index f53bb0a3..1206969a 100644
--- a/concourse/pipeline/res_def.yml
+++ b/concourse/pipeline/res_def.yml
@@ -1,21 +1,3 @@
-resource_types:
-- name: gcs
-  type: docker-image
-  check_every: 1h
-  source:
-    repository: frodenas/gcs-resource
-
-- name: pull-request
-  type: docker-image
-  check_every: 1h
-  source:
-    repository: teliaoss/github-pr-resource
-
-- name: slack-alert
-  type: docker-image
-  source:
-    repository: arbourd/concourse-slack-alert-resource
-
 resources:
 # Pull Request
 - name: diskquota_pr
@@ -49,155 +31,6 @@ resources:
     password: x-oauth-basic
 
 
-# Greenplum sources
-- name: gpdb6_src
-  type: git
-  source:
-    branch: 6X_STABLE
-    uri: https://github.com/greenplum-db/gpdb.git
-- name: gpdb7_src
-  type: git
-  source:
-    branch: main
-    uri: https://github.com/greenplum-db/gpdb.git
-
-# Image Resources
-# centos6
-- name: centos6-gpdb6-image-build
-  type: registry-image
-  source:
-    repository: gcr.io/data-gpdb-public-images/gpdb6-centos6-build
-    tag: latest
-- name: centos6-gpdb6-image-test
-  type: registry-image
-  source:
-    repository: gcr.io/data-gpdb-public-images/gpdb6-centos6-test
-    tag: latest
-# centos7
-- name: centos7-gpdb6-image-build
-  type: registry-image
-  source:
-    repository: gcr.io/data-gpdb-public-images/gpdb6-centos7-build
-    tag: latest
-- name: centos7-gpdb6-image-test
-  type: registry-image
-  source:
-    repository: gcr.io/data-gpdb-public-images/gpdb6-centos7-test
-    tag: latest
-# rhel8
-- name: rhel8-gpdb6-image-build
-  type: registry-image
-  source:
-    repository: gcr.io/data-gpdb-private-images/gpdb6-rhel8-build
-    tag: latest
-    username: _json_key
-    password: ((data-gpdb-private-images-container-registry-readonly-service-account-key))
-- name: rhel8-gpdb6-image-test
-  type: registry-image
-  source:
-    repository: gcr.io/data-gpdb-private-images/gpdb6-rhel8-test
-    tag: latest
-    username: _json_key
-    password: ((data-gpdb-private-images-container-registry-readonly-service-account-key))
-# Ubuntu18
-- name: ubuntu18-gpdb6-image-build
-  type: registry-image
-  source:
-    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-build
-    tag: latest
-- name: ubuntu18-gpdb6-image-test
-  type: registry-image
-  source:
-    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-test
-    tag: latest
-# GPDB7
-# build
-- name: rocky8-gpdb7-image-build
-  type: registry-image
-  source:
-    repository: gcr.io/data-gpdb-public-images/gpdb7-rocky8-build
-    tag: latest
-# test
-- name: rocky8-gpdb7-image-test
-  type: registry-image
-  source:
-    repository: gcr.io/data-gpdb-public-images/gpdb7-rocky8-test
-    tag: latest
-- name: rhel8-gpdb7-image-test
-  type: registry-image
-  source:
-    repository: gcr.io/data-gpdb-private-images/gpdb7-rhel8-test
-    tag: latest
-    username: _json_key
-    password: ((data-gpdb-private-images-container-registry-readonly-service-account-key))
-
-# gpdb binary on gcs is located as different folder for different version
-# Latest build with assertion enabled:
-# --enable-cassert --enable-tap-tests --enable-debug-extensions
-- name: bin_gpdb6_centos6_debug
-  type: gcs
-  source:
-    bucket: pivotal-gpdb-concourse-resources-prod
-    json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/published/gpdb6/server-rc-(.*)-rhel6_x86_64.debug.tar.gz
-- name: bin_gpdb6_centos7_debug
-  type: gcs
-  source:
-    bucket: pivotal-gpdb-concourse-resources-prod
-    json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/published/gpdb6/server-rc-(.*)-rhel7_x86_64.debug.tar.gz
-- name: bin_gpdb6_rhel8_debug
-  type: gcs
-  source:
-    bucket: pivotal-gpdb-concourse-resources-prod
-    json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/published/gpdb6/server-rc-(.*)-rhel8_x86_64.debug.tar.gz
-- name: bin_gpdb6_ubuntu18_debug
-  type: gcs
-  source:
-    bucket: pivotal-gpdb-concourse-resources-prod
-    json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64.debug.tar.gz
-- name: bin_gpdb7_el8_debug
-  type: gcs
-  source:
-    bucket: pivotal-gpdb-concourse-resources-prod
-    json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/published/main/server-rc-(.*)-el8_x86_64.debug.tar.gz
-
-# Latest release candidates, no fault-injector, no assertion:
-# --disable-debug-extensions --disable-tap-tests --enable-ic-proxy
-- name: bin_gpdb6_centos6
-  type: gcs
-  source:
-    bucket: pivotal-gpdb-concourse-resources-prod
-    json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev\.\d+)-.*-centos6.tar.gz
-- name: bin_gpdb6_centos7
-  type: gcs
-  source:
-    bucket: pivotal-gpdb-concourse-resources-prod
-    json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev\.\d+)-.*-centos7.tar.gz
-- name: bin_gpdb6_rhel8
-  type: gcs
-  source:
-    bucket: pivotal-gpdb-concourse-resources-prod
-    json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev\.\d+)-.*-rhel8.tar.gz
-- name: bin_gpdb6_ubuntu18
-  type: gcs
-  source:
-    bucket: pivotal-gpdb-concourse-resources-prod
-    json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev\.\d+)-.*-ubuntu18.04.tar.gz
-- name: bin_gpdb7_el8
-  type: gcs
-  source:
-    bucket: pivotal-gpdb-concourse-resources-prod
-    json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb7/greenplum-db-server-(7\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev\.\d+)-.*-el8.tar.gz
-
 # Diskquota releases
 - name: bin_diskquota_gpdb6_rhel6
   type: gcs
@@ -350,8 +183,3 @@ resources:
     bucket: gp-extensions-ci
     json_key: ((extension/extensions-gcs-service-account-key-dev2))
     regexp: dependencies/cmake-(.*)-linux-x86_64.sh
-
-- name: slack_notify
-  type: slack-alert
-  source:
-    url: ((extension/extensions-slack-webhook))
diff --git a/concourse/pipeline/trigger_def.lib.yml b/concourse/pipeline/trigger_def.lib.yml
index c2854d02..607cb7a3 100644
--- a/concourse/pipeline/trigger_def.lib.yml
+++ b/concourse/pipeline/trigger_def.lib.yml
@@ -26,16 +26,16 @@ on_error:
     path: diskquota_src
     status: failure
 on_success:
-   put: diskquota_pr
-   params:
-     path: diskquota_src
-     status: success
+  put: diskquota_pr
+  params:
+    path: diskquota_src
+    status: success
 #@ end
 
 #! Commit trigger. For master pipelines
 #@ def commit_trigger(res_map):
 #@   add_res_by_name(res_map, "diskquota_commit")
-#@   add_res_by_name(res_map, "slack_notify")
+#@   add_res_by_name(res_map, "slack_notify_extensions")
 auto_trigger: true
 to_get:
 - get: diskquota_src
@@ -46,11 +46,11 @@ to_put_post: #@ []
 #! Unfortunately it doesn't work with Concourse 5.
 on_success:
 on_failure:
-  put: slack_notify
+  put: slack_notify_extensions
   params:
     alert_type: failed
 on_error:
-  put: slack_notify
+  put: slack_notify_extensions
   params:
     alert_type: errored
 #@ end
@@ -74,7 +74,7 @@ on_error:
 #! Commit trigger. For release pipelines
 #@ def release_trigger(res_map):
 #@   add_res_by_name(res_map, "diskquota_commit")
-#@   add_res_by_name(res_map, "slack_notify")
+#@   add_res_by_name(res_map, "slack_notify_extensions")
 auto_trigger: true
 to_get:
 - get: diskquota_src
@@ -86,16 +86,16 @@ to_put_post:
     repository: diskquota_src
     tag: diskquota_src/VERSION
 on_success:
-  put: slack_notify
+  put: slack_notify_extensions
   params:
     alert_type: success
     text: A new diskquota release has been pushed!
 on_failure:
-  put: slack_notify
+  put: slack_notify_extensions
   params:
     alert_type: failed
 on_error:
-  put: slack_notify
+  put: slack_notify_extensions
   params:
     alert_type: errored
 #@ end

From dbd044f46d7672331e19cbd6b95a6f8df67511af Mon Sep 17 00:00:00 2001
From: liuxueyang <liuxueyang457@gmail.com>
Date: Wed, 5 Jul 2023 09:21:28 +0800
Subject: [PATCH 16/19] Squashed 'concourse/lib/' content from commit d51adf5

git-subtree-dir: concourse/lib
git-subtree-split: d51adf5d59e4e5caefb678b29c4335881040c7d2
---
 README.md         |   1 +
 base.lib.yml      |  60 ++++++++++++++++
 fly.sh            | 179 ++++++++++++++++++++++++++++++++++++++++++++++
 res_def_gpdb.yml  | 148 ++++++++++++++++++++++++++++++++++++++
 res_def_misc.yml  |   7 ++
 res_types_def.yml |  21 ++++++
 6 files changed, 416 insertions(+)
 create mode 100644 README.md
 create mode 100644 base.lib.yml
 create mode 100755 fly.sh
 create mode 100644 res_def_gpdb.yml
 create mode 100644 res_def_misc.yml
 create mode 100644 res_types_def.yml

diff --git a/README.md b/README.md
new file mode 100644
index 00000000..0dd8ca55
--- /dev/null
+++ b/README.md
@@ -0,0 +1 @@
+# Common ytt libs for gp-extensions concourse
diff --git a/base.lib.yml b/base.lib.yml
new file mode 100644
index 00000000..78072de5
--- /dev/null
+++ b/base.lib.yml
@@ -0,0 +1,60 @@
+#@ load("@ytt:data", "data")
+#@ load("@ytt:struct", "struct")
+
+#! resources and resource_types could be declared in different data-value files.
+#! We check the key prefix, if they are 'resources' and 'resource_types', just
+#! merge them into local dictionaries.
+#@ resources = []
+#@ resource_types = []
+#@ data_values_dict = struct.decode(data.values)
+#@ for key in data_values_dict.keys():
+#@   if key.startswith('resources'):
+#@     resources.extend(data_values_dict[key])
+#@   end
+#@ end
+#@ for key in data_values_dict.keys():
+#@   if key.startswith('resource_type'):
+#@     resource_types.extend(data_values_dict[key])
+#@   end
+#@ end
+
+#! add_res_by_xxx is to solve the unused resources error for concourse
+#@ def add_res_by_conf(res_map, job_conf):
+#@   for key in job_conf:
+#@     if key.startswith("res_"):
+#@       val = job_conf[key]
+#@       if type(val) == "list" or type(val) == "yamlfragment":
+#@         for res_name in val:
+#@           res_map[res_name] = True
+#@         end
+#@       else:
+#@         res_name = val
+#@         res_map[res_name] = True
+#@       end
+#@     end
+#@   end
+#@ end
+#@
+#@ def add_res_by_name(res_map, res_name):
+#@   res_map[res_name] = True
+#@ end
+#@
+#@ def declare_res(res_type_map, res_map):
+#@   for val in resources:
+#@     res_name = val["name"]
+#@     res_type = val["type"]
+#@     if res_map.get(val["name"]):
+#@       res_type_map[res_type] = True
+  - #@ val
+#@     end
+#@   end
+#@ end
+#@
+#@ def declare_res_type(res_type_map):
+#@   for val in resource_types:
+#@     type_name = val["name"]
+#@     if res_type_map.get(type_name):
+  - #@ val
+#@     end
+#@   end
+#@ end
diff --git a/fly.sh b/fly.sh
new file mode 100755
index 00000000..d68c5c5c
--- /dev/null
+++ b/fly.sh
@@ -0,0 +1,179 @@
+#!/bin/bash
+
+set -e
+
+fly=${FLY:-"fly"}
+echo "'fly' command: ${fly}"
+echo ""
+
+my_path=$(realpath -s "${BASH_SOURCE[0]}")
+my_dir=$(dirname "${my_path}")
+proj_name_file="${my_dir}/PROJ_NAME"
+if [ ! -f "${proj_name_file}" ]; then
+    echo "A 'PROJ_NAME' file is needed in '${my_dir}'"
+    exit 1
+fi
+proj_name=$(cat "${proj_name_file}")
+concourse_team="main"
+
+usage() {
+    if [ -n "$1" ]; then
+        echo "$1" 1>&2
+        echo "" 1>&2
+    fi
+
+    echo "Usage: $0 -t <concourse_target> -c <pr|rel|merge|dev> [-p <postfix>] [-b branch] [-T]"
+    echo "Options:"
+    echo "       '-T' adds '_test' suffix to the pipeline type. Useful for pipeline debugging."
+    exit 1
+}
+
+# Hacky way to find out which concourse team is being used.
+# The team name is needed to generate webhook URL
+detect_concourse_team() {
+    local target="$1"
+    local fly_rc_file="$HOME/.flyrc"
+    local found_target=false
+    while read -r line;
+    do
+        line="$(echo -e "${line}" | tr -d '[:space:]')"
+        if [ ${found_target} != true ] && [ "${line}" = "${target}:" ]; then
+            found_target=true
+        fi
+        if [ ${found_target} = true ] && [[ "${line}" == team:* ]]; then
+            concourse_team=$(echo "${line}" | cut --delimiter=":" --fields=2)
+            echo "Use concourse target: ${target}, team: ${concourse_team}"
+            return
+        fi
+    done < "${fly_rc_file}"
+}
+
+# Parse command line options
+while getopts ":c:t:p:b:T" o; do
+    case "${o}" in
+        c)
+            # pipeline type/config. pr/merge/dev/rel
+            pipeline_config=${OPTARG}
+            ;;
+        t)
+            # concourse target
+            target=${OPTARG}
+            ;;
+        p)
+            # pipeline name
+            postfix=${OPTARG}
+            ;;
+        b)
+            # branch name
+            branch=${OPTARG}
+            ;;
+        T)
+            test_suffix="_test"
+            ;;
+        *)
+            usage ""
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [ -z "${target}" ] || [ -z "${pipeline_config}" ]; then
+    usage ""
+fi
+
+detect_concourse_team "${target}"
+
+pipeline_type=""
+# Decide ytt options to generate pipeline
+case ${pipeline_config} in
+  pr)
+      pipeline_type="pr"
+      config_file="pr.yml"
+      hook_res="${proj_name}_pr"
+    ;;
+  merge|commit)
+      # Default branch is 'gpdb' as it is our main branch
+      if [ -z "${branch}" ]; then
+          branch="gpdb"
+      fi
+      pipeline_type="merge"
+      config_file="commit.yml"
+      hook_res="${proj_name}_commit"
+    ;;
+  dev)
+      if [ -z "${postfix}" ]; then
+          usage "'-p' needs to be supplied to specify the pipeline name postfix for flying a 'dev' pipeline."
+      fi
+      if [ -z "${branch}" ]; then
+          usage "'-b' needs to be supplied to specify the branch for flying a 'dev' pipeline."
+      fi
+      pipeline_type="dev"
+      config_file="dev.yml"
+    ;;
+  release|rel)
+      # Default branch is 'gpdb' as it is our main branch
+      if [ -z "${branch}" ]; then
+          branch="gpdb"
+      fi
+      pipeline_type="rel"
+      config_file="release.yml"
+      hook_res="${proj_name}_commit"
+    ;;
+  *)
+      usage ""
+    ;;
+esac
+
+yml_path="/tmp/${proj_name}.yml"
+pipeline_dir="${my_dir}/pipeline"
+lib_dir="${my_dir}/lib"
+# pipeline cannot contain '/'
+pipeline_name=${pipeline_name/\//"_"}
+
+# Generate pipeline name
+if [ -n "${test_suffix}" ]; then
+    pipeline_type="${pipeline_type}_test"
+fi
+pipeline_name="${pipeline_type}.${proj_name}"
+if [ -n "${branch}" ]; then
+    pipeline_name="${pipeline_name}.${branch}"
+fi
+if [ -n "${postfix}" ]; then
+    pipeline_name="${pipeline_name}.${postfix}"
+fi
+# pipeline cannot contain '/'
+pipeline_name=${pipeline_name/\//"_"}
+
+ytt \
+    --data-values-file "${pipeline_dir}/res_def.yml" \
+    --data-values-file "${lib_dir}/res_def_gpdb.yml" \
+    --data-values-file "${lib_dir}/res_def_misc.yml" \
+    --data-values-file "${lib_dir}/res_types_def.yml" \
+    -f "${lib_dir}/base.lib.yml" \
+    -f "${pipeline_dir}/job_def.lib.yml" \
+    -f "${pipeline_dir}/trigger_def.lib.yml" \
+    -f "${pipeline_dir}/${config_file}" > "${yml_path}"
+echo "Generated pipeline yaml '${yml_path}'."
+
+echo ""
+echo "Fly the pipeline..."
+set -v
+"${fly}" \
+    -t "${target}" \
+    sp \
+    -p "${pipeline_name}" \
+    -c "${yml_path}" \
+    -v "${proj_name}-branch=${branch}"
+set +v
+
+if [ "${pipeline_config}" == "dev" ]; then
+    exit 0
+fi
+
+concourse_url=$(fly targets | awk "{if (\$1 == \"${target}\") {print \$2}}")
+echo ""
+echo "================================================================================"
+echo "Remeber to set the the webhook URL on GitHub:"
+echo "${concourse_url}/api/v1/teams/${concourse_team}/pipelines/${pipeline_name}/resources/${hook_res}/check/webhook?webhook_token=<hook_token>"
+echo "You may need to change the base URL if a different concourse server is used."
+echo "================================================================================"
diff --git a/res_def_gpdb.yml b/res_def_gpdb.yml
new file mode 100644
index 00000000..b4384480
--- /dev/null
+++ b/res_def_gpdb.yml
@@ -0,0 +1,148 @@
+resources_gpdb:
+# Image Resources
+# centos6
+- name: centos6-gpdb6-image-build
+  type: registry-image
+  source:
+    repository: gcr.io/data-gpdb-public-images/gpdb6-centos6-build
+    tag: latest
+- name: centos6-gpdb6-image-test
+  type: registry-image
+  source:
+    repository: gcr.io/data-gpdb-public-images/gpdb6-centos6-test
+    tag: latest
+# centos7
+- name: centos7-gpdb6-image-build
+  type: registry-image
+  source:
+    repository: gcr.io/data-gpdb-public-images/gpdb6-centos7-build
+    tag: latest
+- name: centos7-gpdb6-image-test
+  type: registry-image
+  source:
+    repository: gcr.io/data-gpdb-public-images/gpdb6-centos7-test
+    tag: latest
+# rhel8
+- name: rhel8-gpdb6-image-build
+  type: registry-image
+  source:
+    repository: gcr.io/data-gpdb-private-images/gpdb6-rhel8-build
+    tag: latest
+    username: _json_key
+    password: ((data-gpdb-private-images-container-registry-readonly-service-account-key))
+- name: rhel8-gpdb6-image-test
+  type: registry-image
+  source:
+    repository: gcr.io/data-gpdb-private-images/gpdb6-rhel8-test
+    tag: latest
+    username: _json_key
+    password: ((data-gpdb-private-images-container-registry-readonly-service-account-key))
+# Ubuntu18
+- name: ubuntu18-gpdb6-image-build
+  type: registry-image
+  source:
+    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-build
+    tag: latest
+- name: ubuntu18-gpdb6-image-test
+  type: registry-image
+  source:
+    repository: gcr.io/data-gpdb-public-images/gpdb6-ubuntu18.04-test
+    tag: latest
+# GPDB7
+# build
+- name: rocky8-gpdb7-image-build
+  type: registry-image
+  source:
+    repository: gcr.io/data-gpdb-public-images/gpdb7-rocky8-build
+    tag: latest
+# test
+- name: rocky8-gpdb7-image-test
+  type: registry-image
+  source:
+    repository: gcr.io/data-gpdb-public-images/gpdb7-rocky8-test
+    tag: latest
+- name: rhel8-gpdb7-image-test
+  type: registry-image
+  source:
+    repository: gcr.io/data-gpdb-private-images/gpdb7-rhel8-test
+    tag: latest
+    username: _json_key
+    password: ((data-gpdb-private-images-container-registry-readonly-service-account-key))
+
+# gpdb binary on gcs is located as different folder for different version
+# Latest build with assertion enabled:
+# --enable-cassert --enable-tap-tests --enable-debug-extensions
+- name: bin_gpdb6_centos6_debug
+  type: gcs
+  source:
+    bucket: pivotal-gpdb-concourse-resources-prod
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: server/published/gpdb6/server-rc-(.*\+dev\.\d+.*)-rhel6_x86_64.debug.tar.gz
+- name: bin_gpdb6_centos7_debug
+  type: gcs
+  source:
+    bucket: pivotal-gpdb-concourse-resources-prod
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: server/published/gpdb6/server-rc-(.*\+dev\.\d+.*)-rhel7_x86_64.debug.tar.gz
+- name: bin_gpdb6_rhel8_debug
+  type: gcs
+  source:
+    bucket: pivotal-gpdb-concourse-resources-prod
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: server/published/gpdb6/server-rc-(.*\+dev\.\d+.*)-rhel8_x86_64.debug.tar.gz
+- name: bin_gpdb6_ubuntu18_debug
+  type: gcs
+  source:
+    bucket: pivotal-gpdb-concourse-resources-prod
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: server/published/gpdb6/server-rc-(.*\+dev\.\d+.*)-ubuntu18.04_x86_64.debug.tar.gz
+- name: bin_gpdb7_el8_debug
+  type: gcs
+  source:
+    bucket: pivotal-gpdb-concourse-resources-prod
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: server/published/main/server-rc-(.*\+dev\.\d+.*)-el8_x86_64.debug.tar.gz
+# Latest release candidates, no fault-injector, no assertion:
+# --disable-debug-extensions --disable-tap-tests --enable-ic-proxy
+- name: bin_gpdb6_centos6
+  type: gcs
+  source:
+    bucket: pivotal-gpdb-concourse-resources-prod
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*\+dev\.\d+)-.*-centos6.tar.gz
+- name: bin_gpdb6_centos7
+  type: gcs
+  source:
+    bucket: pivotal-gpdb-concourse-resources-prod
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*\+dev\.\d+)-.*-centos7.tar.gz
+- name: bin_gpdb6_rhel8
+  type: gcs
+  source:
+    bucket: pivotal-gpdb-concourse-resources-prod
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*\+dev\.\d+)-.*-rhel8.tar.gz
+- name: bin_gpdb6_ubuntu18
+  type: gcs
+  source:
+    bucket: pivotal-gpdb-concourse-resources-prod
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: server/release-candidates/gpdb6/greenplum-db-server-(6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*\+dev\.\d+)-.*-ubuntu18.04.tar.gz
+- name: bin_gpdb7_el8
+  type: gcs
+  source:
+    bucket: pivotal-gpdb-concourse-resources-prod
+    json_key: ((concourse-gcs-resources-service-account-key))
+    regexp: server/release-candidates/gpdb7/greenplum-db-server-(7\.([0-9]|([1-8][0-9])|(9[0-8]))\..*\+dev\.\d+)-.*-el8.tar.gz
+
+# Greenplum sources
+- name: gpdb6_src
+  type: git
+  source:
+    branch: 6X_STABLE
+    uri: https://github.com/greenplum-db/gpdb.git
+- name: gpdb7_src
+  type: git
+  source:
+    branch: main
+    uri: https://github.com/greenplum-db/gpdb.git
diff --git a/res_def_misc.yml b/res_def_misc.yml
new file mode 100644
index 00000000..3105dd38
--- /dev/null
+++ b/res_def_misc.yml
@@ -0,0 +1,7 @@
+resources_misc:
+
+# Other dependencies
+- name: slack_notify_extensions
+  type: slack-alert
+  source:
+    url: ((extension/extensions-slack-webhook))
diff --git a/res_types_def.yml b/res_types_def.yml
new file mode 100644
index 00000000..07eebf6e
--- /dev/null
+++ b/res_types_def.yml
@@ -0,0 +1,21 @@
+resource_types_common:
+- name: gcs
+  type: registry-image
+  check_every: 1h
+  source:
+    repository: frodenas/gcs-resource
+    tag: latest
+
+- name: pull-request
+  type: registry-image
+  check_every: 1h
+  source:
+    repository: teliaoss/github-pr-resource
+    tag: latest
+
+- name: slack-alert
+  type: registry-image
+  source:
+    repository: arbourd/concourse-slack-alert-resource
+    tag: latest
+

From d6cae3c81fb0512c76b5242cfba99183c7d08bd0 Mon Sep 17 00:00:00 2001
From: Vasiliy Ivanov <ivi@arenadata.io>
Date: Tue, 25 Jul 2023 15:10:47 +0200
Subject: [PATCH 17/19] Revert "ADBDEV-3685 Error handling for disqkuota worker
 startup stage (#20)"

to reaaply above c2686c9

This reverts commit 3b06e37e4882b4a3d95448df1c944ded79ff9c07.
---
 src/diskquota.c                               | 34 ++++----------
 src/diskquota.h                               |  7 ++-
 src/gp_activetable.c                          |  1 -
 src/quotamodel.c                              | 19 ++------
 .../expected/test_worker_init_failure.out     | 46 -------------------
 tests/isolation2/isolation2_schedule          |  1 -
 .../sql/test_worker_init_failure.sql          | 40 ----------------
 7 files changed, 15 insertions(+), 133 deletions(-)
 delete mode 100644 tests/isolation2/expected/test_worker_init_failure.out
 delete mode 100644 tests/isolation2/sql/test_worker_init_failure.sql

diff --git a/src/diskquota.c b/src/diskquota.c
index 4f0fe43c..d5630700 100644
--- a/src/diskquota.c
+++ b/src/diskquota.c
@@ -142,7 +142,6 @@ static void                    vacuum_db_entry(DiskquotaDBEntry *db);
 static void                    init_bgworker_handles(void);
 static BackgroundWorkerHandle *get_bgworker_handle(uint32 worker_id);
 static void                    free_bgworker_handle(uint32 worker_id);
-static void                    resetBackgroundWorkerCorruption(void);
 #if GP_VERSION_NUM < 70000
 /* WaitForBackgroundWorkerShutdown is copied from gpdb7 */
 static BgwHandleStatus WaitForBackgroundWorkerShutdown(BackgroundWorkerHandle *handle);
@@ -526,7 +525,7 @@ disk_quota_worker_main(Datum main_arg)
 			if (!diskquota_is_paused())
 			{
 				/* Refresh quota model with init mode */
-				refresh_disk_quota_model(MyWorkerInfo->dbEntry);
+				refresh_disk_quota_model(!MyWorkerInfo->dbEntry->inited);
 				MyWorkerInfo->dbEntry->inited = true;
 				is_gang_destroyed             = false;
 			}
@@ -763,7 +762,6 @@ disk_quota_launcher_main(Datum main_arg)
 		{
 			elog(DEBUG1, "[diskquota] got sighup");
 			got_sighup = false;
-			resetBackgroundWorkerCorruption();
 			ProcessConfigFile(PGC_SIGHUP);
 		}
 
@@ -789,12 +787,11 @@ disk_quota_launcher_main(Datum main_arg)
 		 * When curDB->in_use is false means dbEtnry has been romoved
 		 * When curDB->dbid doesn't equtal curDBId, it means the slot has
 		 * been used by another db
-		 * When curDB->corrupted is true means worker couldn't initialize
-		 * the extension in the first run.
+		 *
 		 * For the above conditions, we just skip this loop and try to fetch
 		 * next db to run.
 		 */
-		if (curDB == NULL || !curDB->in_use || curDB->dbid != curDBId || curDB->corrupted)
+		if (curDB == NULL || !curDB->in_use || curDB->dbid != curDBId)
 		{
 			advance_one_db = true;
 			continue;
@@ -1799,9 +1796,7 @@ next_db(DiskquotaDBEntry *curDB)
 		if (nextSlot >= MAX_NUM_MONITORED_DB) nextSlot = 0;
 		DiskquotaDBEntry *dbEntry = &DiskquotaLauncherShmem->dbArray[nextSlot];
 		nextSlot++;
-		if (!dbEntry->in_use || dbEntry->workerId != INVALID_WORKER_ID || dbEntry->dbid == InvalidOid ||
-		    dbEntry->corrupted)
-			continue;
+		if (!dbEntry->in_use || dbEntry->workerId != INVALID_WORKER_ID || dbEntry->dbid == InvalidOid) continue;
 		/* TODO: should release the invalid db related things */
 		if (!is_valid_dbid(dbEntry->dbid)) continue;
 		result = dbEntry;
@@ -1865,11 +1860,10 @@ static void
 vacuum_db_entry(DiskquotaDBEntry *db)
 {
 	if (db == NULL) return;
-	db->dbid      = InvalidOid;
-	db->inited    = false;
-	db->workerId  = INVALID_WORKER_ID;
-	db->in_use    = false;
-	db->corrupted = false;
+	db->dbid     = InvalidOid;
+	db->inited   = false;
+	db->workerId = INVALID_WORKER_ID;
+	db->in_use   = false;
 }
 
 static void
@@ -1904,18 +1898,6 @@ free_bgworker_handle(uint32 worker_id)
 	}
 }
 
-static void
-resetBackgroundWorkerCorruption(void)
-{
-	LWLockAcquire(diskquota_locks.dblist_lock, LW_EXCLUSIVE);
-	for (int i = 0; i < MAX_NUM_MONITORED_DB; i++)
-	{
-		DiskquotaDBEntry *dbEntry = &DiskquotaLauncherShmem->dbArray[i];
-		if (dbEntry->corrupted) dbEntry->corrupted = false;
-	}
-	LWLockRelease(diskquota_locks.dblist_lock);
-}
-
 #if GP_VERSION_NUM < 70000
 static BgwHandleStatus
 WaitForBackgroundWorkerShutdown(BackgroundWorkerHandle *handle)
diff --git a/src/diskquota.h b/src/diskquota.h
index 58a00f46..f044773b 100644
--- a/src/diskquota.h
+++ b/src/diskquota.h
@@ -215,9 +215,8 @@ struct DiskquotaDBEntry
 	TimestampTz last_run_time;
 	int16       cost; // ms
 
-	bool inited;    // this entry is inited, will set to true after the worker finish the frist run.
-	bool in_use;    // this slot is in using. AKA dbid != 0
-	bool corrupted; // consider this entry as invalid to start the worker on
+	bool inited; // this entry is inited, will set to true after the worker finish the frist run.
+	bool in_use; // this slot is in using. AKA dbid != 0
 };
 
 typedef enum MonitorDBStatus
@@ -250,7 +249,7 @@ extern void invalidate_database_rejectmap(Oid dbid);
 /* quota model interface*/
 extern void init_disk_quota_shmem(void);
 extern void init_disk_quota_model(uint32 id);
-extern void refresh_disk_quota_model(DiskquotaDBEntry *dbEntry);
+extern void refresh_disk_quota_model(bool force);
 extern bool check_diskquota_state_is_ready(void);
 extern bool quota_check_common(Oid reloid, RelFileNode *relfilenode);
 
diff --git a/src/gp_activetable.c b/src/gp_activetable.c
index cbf6e7b6..cf3178b3 100644
--- a/src/gp_activetable.c
+++ b/src/gp_activetable.c
@@ -378,7 +378,6 @@ gp_fetch_active_tables(bool is_init)
 
 	if (is_init)
 	{
-		SIMPLE_FAULT_INJECTOR("diskquota_worker_initialization");
 		load_table_size(local_table_stats_map);
 	}
 	else
diff --git a/src/quotamodel.c b/src/quotamodel.c
index a0f01dbd..6b8507b3 100644
--- a/src/quotamodel.c
+++ b/src/quotamodel.c
@@ -227,7 +227,7 @@ static void clear_all_quota_maps(void);
 static void transfer_table_for_quota(int64 totalsize, QuotaType type, Oid *old_keys, Oid *new_keys, int16 segid);
 
 /* functions to refresh disk quota model*/
-static void refresh_disk_quota_usage(DiskquotaDBEntry *dbEntry);
+static void refresh_disk_quota_usage(bool is_init);
 static void calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map);
 static void flush_to_table_size(void);
 static bool flush_local_reject_map(void);
@@ -761,10 +761,8 @@ do_check_diskquota_state_is_ready(void)
  * recalculate the changed disk usage.
  */
 void
-refresh_disk_quota_model(DiskquotaDBEntry *dbEntry)
+refresh_disk_quota_model(bool is_init)
 {
-	bool is_init = !dbEntry->inited;
-
 	SEGCOUNT = getgpsegmentCount();
 	if (SEGCOUNT <= 0)
 	{
@@ -775,7 +773,7 @@ refresh_disk_quota_model(DiskquotaDBEntry *dbEntry)
 	/* skip refresh model when load_quotas failed */
 	if (load_quotas())
 	{
-		refresh_disk_quota_usage(dbEntry);
+		refresh_disk_quota_usage(is_init);
 	}
 	if (is_init) ereport(LOG, (errmsg("[diskquota] initialize quota model finished")));
 }
@@ -787,12 +785,11 @@ refresh_disk_quota_model(DiskquotaDBEntry *dbEntry)
  * process is constructing quota model.
  */
 static void
-refresh_disk_quota_usage(DiskquotaDBEntry *dbEntry)
+refresh_disk_quota_usage(bool is_init)
 {
 	bool  connected                   = false;
 	bool  pushed_active_snap          = false;
 	bool  ret                         = true;
-	bool  is_init                     = !dbEntry->inited;
 	HTAB *local_active_table_stat_map = NULL;
 
 	StartTransactionCommand();
@@ -844,14 +841,6 @@ refresh_disk_quota_usage(DiskquotaDBEntry *dbEntry)
 	}
 	PG_CATCH();
 	{
-		/* Initialization failed.  */
-		if (is_init)
-		{
-			LWLockAcquire(diskquota_locks.dblist_lock, LW_EXCLUSIVE);
-			dbEntry->corrupted = true;
-			LWLockRelease(diskquota_locks.dblist_lock);
-			PG_RE_THROW();
-		}
 		/* Prevents interrupts while cleaning up */
 		HOLD_INTERRUPTS();
 		EmitErrorReport();
diff --git a/tests/isolation2/expected/test_worker_init_failure.out b/tests/isolation2/expected/test_worker_init_failure.out
deleted file mode 100644
index 75e6b6e0..00000000
--- a/tests/isolation2/expected/test_worker_init_failure.out
+++ /dev/null
@@ -1,46 +0,0 @@
---
--- Tests for error handling when the worker catches the error during
--- its first run.
---
-
--- Function checking whether worker on given db is up
-CREATE or REPLACE LANGUAGE plpython2u;
-CREATE
-CREATE or REPLACE FUNCTION check_worker_presence(dbname text, wait_time int) RETURNS boolean AS $$ import psutil import time worker_name = 'bgworker: [diskquota] ' + dbname time.sleep(wait_time) for proc in psutil.process_iter(): try: if 'postgres' in proc.name().lower(): for val in proc.cmdline(): if worker_name in val: return True except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): pass return False $$ LANGUAGE plpython2u EXECUTE ON MASTER;
-CREATE
-
--- Test diskquota behavior when an error occurs during the worker's first run.
--- The error leads to process termination. And launcher won't start it again
--- until extension reload or SIGHUP signal.
-CREATE EXTENSION diskquota;
-CREATE
-SELECT check_worker_presence(current_database(), 0);
- check_worker_presence 
------------------------
- t                     
-(1 row)
-SELECT gp_inject_fault('diskquota_worker_initialization', 'error', dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1;
- gp_inject_fault 
------------------
- Success:        
-(1 row)
-SELECT diskquota.init_table_size_table();
- init_table_size_table 
------------------------
-                       
-(1 row)
-SELECT check_worker_presence(current_database(), current_setting('diskquota.worker_timeout')::int / 2);
- check_worker_presence 
------------------------
- f                     
-(1 row)
--- Reload configuration and check that worker is up again
-!\retcode gpstop -u;
-(exited with code 0)
-SELECT check_worker_presence(current_database(), current_setting('diskquota.worker_timeout')::int / 2);
- check_worker_presence 
------------------------
- t                     
-(1 row)
-DROP EXTENSION diskquota;
-DROP
diff --git a/tests/isolation2/isolation2_schedule b/tests/isolation2/isolation2_schedule
index c61f3d97..090c5cc5 100644
--- a/tests/isolation2/isolation2_schedule
+++ b/tests/isolation2/isolation2_schedule
@@ -11,5 +11,4 @@ test: test_per_segment_config
 test: test_relation_cache
 test: test_ereport_from_seg
 test: test_drop_extension
-test: test_worker_init_failure
 test: reset_config
diff --git a/tests/isolation2/sql/test_worker_init_failure.sql b/tests/isolation2/sql/test_worker_init_failure.sql
deleted file mode 100644
index 4e48908a..00000000
--- a/tests/isolation2/sql/test_worker_init_failure.sql
+++ /dev/null
@@ -1,40 +0,0 @@
---
--- Tests for error handling when the worker catches the error during
--- its first run.
---
-
--- Function checking whether worker on given db is up
-CREATE or REPLACE LANGUAGE plpython2u;
-CREATE or REPLACE FUNCTION check_worker_presence(dbname text, wait_time int)
-  RETURNS boolean
-AS $$
-    import psutil
-    import time
-    worker_name = 'bgworker: [diskquota] ' + dbname
-    time.sleep(wait_time)
-    for proc in psutil.process_iter():
-        try:
-            if 'postgres' in proc.name().lower():
-                for val in proc.cmdline():
-                    if worker_name in val:
-                        return True
-        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
-            pass
-    return False
-$$ LANGUAGE plpython2u EXECUTE ON MASTER;
-
--- Test diskquota behavior when an error occurs during the worker's first run.
--- The error leads to process termination. And launcher won't start it again
--- until extension reload or SIGHUP signal.
-CREATE EXTENSION diskquota;
-SELECT check_worker_presence(current_database(), 0);
-SELECT gp_inject_fault('diskquota_worker_initialization', 'error', dbid)
-  FROM gp_segment_configuration WHERE role='p' AND content=-1;
-SELECT diskquota.init_table_size_table();
-SELECT check_worker_presence(current_database(),
-  current_setting('diskquota.worker_timeout')::int / 2);
--- Reload configuration and check that worker is up again
-!\retcode gpstop -u;
-SELECT check_worker_presence(current_database(),
-  current_setting('diskquota.worker_timeout')::int / 2);
-DROP EXTENSION diskquota;

From f280a884c6056ad0d89a34c80b91485cf360f3ab Mon Sep 17 00:00:00 2001
From: Alexander Kondakov <a.kondakov@arenadata.io>
Date: Thu, 29 Jun 2023 09:50:35 +0300
Subject: [PATCH 18/19] ADBDEV-3685 Error handling for disqkuota worker startup
 stage (#20)

During diskquota worker's first run the initial set of active tables
with their sizes is being loaded from diskquota.table_size table in
order to warm up diskquota rejectmap and other shared memory objects.
If an error occurs during this initialization process, the error will be
ignored in PG_CATCH() block. Because of that local_active_table_stat_map
will not be filled properly. And at the next loop iteration tables, that
are not in acitive table list will be marked as irrelevant and to be deleted
both from table_size_map and table_size table in flush_to_table_size function.
In case when the inital set of active tables is huge (thousands of tables),
this error ignorance could lead to the formation of a too long
delete statement, which the SPI executor won't be able to process due to
memory limits. And this case can lead to worker's segmentation fault or
other errorneous behaviour of whole extension.

This commit proposes the handling of the initialization errors, which
occur during worker's first run. In the DiskquotaDBEntry structure the
bool variable "corrupted" is added in order to indicate, that the
worker wasn't able to initialize itself on given database. And
DiskquotaDBEntry also is now passed to refresh_disk_quota_model function
from worker main loop, because one need to change the state of dbEntry.
The state is changed when the refresh_disk_quota_usage function catches
an error, which occured during the initialization step, in PG_CATCH()
block. And after the error is catched, the "corrupted" flag is set in
given dbEntry, and then the error is rethrown. This leads to worker
process termination. The launcher will not be able to start it again,
because added flag is set in the database structure, and this flag is
being checked inside the disk_quota_launcher_main function. The flag
can be reseted by calling resetBackgroundWorkerCorruption function,
which is currently called in SIGHUP handler.

Cherry-picked-from: 3b06e37
to reapply above c2686c9
---
 src/diskquota.c                               | 34 ++++++++++----
 src/diskquota.h                               |  3 +-
 src/gp_activetable.c                          |  1 +
 src/quotamodel.c                              | 19 ++++++--
 .../expected/test_worker_init_failure.out     | 46 +++++++++++++++++++
 tests/isolation2/isolation2_schedule          |  1 +
 .../sql/test_worker_init_failure.sql          | 40 ++++++++++++++++
 7 files changed, 131 insertions(+), 13 deletions(-)
 create mode 100644 tests/isolation2/expected/test_worker_init_failure.out
 create mode 100644 tests/isolation2/sql/test_worker_init_failure.sql

diff --git a/src/diskquota.c b/src/diskquota.c
index 84ac5e07..83486722 100644
--- a/src/diskquota.c
+++ b/src/diskquota.c
@@ -144,6 +144,7 @@ static void                    vacuum_db_entry(DiskquotaDBEntry *db);
 static void                    init_bgworker_handles(void);
 static BackgroundWorkerHandle *get_bgworker_handle(uint32 worker_id);
 static void                    free_bgworker_handle(uint32 worker_id);
+static void                    resetBackgroundWorkerCorruption(void);
 #if GP_VERSION_NUM < 70000
 /* WaitForBackgroundWorkerShutdown is copied from gpdb7 */
 static BgwHandleStatus WaitForBackgroundWorkerShutdown(BackgroundWorkerHandle *handle);
@@ -575,7 +576,7 @@ disk_quota_worker_main(Datum main_arg)
 			if (!diskquota_is_paused())
 			{
 				/* Refresh quota model with init mode */
-				refresh_disk_quota_model(!MyWorkerInfo->dbEntry->inited);
+				refresh_disk_quota_model(MyWorkerInfo->dbEntry);
 				MyWorkerInfo->dbEntry->inited = true;
 				is_gang_destroyed             = false;
 			}
@@ -812,6 +813,7 @@ disk_quota_launcher_main(Datum main_arg)
 		{
 			elog(DEBUG1, "[diskquota] got sighup");
 			got_sighup = false;
+			resetBackgroundWorkerCorruption();
 			ProcessConfigFile(PGC_SIGHUP);
 		}
 
@@ -837,11 +839,12 @@ disk_quota_launcher_main(Datum main_arg)
 		 * When curDB->in_use is false means dbEtnry has been romoved
 		 * When curDB->dbid doesn't equtal curDBId, it means the slot has
 		 * been used by another db
-		 *
+		 * When curDB->corrupted is true means worker couldn't initialize
+		 * the extension in the first run.
 		 * For the above conditions, we just skip this loop and try to fetch
 		 * next db to run.
 		 */
-		if (curDB == NULL || !curDB->in_use || curDB->dbid != curDBId)
+		if (curDB == NULL || !curDB->in_use || curDB->dbid != curDBId || curDB->corrupted)
 		{
 			advance_one_db = true;
 			continue;
@@ -1847,7 +1850,9 @@ next_db(DiskquotaDBEntry *curDB)
 		if (nextSlot >= MAX_NUM_MONITORED_DB) nextSlot = 0;
 		DiskquotaDBEntry *dbEntry = &DiskquotaLauncherShmem->dbArray[nextSlot];
 		nextSlot++;
-		if (!dbEntry->in_use || dbEntry->workerId != INVALID_WORKER_ID || dbEntry->dbid == InvalidOid) continue;
+		if (!dbEntry->in_use || dbEntry->workerId != INVALID_WORKER_ID || dbEntry->dbid == InvalidOid ||
+		    dbEntry->corrupted)
+			continue;
 		/* TODO: should release the invalid db related things */
 		if (!is_valid_dbid(dbEntry->dbid)) continue;
 		result = dbEntry;
@@ -1911,10 +1916,11 @@ static void
 vacuum_db_entry(DiskquotaDBEntry *db)
 {
 	if (db == NULL) return;
-	db->dbid     = InvalidOid;
-	db->inited   = false;
-	db->workerId = INVALID_WORKER_ID;
-	db->in_use   = false;
+	db->dbid      = InvalidOid;
+	db->inited    = false;
+	db->workerId  = INVALID_WORKER_ID;
+	db->in_use    = false;
+	db->corrupted = false;
 }
 
 static void
@@ -1949,6 +1955,18 @@ free_bgworker_handle(uint32 worker_id)
 	}
 }
 
+static void
+resetBackgroundWorkerCorruption(void)
+{
+	LWLockAcquire(diskquota_locks.dblist_lock, LW_EXCLUSIVE);
+	for (int i = 0; i < MAX_NUM_MONITORED_DB; i++)
+	{
+		DiskquotaDBEntry *dbEntry = &DiskquotaLauncherShmem->dbArray[i];
+		if (dbEntry->corrupted) dbEntry->corrupted = false;
+	}
+	LWLockRelease(diskquota_locks.dblist_lock);
+}
+
 #if GP_VERSION_NUM < 70000
 static BgwHandleStatus
 WaitForBackgroundWorkerShutdown(BackgroundWorkerHandle *handle)
diff --git a/src/diskquota.h b/src/diskquota.h
index 8e46d317..c218148e 100644
--- a/src/diskquota.h
+++ b/src/diskquota.h
@@ -217,6 +217,7 @@ struct DiskquotaDBEntry
 
 	bool inited; // this entry is inited, will set to true after the worker finish the frist run.
 	bool in_use; // this slot is in using. AKA dbid != 0
+	bool corrupted; // consider this entry as invalid to start the worker on
 
 	TimestampTz last_log_time; // the last time log current database info.
 };
@@ -251,7 +252,7 @@ extern void invalidate_database_rejectmap(Oid dbid);
 /* quota model interface*/
 extern void init_disk_quota_shmem(void);
 extern void init_disk_quota_model(uint32 id);
-extern void refresh_disk_quota_model(bool force);
+extern void refresh_disk_quota_model(DiskquotaDBEntry *dbEntry);
 extern bool check_diskquota_state_is_ready(void);
 extern bool quota_check_common(Oid reloid, RelFileNode *relfilenode);
 
diff --git a/src/gp_activetable.c b/src/gp_activetable.c
index a0d1c524..23c144fe 100644
--- a/src/gp_activetable.c
+++ b/src/gp_activetable.c
@@ -378,6 +378,7 @@ gp_fetch_active_tables(bool is_init)
 
 	if (is_init)
 	{
+		SIMPLE_FAULT_INJECTOR("diskquota_worker_initialization");
 		load_table_size(local_table_stats_map);
 	}
 	else
diff --git a/src/quotamodel.c b/src/quotamodel.c
index 6b8507b3..a0f01dbd 100644
--- a/src/quotamodel.c
+++ b/src/quotamodel.c
@@ -227,7 +227,7 @@ static void clear_all_quota_maps(void);
 static void transfer_table_for_quota(int64 totalsize, QuotaType type, Oid *old_keys, Oid *new_keys, int16 segid);
 
 /* functions to refresh disk quota model*/
-static void refresh_disk_quota_usage(bool is_init);
+static void refresh_disk_quota_usage(DiskquotaDBEntry *dbEntry);
 static void calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map);
 static void flush_to_table_size(void);
 static bool flush_local_reject_map(void);
@@ -761,8 +761,10 @@ do_check_diskquota_state_is_ready(void)
  * recalculate the changed disk usage.
  */
 void
-refresh_disk_quota_model(bool is_init)
+refresh_disk_quota_model(DiskquotaDBEntry *dbEntry)
 {
+	bool is_init = !dbEntry->inited;
+
 	SEGCOUNT = getgpsegmentCount();
 	if (SEGCOUNT <= 0)
 	{
@@ -773,7 +775,7 @@ refresh_disk_quota_model(bool is_init)
 	/* skip refresh model when load_quotas failed */
 	if (load_quotas())
 	{
-		refresh_disk_quota_usage(is_init);
+		refresh_disk_quota_usage(dbEntry);
 	}
 	if (is_init) ereport(LOG, (errmsg("[diskquota] initialize quota model finished")));
 }
@@ -785,11 +787,12 @@ refresh_disk_quota_model(bool is_init)
  * process is constructing quota model.
  */
 static void
-refresh_disk_quota_usage(bool is_init)
+refresh_disk_quota_usage(DiskquotaDBEntry *dbEntry)
 {
 	bool  connected                   = false;
 	bool  pushed_active_snap          = false;
 	bool  ret                         = true;
+	bool  is_init                     = !dbEntry->inited;
 	HTAB *local_active_table_stat_map = NULL;
 
 	StartTransactionCommand();
@@ -841,6 +844,14 @@ refresh_disk_quota_usage(bool is_init)
 	}
 	PG_CATCH();
 	{
+		/* Initialization failed.  */
+		if (is_init)
+		{
+			LWLockAcquire(diskquota_locks.dblist_lock, LW_EXCLUSIVE);
+			dbEntry->corrupted = true;
+			LWLockRelease(diskquota_locks.dblist_lock);
+			PG_RE_THROW();
+		}
 		/* Prevents interrupts while cleaning up */
 		HOLD_INTERRUPTS();
 		EmitErrorReport();
diff --git a/tests/isolation2/expected/test_worker_init_failure.out b/tests/isolation2/expected/test_worker_init_failure.out
new file mode 100644
index 00000000..75e6b6e0
--- /dev/null
+++ b/tests/isolation2/expected/test_worker_init_failure.out
@@ -0,0 +1,46 @@
+--
+-- Tests for error handling when the worker catches the error during
+-- its first run.
+--
+
+-- Function checking whether worker on given db is up
+CREATE or REPLACE LANGUAGE plpython2u;
+CREATE
+CREATE or REPLACE FUNCTION check_worker_presence(dbname text, wait_time int) RETURNS boolean AS $$ import psutil import time worker_name = 'bgworker: [diskquota] ' + dbname time.sleep(wait_time) for proc in psutil.process_iter(): try: if 'postgres' in proc.name().lower(): for val in proc.cmdline(): if worker_name in val: return True except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): pass return False $$ LANGUAGE plpython2u EXECUTE ON MASTER;
+CREATE
+
+-- Test diskquota behavior when an error occurs during the worker's first run.
+-- The error leads to process termination. And launcher won't start it again
+-- until extension reload or SIGHUP signal.
+CREATE EXTENSION diskquota;
+CREATE
+SELECT check_worker_presence(current_database(), 0);
+ check_worker_presence 
+-----------------------
+ t                     
+(1 row)
+SELECT gp_inject_fault('diskquota_worker_initialization', 'error', dbid) FROM gp_segment_configuration WHERE role='p' AND content=-1;
+ gp_inject_fault 
+-----------------
+ Success:        
+(1 row)
+SELECT diskquota.init_table_size_table();
+ init_table_size_table 
+-----------------------
+                       
+(1 row)
+SELECT check_worker_presence(current_database(), current_setting('diskquota.worker_timeout')::int / 2);
+ check_worker_presence 
+-----------------------
+ f                     
+(1 row)
+-- Reload configuration and check that worker is up again
+!\retcode gpstop -u;
+(exited with code 0)
+SELECT check_worker_presence(current_database(), current_setting('diskquota.worker_timeout')::int / 2);
+ check_worker_presence 
+-----------------------
+ t                     
+(1 row)
+DROP EXTENSION diskquota;
+DROP
diff --git a/tests/isolation2/isolation2_schedule b/tests/isolation2/isolation2_schedule
index 090c5cc5..c61f3d97 100644
--- a/tests/isolation2/isolation2_schedule
+++ b/tests/isolation2/isolation2_schedule
@@ -11,4 +11,5 @@ test: test_per_segment_config
 test: test_relation_cache
 test: test_ereport_from_seg
 test: test_drop_extension
+test: test_worker_init_failure
 test: reset_config
diff --git a/tests/isolation2/sql/test_worker_init_failure.sql b/tests/isolation2/sql/test_worker_init_failure.sql
new file mode 100644
index 00000000..4e48908a
--- /dev/null
+++ b/tests/isolation2/sql/test_worker_init_failure.sql
@@ -0,0 +1,40 @@
+--
+-- Tests for error handling when the worker catches the error during
+-- its first run.
+--
+
+-- Function checking whether worker on given db is up
+CREATE or REPLACE LANGUAGE plpython2u;
+CREATE or REPLACE FUNCTION check_worker_presence(dbname text, wait_time int)
+  RETURNS boolean
+AS $$
+    import psutil
+    import time
+    worker_name = 'bgworker: [diskquota] ' + dbname
+    time.sleep(wait_time)
+    for proc in psutil.process_iter():
+        try:
+            if 'postgres' in proc.name().lower():
+                for val in proc.cmdline():
+                    if worker_name in val:
+                        return True
+        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+            pass
+    return False
+$$ LANGUAGE plpython2u EXECUTE ON MASTER;
+
+-- Test diskquota behavior when an error occurs during the worker's first run.
+-- The error leads to process termination. And launcher won't start it again
+-- until extension reload or SIGHUP signal.
+CREATE EXTENSION diskquota;
+SELECT check_worker_presence(current_database(), 0);
+SELECT gp_inject_fault('diskquota_worker_initialization', 'error', dbid)
+  FROM gp_segment_configuration WHERE role='p' AND content=-1;
+SELECT diskquota.init_table_size_table();
+SELECT check_worker_presence(current_database(),
+  current_setting('diskquota.worker_timeout')::int / 2);
+-- Reload configuration and check that worker is up again
+!\retcode gpstop -u;
+SELECT check_worker_presence(current_database(),
+  current_setting('diskquota.worker_timeout')::int / 2);
+DROP EXTENSION diskquota;

From 16fecbb0db0a18d5a2d1d4dca551f9c22d6ce44e Mon Sep 17 00:00:00 2001
From: Vasiliy Ivanov <ivi@arenadata.io>
Date: Wed, 26 Jul 2023 16:48:24 +0200
Subject: [PATCH 19/19] mute alter extension tests

existing upgrade and alter tests are useless for ADB:
- previous releases incompatible with current due PG-module version check
- our upgrade process exclude existing of previous versions during upgrade
---
 concourse/scripts/test_diskquota.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/concourse/scripts/test_diskquota.sh b/concourse/scripts/test_diskquota.sh
index cf852d9b..cda28178 100755
--- a/concourse/scripts/test_diskquota.sh
+++ b/concourse/scripts/test_diskquota.sh
@@ -40,7 +40,10 @@ function _main() {
     fi
     popd
 
-    time /home/gpadmin/diskquota_src/upgrade_test/alter_test.sh
+    # TODO: existing upgrade and alter tests are useless for ADB
+    # - previous releases incompatible with current on PG-module version check
+    # - our upgrade process exclude existing of previous versions during upgrade
+    # time /home/gpadmin/diskquota_src/upgrade_test/alter_test.sh
 }
 
 _main