Merge pull request #18 from arenadata/2.2.1-sync

Sync 2.2.1 changes

.editorconfig

@@ -20,3 +20,7 @@ indent_size = 4
 indent_style = space
 indent_size = 2
 
+[cmake/**.cmake]
+indent_style = space
+indent_size = 4
+

CMakeLists.txt

@@ -1,5 +1,5 @@
-cmake_minimum_required(VERSION 3.18)
-# file(ARCHIVE_EXTRACT foo) need 3.18
+cmake_minimum_required(VERSION 3.20)
+# cmake_path requires 3.20
 
 project(diskquota)
 
@@ -64,32 +64,11 @@ add_compile_definitions(
   DISKQUOTA_PATCH_VERSION=${DISKQUOTA_PATCH_VERSION}
   DISKQUOTA_BINARY_NAME="${DISKQUOTA_BINARY_NAME}")
 
-list(
-  APPEND
-  diskquota_SRC
-  diskquota.c
-  diskquota_utility.c
-  enforcement.c
-  gp_activetable.c
-  quotamodel.c
-  relation_cache.c
-  monitored_db.c)
-
-list(
-  APPEND
-  diskquota_DDL
-  diskquota.control
-  diskquota--1.0.sql
-  diskquota--1.0--2.0.sql
-  diskquota--1.0.3--2.0.sql
-  diskquota--2.0.sql
-  diskquota--2.0--1.0.sql
-  diskquota--2.1.sql
-  diskquota--2.0--2.1.sql
-  diskquota--2.1--2.0.sql
-  diskquota--2.2.sql
-  diskquota--2.1--2.2.sql
-  diskquota--2.2--2.1.sql)
+set(SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src")
+file(GLOB diskquota_SRC "${SRC_DIR}/*.c")
+
+set(DISKQUOTA_DDL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/control/ddl")
+file(GLOB diskquota_DDL "${DISKQUOTA_DDL_DIR}/*")
 
 add_library(diskquota MODULE ${diskquota_SRC})
 
@@ -171,20 +150,14 @@ BuildInfo_Create(${build_info_PATH}
 
 # Add installcheck targets
 add_subdirectory(tests)
-if(NOT DEFINED ENABLE_UPGRADE_TEST)
-  set(ENABLE_UPGRADE_TEST ON)
-endif()
-if(ENABLE_UPGRADE_TEST)
-  add_subdirectory(upgrade_test)
-endif()
 
 # NOTE: keep install part at the end of file, to overwrite previous binary
 install(PROGRAMS "cmake/install_gpdb_component" DESTINATION ".")
 install(FILES ${diskquota_DDL} DESTINATION "share/postgresql/extension/")
 install(TARGETS diskquota DESTINATION "lib/postgresql/")
 install(FILES ${build_info_PATH} DESTINATION ".")
 
-file(GLOB sql_files RELATIVE ${CMAKE_SOURCE_DIR} diskquota--2.*.sql)
+file(GLOB sql_files RELATIVE ${DISKQUOTA_DDL_DIR} "${DISKQUOTA_DDL_DIR}/diskquota--2.*.sql")
 list(FILTER sql_files EXCLUDE REGEX ".*--.*--.*")
 list(FILTER sql_files EXCLUDE REGEX ".*diskquota--${DISKQUOTA_MAJOR_VERSION}.${DISKQUOTA_MINOR_VERSION}.*")
 foreach(so IN LISTS sql_files)

README.md

@@ -84,7 +84,7 @@ and their disk usage is isolated between databases.
 
 # Development
 
-[cmake](https://cmake.org) (>= 3.18) needs to be installed.
+[cmake](https://cmake.org) (>= 3.20) needs to be installed.
 
 1. Build & install disk quota
 ```

VERSION

@@ -1 +1 @@
-2.2.0
+2.2.1

cmake/Regress.cmake

@@ -2,8 +2,8 @@
 #
 # Usage:
 # RegressTarget_Add(<name>
-#   SQL_DIR <sql_dir>
-#   EXPECTED_DIR <expected_dir>
+#   SQL_DIR <sql_dir> [<sql_dir_2> ...]
+#   EXPECTED_DIR <expected_dir> [<expected_dir_2> ...]
 #   RESULTS_DIR <results_dir>
 #   [INIT_FILE <init_file_1> <init_file_2> ...]
 #   [SCHEDULE_FILE <schedule_file_1> <schedule_file_2> ...]
@@ -12,6 +12,7 @@
 #   [REGRESS_OPTS <opt1> <opt2> ...]
 #   [REGRESS_TYPE isolation2/regress]
 #   [RUN_TIMES <times>]
+#   [EXCLUDE_FAULT_INJECT_TEST <ON/OFF>]
 # )
 # All the file path can be the relative path to ${CMAKE_CURRENT_SOURCE_DIR}.
 # A bunch of diff targets will be created as well for comparing the regress results. The diff
@@ -24,6 +25,13 @@
 #  - regress_show_diff.sh
 #  - regress_loop.sh
 #
+# NOTE: If the input sql file extension is ".in.sql" instead of ".sql", the "@VAR@" in the input
+# file will be replaced by the corresponding cmake VAR before tests are executed.
+#
+# NOTE: The directory that comes later in the SQL_DIR/EXPECTED_DIR list has a higher priory. The
+# test case with the same name will be overwritten by the case that comes after in the directory
+# list.t
+#
 # Example:
 # RegressTarget_Add(installcheck_avro_fmt
 #    REGRESS ${avro_regress_TARGETS}
@@ -43,17 +51,64 @@ function(_PGIsolation2Target_Add working_DIR)
 
     add_custom_target(
         pg_isolation2_regress
+        COMMAND
+        make -C ${PG_SRC_DIR}/src/test/isolation2 install
+        COMMAND
         ${CMAKE_COMMAND} -E copy_if_different
         ${PG_SRC_DIR}/src/test/isolation2/sql_isolation_testcase.py ${working_DIR}
     )
 endfunction()
 
+# Find all tests in the given directory which uses fault injector, and add them to
+# fault_injector_test_list.
+function(_Find_FaultInjector_Tests sql_DIR)
+    if (NOT fault_injector_test_list)
+        set(fault_injector_test_list "" PARENT_SCOPE)
+    endif()
+    set(test_list ${fault_injector_test_list})
+
+    get_filename_component(sql_DIR ${sql_DIR} ABSOLUTE)
+    file(GLOB files "${sql_DIR}/*.sql")
+    foreach(f ${files})
+        set(ret 1)
+        execute_process(
+            COMMAND
+            grep gp_inject_fault ${f}
+            OUTPUT_QUIET
+            RESULT_VARIABLE ret)
+        if(ret EQUAL 0)
+            get_filename_component(test_name ${f} NAME_WE)
+            if (NOT test_name IN_LIST test_list)
+                list(APPEND test_list ${test_name})
+            endif()
+        endif()
+    endforeach()
+
+    set(fault_injector_test_list ${test_list} PARENT_SCOPE)
+endfunction()
+
+# Create symbolic links in the binary dir to input SQL files.
+function(_Link_Test_Files src_DIR dest_DIR suffix)
+    get_filename_component(src_DIR ${src_DIR} ABSOLUTE)
+    file(MAKE_DIRECTORY ${dest_DIR})
+    file(GLOB files "${src_DIR}/*.${suffix}")
+    foreach(f ${files})
+        get_filename_component(file_name ${f} NAME)
+        file(CREATE_LINK ${f} ${dest_DIR}/${file_name} SYMBOLIC)
+    endforeach()
+    file(GLOB files "${src_DIR}/*.in.${suffix}")
+    foreach(f ${files})
+        get_filename_component(file_name ${f} NAME_WE)
+        configure_file(${f} ${dest_DIR}/${file_name}.${suffix})
+    endforeach()
+endfunction()
+
 function(RegressTarget_Add name)
     cmake_parse_arguments(
         arg
         ""
-        "SQL_DIR;EXPECTED_DIR;RESULTS_DIR;DATA_DIR;REGRESS_TYPE;RUN_TIMES"
-        "REGRESS;EXCLUDE;REGRESS_OPTS;INIT_FILE;SCHEDULE_FILE"
+        "RESULTS_DIR;DATA_DIR;REGRESS_TYPE;RUN_TIMES;EXCLUDE_FAULT_INJECT_TEST"
+        "SQL_DIR;EXPECTED_DIR;REGRESS;EXCLUDE;REGRESS_OPTS;INIT_FILE;SCHEDULE_FILE"
         ${ARGN})
     if (NOT arg_EXPECTED_DIR)
         message(FATAL_ERROR
@@ -82,8 +137,25 @@ function(RegressTarget_Add name)
         endif()
     endif()
 
+    # Link input sql files to the build dir
+    foreach(sql_DIR IN LISTS arg_SQL_DIR)
+        _Link_Test_Files(${sql_DIR} ${working_DIR}/sql sql)
+        # Find all tests using fault injector
+        if(arg_EXCLUDE_FAULT_INJECT_TEST)
+            _Find_FaultInjector_Tests(${sql_DIR})
+        endif()
+    endforeach()
+
+    # Link output out files to the build dir
+    foreach(expected_DIR IN LISTS arg_EXPECTED_DIR)
+        _Link_Test_Files(${expected_DIR} ${working_DIR}/expected out)
+    endforeach()
+
     # Set REGRESS test cases
     foreach(r IN LISTS arg_REGRESS)
+        if (arg_EXCLUDE_FAULT_INJECT_TEST AND (r IN_LIST fault_injector_test_list))
+            continue()
+        endif()
         set(regress_arg ${regress_arg} ${r})
     endforeach()
 
@@ -99,17 +171,23 @@ function(RegressTarget_Add name)
     foreach(o IN LISTS arg_EXCLUDE)
         list(APPEND to_exclude ${o})
     endforeach()
+    if(arg_EXCLUDE_FAULT_INJECT_TEST)
+        list(APPEND to_exclude ${fault_injector_test_list})
+    endif()
     if (to_exclude)
         set(exclude_arg "--exclude-tests=${to_exclude}")
         string(REPLACE ";" "," exclude_arg "${exclude_arg}")
         set(regress_opts_arg ${regress_opts_arg} ${exclude_arg})
     endif()
     foreach(o IN LISTS arg_REGRESS_OPTS)
+        # If the fault injection tests are excluded, ignore the --load-extension=gp_inject_fault as
+        # well.
+        if (arg_EXCLUDE_FAULT_INJECT_TEST AND (o MATCHES ".*inject_fault"))
+            continue()
+        endif()
         set(regress_opts_arg ${regress_opts_arg} ${o})
     endforeach()
 
-    get_filename_component(sql_DIR ${arg_SQL_DIR} ABSOLUTE)
-    get_filename_component(expected_DIR ${arg_EXPECTED_DIR} ABSOLUTE)
     get_filename_component(results_DIR ${arg_RESULTS_DIR} ABSOLUTE)
     if (arg_DATA_DIR)
         get_filename_component(data_DIR ${arg_DATA_DIR} ABSOLUTE)
@@ -131,10 +209,6 @@ function(RegressTarget_Add name)
     add_custom_target(
         ${name}
         WORKING_DIRECTORY ${working_DIR}
-        COMMAND rm -f sql
-        COMMAND ln -s ${sql_DIR} sql
-        COMMAND rm -f expected
-        COMMAND ln -s ${expected_DIR} expected
         COMMAND rm -f results
         COMMAND mkdir -p ${results_DIR}
         COMMAND ln -s ${results_DIR} results

concourse/pipeline/job_def.lib.yml

@@ -15,7 +15,7 @@
 res_build_image: centos6-gpdb6-image-build
 res_test_images: [centos6-gpdb6-image-test]
 res_gpdb_bin: #@ "bin_gpdb6_centos6" + ("" if release_build else "_debug")
-#! res_diskquota_bin: bin_diskquota_gpdb6_rhel6
+res_diskquota_bin: bin_diskquota_gpdb6_rhel6
 res_intermediates_bin: #@ inter_bin_name("bin_diskquota_gpdb6_rhel6_intermediates", release_build)
 release_bin: bin_diskquota_gpdb6_rhel6_release
 os: rhel6
@@ -28,7 +28,7 @@ build_type: #@ "Release" if release_build else "Debug"
 res_build_image: centos7-gpdb6-image-build
 res_test_images: [centos7-gpdb6-image-test]
 res_gpdb_bin: #@ "bin_gpdb6_centos7" + ("" if release_build else "_debug")
-#! res_diskquota_bin: bin_diskquota_gpdb6_rhel7
+res_diskquota_bin: bin_diskquota_gpdb6_rhel7
 res_intermediates_bin: #@ inter_bin_name("bin_diskquota_gpdb6_rhel7_intermediates", release_build)
 release_bin: bin_diskquota_gpdb6_rhel7_release
 os: rhel7
@@ -41,7 +41,7 @@ build_type: #@ "Release" if release_build else "Debug"
 res_build_image: rhel8-gpdb6-image-build
 res_test_images: [rhel8-gpdb6-image-test]
 res_gpdb_bin: #@ "bin_gpdb6_rhel8" + ("" if release_build else "_debug")
-#! res_diskquota_bin: bin_diskquota_gpdb6_rhel8
+res_diskquota_bin: bin_diskquota_gpdb6_rhel8
 res_intermediates_bin: #@ inter_bin_name("bin_diskquota_gpdb6_rhel8_intermediates", release_build)
 release_bin: bin_diskquota_gpdb6_rhel8_release
 os: rhel8
@@ -54,7 +54,7 @@ build_type: #@ "Release" if release_build else "Debug"
 res_build_image: ubuntu18-gpdb6-image-build
 res_test_images: [ubuntu18-gpdb6-image-test]
 res_gpdb_bin: #@ "bin_gpdb6_ubuntu18" + ("" if release_build else "_debug")
-#! res_diskquota_bin: bin_diskquota_gpdb6_ubuntu18
+res_diskquota_bin: bin_diskquota_gpdb6_ubuntu18
 res_intermediates_bin: #@ inter_bin_name("bin_diskquota_gpdb6_ubuntu18_intermediates", release_build)
 release_bin: bin_diskquota_gpdb6_ubuntu18_release
 os: ubuntu18.04
@@ -66,8 +66,8 @@ build_type: #@ "Release" if release_build else "Debug"
 #@ def rhel8_gpdb7_conf(release_build=False):
 res_build_image: rocky8-gpdb7-image-build
 res_test_images: [rocky8-gpdb7-image-test, rhel8-gpdb7-image-test]
-res_gpdb_bin: #@ "bin_gpdb7_rhel8" + ("" if release_build else "_debug")
-#! res_diskquota_bin: bin_diskquota_gpdb7_rhel8
+res_gpdb_bin: #@ "bin_gpdb7_el8" + ("" if release_build else "_debug")
+res_diskquota_bin: bin_diskquota_gpdb7_rhel8
 res_intermediates_bin: #@ inter_bin_name("bin_diskquota_gpdb7_rhel8_intermediates", release_build)
 release_bin: bin_diskquota_gpdb7_rhel8_release
 os: rhel8
@@ -228,12 +228,10 @@ plan:
   - get: #@ test_image
 #@   end
   - get: #@ conf["res_gpdb_bin"]
-  #! - get: last_released_diskquota_bin
-  #!   resource: #@ conf["res_diskquota_bin"]
+  - get: last_released_diskquota_bin
+    resource: #@ conf["res_diskquota_bin"]
 - #@ _build_task(conf)
-#@   if conf["build_type"] != "Release":
 - #@ _test_task(conf)
-#@   end
 - put: #@ conf["res_intermediates_bin"]
   params:
     file: diskquota_artifacts/diskquota.tar.gz

concourse/pipeline/res_def.yml

@@ -158,12 +158,12 @@ resources:
     bucket: pivotal-gpdb-concourse-resources-prod
     json_key: ((concourse-gcs-resources-service-account-key))
     regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64.debug.tar.gz
-- name: bin_gpdb7_rhel8_debug
+- name: bin_gpdb7_el8_debug
   type: gcs
   source:
     bucket: pivotal-gpdb-concourse-resources-prod
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/published/main/server-rc-(.*)-rhel8_x86_64.debug.tar.gz
+    regexp: server/published/main/server-rc-(.*)-el8_x86_64.debug.tar.gz
 
 # Latest release candidates, no fault-injector, no assertion:
 # --disable-debug-extensions --disable-tap-tests --enable-ic-proxy
@@ -172,31 +172,31 @@ resources:
   source:
     bucket: pivotal-gpdb-concourse-resources-prod
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.((9[0-8])|([1-8]?\d))\.(.*)-centos6.tar.gz
+    regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-centos6.tar.gz
 - name: bin_gpdb6_centos7
   type: gcs
   source:
     bucket: pivotal-gpdb-concourse-resources-prod
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.((9[0-8])|([1-8]?\d))\.(.*)-centos7.tar.gz
+    regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-centos7.tar.gz
 - name: bin_gpdb6_rhel8
   type: gcs
   source:
     bucket: pivotal-gpdb-concourse-resources-prod
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.((9[0-8])|([1-8]?\d))\.(.*)-rhel8.tar.gz
+    regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-rhel8.tar.gz
 - name: bin_gpdb6_ubuntu18
   type: gcs
   source:
     bucket: pivotal-gpdb-concourse-resources-prod
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.((9[0-8])|([1-8]?\d))\.(.*)-ubuntu18.04.tar.gz
-- name: bin_gpdb7_rhel8
+    regexp: server/release-candidates/gpdb6/greenplum-db-server-6\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-ubuntu18.04.tar.gz
+- name: bin_gpdb7_el8
   type: gcs
   source:
     bucket: pivotal-gpdb-concourse-resources-prod
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/release-candidates/gpdb7/greenplum-db-server-7\.((9[0-8])|([1-8]?\d))\.(.*)-rhel8.tar.gz
+    regexp: server/release-candidates/gpdb7/greenplum-db-server-7\.([0-9]|([1-8][0-9])|(9[0-8]))\..*-dev.*-el8.tar.gz
 
 # Diskquota releases
 - name: bin_diskquota_gpdb6_rhel6

concourse/scripts/build_diskquota.sh

@@ -11,11 +11,8 @@ function pkg() {
     export CXX="$(which g++)"
 
     pushd /home/gpadmin/diskquota_artifacts
-    local last_release_path
-    # last_release_path=$(readlink -e /home/gpadmin/last_released_diskquota_bin/diskquota-*.tar.gz)
     cmake /home/gpadmin/diskquota_src \
         -DCMAKE_BUILD_TYPE="${BUILD_TYPE}"
-        # -DDISKQUOTA_LAST_RELEASE_PATH="${last_release_path}" \
     cmake --build . --target create_artifact
     popd
 }