diff --git a/CMakeLists.txt b/CMakeLists.txt index 417f3aec..8fe7872c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,7 +97,7 @@ set_target_properties( PROPERTIES OUTPUT_NAME ${DISKQUOTA_BINARY_NAME} PREFIX "" C_STANDARD 99 - LINKER_LANGUAGE "CXX") + LINKER_LANGUAGE "C") # packing part, move to a separate file if this part is too large include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Distro.cmake) @@ -146,7 +146,7 @@ add_custom_target(create_artifact COMMAND ${CMAKE_COMMAND} --build . --target package COMMAND - ${CMAKE_COMMAND} -E tar cvf ${artifact_NAME} "${tgz_NAME}.tar.gz") + ${CMAKE_COMMAND} -E tar czvf ${artifact_NAME} "${tgz_NAME}.tar.gz") # packing end # Create build-info diff --git a/VERSION b/VERSION index 227cea21..38f77a65 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.0 +2.0.1 diff --git a/concourse/README.md b/concourse/README.md index 4106ce39..14492679 100644 --- a/concourse/README.md +++ b/concourse/README.md @@ -2,20 +2,21 @@ ## Naming Prefix Rule -- `PR:` for pull-request pipelines -- `COMMIT::` for branch pipelines. It will be executed when a commit committed/merged into the branch. -- `DEV:_[any_other_info]` for personal development usage. Put your name into the pipeline name so others can know who own it. +- `pr.` for pull-request pipelines +- `merge..` for branch pipelines. It will be executed when a commit committed/merged into the branch. +- `dev...` for personal development usage. Put your name into the pipeline name so others can know who own it. +- `_test..` for pipeline debugging. ## Pipelines for daily work ### PR Pipeline -https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/PR:diskquota +https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/pr.diskquota ### Main Branch Pipeline -The development happens on the `gpdb` branch. The commit pipeline for the `gpdb` -https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/COMMIT:diskquota:gpdb +The development happens on the `gpdb` branch. The merge pipeline for the `gpdb` branch is +https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/merge.diskquota:gpdb # Fly a pipeline @@ -38,10 +39,10 @@ https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/COMMIT:diskquota:gpdb ./fly.sh -t extension -c pr ``` -## Fly the commit pipeline +## Fly the merge pipeline ``` -./fly.sh -t extension -c commit +./fly.sh -t extension -c merge ``` ## Fly the release pipeline @@ -54,24 +55,24 @@ The release pipeline should be located in https://prod.ci.gpdb.pivotal.io # Login to prod fly -t prod login -c https://prod.ci.gpdb.pivotal.io # Fly the release pipeline -./fly.sh -t prod -c release +./fly.sh -t prod -c rel ``` To fly a release pipeline from a specific branch: ``` -./fly.sh -t -c release -b release/. +./fly.sh -t -c rel -b release/. ``` ## Fly the dev pipeline ``` -./fly.sh -t extension -c dev -p _diskquota -b +./fly.sh -t extension -c dev -p -b ``` ## Webhook -By default, the PR and commit pipelines are using webhook instead of polling to trigger a build. The webhook URL will be printed when flying such a pipeline by `fly.sh`. The webhook needs to be set in the `github repository` -> `Settings` -> `Webhooks` with push notification enabled. +By default, the PR and merge pipelines are using webhook instead of polling to trigger a build. The webhook URL will be printed when flying such a pipeline by `fly.sh`. The webhook needs to be set in the `github repository` -> `Settings` -> `Webhooks` with push notification enabled. To test if the webhook works, use `curl` to send a `POST` request to the hook URL with some random data. If it is the right URL, the relevant resource will be refreshed on the Concourse UI. The command line looks like: @@ -83,6 +84,6 @@ curl --data-raw "foo" ## PR pipeline is not triggered. -The PR pipeline relies on the webhook to detect the new PR. However, due the the limitation of the webhook implemention of concourse, we rely on the push hook for this. It means if the PR is from a forked repo, the PR pipeline won't be triggered immediately. To manually trigger the pipeline, go to https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/PR:diskquota/resources/diskquota_pr and click ⟳ button there. +The PR pipeline relies on the webhook to detect the new PR. However, due to the the limitation of the webhook implemention of concourse, we rely on the push hook for this. It means if the PR is from a forked repo, the PR pipeline won't be triggered immediately. To manually trigger the pipeline, go to https://extensions.ci.gpdb.pivotal.io/teams/main/pipelines/pr.diskquota/resources/diskquota_pr and click ⟳ button there. TIPS: Just don't fork, name your branch as `/` and push it here to create PR. diff --git a/concourse/fly.sh b/concourse/fly.sh index d63cb58e..37c3bbe6 100755 --- a/concourse/fly.sh +++ b/concourse/fly.sh @@ -2,25 +2,28 @@ set -e -workspace=${WORKSPACE:-"$HOME/workspace"} fly=${FLY:-"fly"} -echo "'workspace' location: ${workspace}" echo "'fly' command: ${fly}" echo "" +proj_name="diskquota" usage() { - echo "Usage: $0 -t -c [-p ] [-b branch]" 1>&2 if [ -n "$1" ]; then - echo "$1" + echo "$1" 1>&2 + echo "" 1>&2 fi + + echo "Usage: $0 -t -c [-p ] [-b branch] [-T]" + echo "Options:" + echo " '-T' adds '_test' suffix to the pipeline type. Useful for pipeline debugging." exit 1 } # Parse command line options -while getopts ":c:t:p:b:" o; do +while getopts ":c:t:p:b:T" o; do case "${o}" in c) - # pipeline type/config. pr/commit/dev/release + # pipeline type/config. pr/merge/dev/rel pipeline_config=${OPTARG} ;; t) @@ -29,12 +32,15 @@ while getopts ":c:t:p:b:" o; do ;; p) # pipeline name - pipeline_name=${OPTARG} + postfix=${OPTARG} ;; b) # branch name branch=${OPTARG} ;; + T) + test_suffix="_test" + ;; *) usage "" ;; @@ -46,52 +52,66 @@ if [ -z "${target}" ] || [ -z "${pipeline_config}" ]; then usage "" fi +pipeline_type="" # Decide ytt options to generate pipeline case ${pipeline_config} in pr) - if [ -z "${pipeline_name}" ]; then - pipeline_name="PR:diskquota" - fi + pipeline_type="pr" config_file="pr.yml" - hook_res="diskquota_pr" + hook_res="${proj_name}_pr" ;; - commit) - if [ -z "${pipeline_name}" ]; then - pipeline_name="COMMIT:diskquota:gpdb" - fi + merge|commit) # Default branch is 'gpdb' as it is our main branch if [ -z "${branch}" ]; then branch="gpdb" fi + pipeline_type="merge" config_file="commit.yml" - hook_res="diskquota_commit" + hook_res="${proj_name}_commit" ;; dev) - if [ -z "${pipeline_name}" ]; then - usage "'-p' needs to be supplied to specify the pipeline name for flying a 'dev' pipeline." + if [ -z "${postfix}" ]; then + usage "'-p' needs to be supplied to specify the pipeline name postfix for flying a 'dev' pipeline." + fi + if [ -z "${branch}" ]; then + usage "'-b' needs to be supplied to specify the branch for flying a 'dev' pipeline." fi - pipeline_name="DEV:${pipeline_name}" + pipeline_type="dev" config_file="dev.yml" ;; - release) + release|rel) # Default branch is 'gpdb' as it is our main branch if [ -z "${branch}" ]; then branch="gpdb" fi - if [ -z "${pipeline_name}" ]; then - pipeline_name="RELEASE:diskquota:${branch}" - fi + pipeline_type="rel" config_file="release.yml" - hook_res="diskquota_commit" + hook_res="${proj_name}_commit" ;; *) usage "" ;; esac -yml_path="/tmp/diskquota_pipeline.yml" +yml_path="/tmp/${proj_name}.yml" my_path=$(realpath "${BASH_SOURCE[0]}") ytt_base=$(dirname "${my_path}")/pipeline +# pipeline cannot contain '/' +pipeline_name=${pipeline_name/\//"_"} + +# Generate pipeline name +if [ -n "${test_suffix}" ]; then + pipeline_type="${pipeline_type}_test" +fi +pipeline_name="${pipeline_type}.${proj_name}" +if [ -n "${branch}" ]; then + pipeline_name="${pipeline_name}.${branch}" +fi +if [ -n "${postfix}" ]; then + pipeline_name="${pipeline_name}.${postfix}" +fi +# pipeline cannot contain '/' +pipeline_name=${pipeline_name/\//"_"} ytt --data-values-file "${ytt_base}/res_def.yml" \ -f "${ytt_base}/base.lib.yml" \ @@ -108,10 +128,7 @@ set -v sp \ -p "${pipeline_name}" \ -c "${yml_path}" \ - -l "${workspace}/gp-continuous-integration/secrets/gpdb_common-ci-secrets.yml" \ - -l "${workspace}/gp-continuous-integration/secrets/gp-extensions-common.yml" \ - -l "${workspace}/gp-continuous-integration/secrets/gpdb_6X_STABLE-ci-secrets.prod.yml" \ - -v "diskquota-branch=${branch}" + -v "${proj_name}-branch=${branch}" set +v if [ "${pipeline_config}" == "dev" ]; then diff --git a/concourse/pipeline/job_def.lib.yml b/concourse/pipeline/job_def.lib.yml index 76798f09..c3d372da 100644 --- a/concourse/pipeline/job_def.lib.yml +++ b/concourse/pipeline/job_def.lib.yml @@ -1,6 +1,14 @@ #@ load("base.lib.yml", "add_res_by_conf", "add_res_by_name") #@ load("@ytt:template", "template") +#@ def inter_bin_name(base_name, release_build): +#@ if release_build: +#@ return base_name + "_rel" +#@ end +#@ +#@ return base_name +#@ end + #! Job config for centos6 #! Use bin_gpdb_postfix="" to use a release version of gpdb binary #@ def centos6_gpdb6_conf(release_build=False): @@ -8,7 +16,7 @@ res_build_image: centos6-gpdb6-image-build res_test_image: centos6-gpdb6-image-test res_gpdb_bin: #@ "bin_gpdb6_centos6" + ("" if release_build else "_debug") res_diskquota_bin: bin_diskquota_gpdb6_rhel6 -res_intermediates_bin: bin_diskquota_gpdb6_rhel6_intermediates +res_intermediates_bin: #@ inter_bin_name("bin_diskquota_gpdb6_rhel6_intermediates", release_build) release_bin: bin_diskquota_gpdb6_rhel6_release os: rhel6 build_type: #@ "Release" if release_build else "Debug" @@ -20,7 +28,7 @@ res_build_image: centos7-gpdb6-image-build res_test_image: centos7-gpdb6-image-test res_gpdb_bin: #@ "bin_gpdb6_centos7" + ("" if release_build else "_debug") res_diskquota_bin: bin_diskquota_gpdb6_rhel7 -res_intermediates_bin: bin_diskquota_gpdb6_rhel7_intermediates +res_intermediates_bin: #@ inter_bin_name("bin_diskquota_gpdb6_rhel7_intermediates", release_build) release_bin: bin_diskquota_gpdb6_rhel7_release os: rhel7 build_type: #@ "Release" if release_build else "Debug" @@ -32,7 +40,7 @@ res_build_image: rhel8-gpdb6-image-build res_test_image: rhel8-gpdb6-image-test res_gpdb_bin: #@ "bin_gpdb6_rhel8" + ("" if release_build else "_debug") res_diskquota_bin: bin_diskquota_gpdb6_rhel8 -res_intermediates_bin: bin_diskquota_gpdb6_rhel8_intermediates +res_intermediates_bin: #@ inter_bin_name("bin_diskquota_gpdb6_rhel8_intermediates", release_build) release_bin: bin_diskquota_gpdb6_rhel8_release os: rhel8 build_type: #@ "Release" if release_build else "Debug" @@ -44,7 +52,7 @@ res_build_image: ubuntu18-gpdb6-image-build res_test_image: ubuntu18-gpdb6-image-test res_gpdb_bin: #@ "bin_gpdb6_ubuntu18" + ("" if release_build else "_debug") res_diskquota_bin: bin_diskquota_gpdb6_ubuntu18 -res_intermediates_bin: bin_diskquota_gpdb6_ubuntu18_intermediates +res_intermediates_bin: #@ inter_bin_name("bin_diskquota_gpdb6_ubuntu18_intermediates", release_build) release_bin: bin_diskquota_gpdb6_ubuntu18_release os: ubuntu18.04 build_type: #@ "Release" if release_build else "Debug" @@ -153,9 +161,12 @@ plan: #@ end - in_parallel: steps: -#@ for conf in confs: +#@ for i in range(len(confs)): +#@ conf = confs[i] - do: - get: #@ conf["res_intermediates_bin"] + passed: + - #@ passed_jobs[i] params: unpack: true - put: #@ conf["release_bin"] diff --git a/concourse/pipeline/res_def.yml b/concourse/pipeline/res_def.yml index 55411888..7f9ccfe0 100644 --- a/concourse/pipeline/res_def.yml +++ b/concourse/pipeline/res_def.yml @@ -21,23 +21,23 @@ resources: - name: diskquota_pr type: pull-request # We should rely on the webhook. See README if webhook doesn't work - webhook_token: ((diskquota-webhook-token)) + webhook_token: ((extension/diskquota-webhook-token)) check_every: 24h source: disable_forks: false repository: greenplum-db/diskquota - access_token: ((github-access-token)) + access_token: ((extension/github-access-token)) base_branch: gpdb # Commit trigger - name: diskquota_commit type: git # We should rely on the webhook. See README if webhook doesn't work - webhook_token: ((diskquota-webhook-token)) + webhook_token: ((extension/diskquota-webhook-token)) check_every: 1h source: branch: ((diskquota-branch)) uri: https://github.com/greenplum-db/diskquota.git - username: ((github-access-token)) + username: ((extension/github-access-token)) password: x-oauth-basic # Commit dev trigger. Not using webhook - name: diskquota_commit_dev @@ -46,7 +46,7 @@ resources: source: branch: ((diskquota-branch)) uri: https://github.com/greenplum-db/diskquota.git - username: ((github-access-token)) + username: ((extension/github-access-token)) password: x-oauth-basic @@ -87,14 +87,14 @@ resources: repository: gcr.io/data-gpdb-private-images/gpdb6-rhel8-build tag: latest username: _json_key - password: ((container-registry-readonly-service-account-key)) + password: ((data-gpdb-private-images-container-registry-readonly-service-account-key)) - name: rhel8-gpdb6-image-test type: registry-image source: repository: gcr.io/data-gpdb-private-images/gpdb6-rhel8-test tag: latest username: _json_key - password: ((container-registry-readonly-service-account-key)) + password: ((data-gpdb-private-images-container-registry-readonly-service-account-key)) # Ubuntu18 - name: ubuntu18-gpdb6-image-build type: registry-image @@ -113,7 +113,7 @@ resources: repository: gcr.io/data-gpdb-extensions/common/clang-format tag: 13 username: _json_key - password: ((extensions-gcs-service-account-key)) + password: ((extension/extensions-gcs-service-account-key)) # gpdb binary on gcs is located as different folder for different version # Latest build with assertion enabled: @@ -121,25 +121,25 @@ resources: - name: bin_gpdb6_centos6_debug type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: server/published/gpdb6/server-rc-(.*)-rhel6_x86_64.debug.tar.gz - name: bin_gpdb6_centos7_debug type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: server/published/gpdb6/server-rc-(.*)-rhel7_x86_64.debug.tar.gz - name: bin_gpdb6_rhel8_debug type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: server/published/gpdb6/server-rc-(.*)-rhel8_x86_64.debug.tar.gz - name: bin_gpdb6_ubuntu18_debug type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64.debug.tar.gz # Latest release candidates, no fault-injector, no assertion: @@ -147,25 +147,25 @@ resources: - name: bin_gpdb6_centos6 type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: server/published/gpdb6/server-rc-(.*)-rhel6_x86_64.tar.gz - name: bin_gpdb6_centos7 type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: server/published/gpdb6/server-rc-(.*)-rhel7_x86_64.tar.gz - name: bin_gpdb6_rhel8 type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: server/published/gpdb6/server-rc-(.*)-rhel8_x86_64.tar.gz - name: bin_gpdb6_ubuntu18 type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64.tar.gz @@ -173,86 +173,116 @@ resources: - name: bin_diskquota_gpdb6_rhel6 type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: diskquota/released/gpdb6/diskquota-(.*)-rhel6_x86_64.tar.gz - name: bin_diskquota_gpdb6_rhel7 type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: diskquota/released/gpdb6/diskquota-(.*)-rhel7_x86_64.tar.gz - name: bin_diskquota_gpdb6_rhel8 type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: diskquota/released/gpdb6/diskquota-(.*)-rhel8_x86_64.tar.gz - name: bin_diskquota_gpdb6_ubuntu18 type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: diskquota/released/gpdb6/diskquota-(.*)-ubuntu18.04_x86_64.tar.gz # For uploading every build to gcs +# Dev - name: bin_diskquota_gpdb6_rhel6_intermediates type: gcs source: bucket: gpdb-extensions-concourse-resources - json_key: ((extensions-gcs-service-account-key)) + json_key: ((extension/extensions-gcs-service-account-key)) versioned_file: intermediates/diskquota/diskquota_rhel6_gpdb6.tar.gz - name: bin_diskquota_gpdb6_rhel7_intermediates type: gcs source: bucket: gpdb-extensions-concourse-resources - json_key: ((extensions-gcs-service-account-key)) + json_key: ((extension/extensions-gcs-service-account-key)) versioned_file: intermediates/diskquota/diskquota_rhel7_gpdb6.tar.gz - name: bin_diskquota_gpdb6_rhel8_intermediates type: gcs source: bucket: gpdb-extensions-concourse-resources - json_key: ((extensions-gcs-service-account-key)) + json_key: ((extension/extensions-gcs-service-account-key)) versioned_file: intermediates/diskquota/diskquota_rhel8_gpdb6.tar.gz - name: bin_diskquota_gpdb6_ubuntu18_intermediates type: gcs source: bucket: gpdb-extensions-concourse-resources - json_key: ((extensions-gcs-service-account-key)) + json_key: ((extension/extensions-gcs-service-account-key)) versioned_file: intermediates/diskquota/diskquota_ubuntu18_gpdb6.tar.gz +# Rel +- name: bin_diskquota_gpdb6_rhel6_intermediates_rel + type: gcs + source: + bucket: gpdb-extensions-concourse-resources + json_key: ((extension/extensions-gcs-service-account-key)) + versioned_file: intermediates_release/diskquota/diskquota_rhel6_gpdb6.tar.gz + +- name: bin_diskquota_gpdb6_rhel7_intermediates_rel + type: gcs + source: + bucket: gpdb-extensions-concourse-resources + json_key: ((extension/extensions-gcs-service-account-key)) + versioned_file: intermediates_release/diskquota/diskquota_rhel7_gpdb6.tar.gz + +- name: bin_diskquota_gpdb6_rhel8_intermediates_rel + type: gcs + source: + bucket: gpdb-extensions-concourse-resources + json_key: ((extension/extensions-gcs-service-account-key)) + versioned_file: intermediates_release/diskquota/diskquota_rhel8_gpdb6.tar.gz + +- name: bin_diskquota_gpdb6_ubuntu18_intermediates_rel + type: gcs + source: + bucket: gpdb-extensions-concourse-resources + json_key: ((extension/extensions-gcs-service-account-key)) + versioned_file: intermediates_release/diskquota/diskquota_ubuntu18_gpdb6.tar.gz + # For uploading to the release bucket - name: bin_diskquota_gpdb6_rhel6_release type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: diskquota/released/gpdb6/diskquota-(.*).tar.gz - name: bin_diskquota_gpdb6_rhel7_release type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: diskquota/released/gpdb6/diskquota-(.*).tar.gz - name: bin_diskquota_gpdb6_rhel8_release type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: diskquota/released/gpdb6/diskquota-(.*).tar.gz - name: bin_diskquota_gpdb6_ubuntu18_release type: gcs source: - bucket: ((gcs-bucket)) + bucket: pivotal-gpdb-concourse-resources-prod json_key: ((concourse-gcs-resources-service-account-key)) regexp: diskquota/released/gpdb6/diskquota-(.*).tar.gz @@ -261,10 +291,10 @@ resources: type: gcs source: bucket: gpdb-extensions-concourse-resources - json_key: ((extensions-gcs-service-account-key)) + json_key: ((extension/extensions-gcs-service-account-key)) regexp: dependencies/cmake-(.*)-linux-x86_64.sh - name: slack_notify type: slack-alert source: - url: ((extensions-slack-webhook)) + url: ((extension/extensions-slack-webhook)) diff --git a/concourse/scripts/build_diskquota.sh b/concourse/scripts/build_diskquota.sh index 312e4989..ed87ced6 100755 --- a/concourse/scripts/build_diskquota.sh +++ b/concourse/scripts/build_diskquota.sh @@ -6,9 +6,9 @@ function pkg() { [ -f /opt/gcc_env.sh ] && source /opt/gcc_env.sh source /usr/local/greenplum-db-devel/greenplum_path.sh - if [ "${DISKQUOTA_OS}" = "rhel6" ]; then - export CC="$(which gcc)" - fi + # Always use the gcc from $PATH, to avoid using a lower version compiler by /usr/bin/cc + export CC="$(which gcc)" + export CXX="$(which g++)" pushd /home/gpadmin/diskquota_artifacts local last_release_path diff --git a/diskquota.c b/diskquota.c index c552acc8..db5fb4a9 100644 --- a/diskquota.c +++ b/diskquota.c @@ -25,6 +25,8 @@ #include "cdb/cdbvars.h" #include "commands/dbcommands.h" #include "executor/spi.h" +#include "libpq/libpq-be.h" +#include "miscadmin.h" #include "port/atomics.h" #include "storage/ipc.h" #include "storage/proc.h" @@ -32,6 +34,7 @@ #include "tcop/utility.h" #include "utils/builtins.h" #include "utils/faultinjector.h" +#include "utils/guc.h" #include "utils/ps_status.h" #include "utils/snapmgr.h" #include "utils/syscache.h" @@ -93,6 +96,48 @@ diskquota_is_paused() return paused; } +bool +diskquota_is_readiness_logged() +{ + Assert(MyDatabaseId != InvalidOid); + bool is_readiness_logged; + + LWLockAcquire(diskquota_locks.worker_map_lock, LW_SHARED); + { + DiskQuotaWorkerEntry *hash_entry; + bool found; + + hash_entry = + (DiskQuotaWorkerEntry *)hash_search(disk_quota_worker_map, (void *)&MyDatabaseId, HASH_FIND, &found); + is_readiness_logged = found ? hash_entry->is_readiness_logged : false; + } + LWLockRelease(diskquota_locks.worker_map_lock); + + return is_readiness_logged; +} + +void +diskquota_set_readiness_logged() +{ + Assert(MyDatabaseId != InvalidOid); + + /* + * We actually need ROW EXCLUSIVE lock here. Given that the current worker + * is the the only process that modifies the entry, it is safe to only take + * the shared lock. + */ + LWLockAcquire(diskquota_locks.worker_map_lock, LW_SHARED); + { + DiskQuotaWorkerEntry *hash_entry; + bool found; + + hash_entry = + (DiskQuotaWorkerEntry *)hash_search(disk_quota_worker_map, (void *)&MyDatabaseId, HASH_FIND, &found); + hash_entry->is_readiness_logged = true; + } + LWLockRelease(diskquota_locks.worker_map_lock); +} + /* functions of disk quota*/ void _PG_init(void); void _PG_fini(void); @@ -257,6 +302,12 @@ disk_quota_worker_main(Datum main_arg) { char *dbname = MyBgworkerEntry->bgw_name; + MyProcPort = (Port *)calloc(1, sizeof(Port)); + MyProcPort->database_name = dbname; /* To show the database in the log */ + + /* Disable ORCA to avoid fallback */ + optimizer = false; + ereport(LOG, (errmsg("[diskquota] start disk quota worker process to monitor database:%s", dbname))); /* Establish signal handlers before unblocking signals. */ @@ -1010,7 +1061,8 @@ worker_create_entry(Oid dbid) { workerentry->handle = NULL; pg_atomic_write_u32(&(workerentry->epoch), 0); - workerentry->is_paused = false; + workerentry->is_paused = false; + workerentry->is_readiness_logged = false; } LWLockRelease(diskquota_locks.worker_map_lock); diff --git a/diskquota.h b/diskquota.h index b86e7e72..9f6a1e30 100644 --- a/diskquota.h +++ b/diskquota.h @@ -137,8 +137,9 @@ typedef struct DiskQuotaWorkerEntry DiskQuotaWorkerEntry; struct DiskQuotaWorkerEntry { Oid dbid; - pg_atomic_uint32 epoch; /* this counter will be increased after each worker loop */ - bool is_paused; /* true if this worker is paused */ + pg_atomic_uint32 epoch; /* this counter will be increased after each worker loop */ + bool is_paused; /* true if this worker is paused */ + bool is_readiness_logged; /* true if we have logged the error message for not ready */ // NOTE: this field only can access in diskquota launcher, in other process it is dangling pointer BackgroundWorkerHandle *handle; @@ -182,6 +183,8 @@ extern Oid diskquota_parse_primary_table_oid(Oid namespace, char *relname); extern bool worker_increase_epoch(Oid database_oid); extern unsigned int worker_get_epoch(Oid database_oid); extern bool diskquota_is_paused(void); -extern void do_check_diskquota_state_is_ready(void); +extern bool do_check_diskquota_state_is_ready(void); +extern bool diskquota_is_readiness_logged(void); +extern void diskquota_set_readiness_logged(void); #endif diff --git a/quotamodel.c b/quotamodel.c index 74f748e6..62d4c9d2 100644 --- a/quotamodel.c +++ b/quotamodel.c @@ -564,8 +564,7 @@ check_diskquota_state_is_ready(void) PushActiveSnapshot(GetTransactionSnapshot()); pushed_active_snap = true; dispatch_my_db_to_all_segments(); - do_check_diskquota_state_is_ready(); - is_ready = true; + is_ready = do_check_diskquota_state_is_ready(); } PG_CATCH(); { @@ -589,7 +588,8 @@ check_diskquota_state_is_ready(void) } /* - * Check whether the diskquota state is ready. Throw an error if it is not. + * Check whether the diskquota state is ready. + * Throw an error or return false if it is not. * * For empty database, table diskquota.state would be ready after * 'CREATE EXTENSION diskquota;'. But for non-empty database, @@ -597,7 +597,7 @@ check_diskquota_state_is_ready(void) * manually to get all the table size information and * store them into table diskquota.table_size */ -void +bool do_check_diskquota_state_is_ready(void) { int ret; @@ -621,14 +621,17 @@ do_check_diskquota_state_is_ready(void) int state; bool isnull; - dat = SPI_getbinval(tup, tupdesc, 1, &isnull); - state = isnull ? DISKQUOTA_UNKNOWN_STATE : DatumGetInt32(dat); + dat = SPI_getbinval(tup, tupdesc, 1, &isnull); + state = isnull ? DISKQUOTA_UNKNOWN_STATE : DatumGetInt32(dat); + bool is_ready = state == DISKQUOTA_READY_STATE; - if (state != DISKQUOTA_READY_STATE) + if (!is_ready && !diskquota_is_readiness_logged()) { - ereport(ERROR, (errmsg("[diskquota] diskquota is not ready"), - errhint("please run 'SELECT diskquota.init_table_size_table();' to initialize diskquota"))); + diskquota_set_readiness_logged(); + ereport(WARNING, (errmsg("[diskquota] diskquota is not ready"), + errhint("please run 'SELECT diskquota.init_table_size_table();' to initialize diskquota"))); } + return is_ready; } /* diff --git a/tests/regress/diskquota_schedule b/tests/regress/diskquota_schedule index 899d60f6..61ccc933 100644 --- a/tests/regress/diskquota_schedule +++ b/tests/regress/diskquota_schedule @@ -1,5 +1,6 @@ test: config test: test_create_extension +test: test_readiness_logged test: test_init_table_size_table test: test_relation_size test: test_relation_cache diff --git a/tests/regress/expected/test_readiness_logged.out b/tests/regress/expected/test_readiness_logged.out new file mode 100644 index 00000000..bd2bfae9 --- /dev/null +++ b/tests/regress/expected/test_readiness_logged.out @@ -0,0 +1,37 @@ +CREATE DATABASE test_readiness_logged; +\c test_readiness_logged +CREATE TABLE t (i int) DISTRIBUTED BY (i); +CREATE EXTENSION diskquota; +WARNING: [diskquota] diskquota is not ready because current database is not empty +HINT: please run 'SELECT diskquota.init_table_size_table();' to initialize diskquota +SELECT pg_sleep(5); --Wait for the check completes + pg_sleep +---------- + +(1 row) + +SELECT count(*) FROM gp_toolkit.gp_log_database +WHERE logmessage = '[diskquota] diskquota is not ready'; + count +------- + 1 +(1 row) + +\! gpstop -raf > /dev/null +\c +SELECT pg_sleep(1); --Wait for the check completes + pg_sleep +---------- + +(1 row) + +SELECT count(*) FROM gp_toolkit.gp_log_database +WHERE logmessage = '[diskquota] diskquota is not ready'; + count +------- + 2 +(1 row) + +DROP EXTENSION diskquota; +\c contrib_regression +DROP DATABASE test_readiness_logged; diff --git a/tests/regress/sql/test_readiness_logged.sql b/tests/regress/sql/test_readiness_logged.sql new file mode 100644 index 00000000..84f75b55 --- /dev/null +++ b/tests/regress/sql/test_readiness_logged.sql @@ -0,0 +1,22 @@ +CREATE DATABASE test_readiness_logged; +\c test_readiness_logged + +CREATE TABLE t (i int) DISTRIBUTED BY (i); + +CREATE EXTENSION diskquota; +SELECT pg_sleep(5); --Wait for the check completes + +SELECT count(*) FROM gp_toolkit.gp_log_database +WHERE logmessage = '[diskquota] diskquota is not ready'; + +\! gpstop -raf > /dev/null +\c +SELECT pg_sleep(1); --Wait for the check completes + +SELECT count(*) FROM gp_toolkit.gp_log_database +WHERE logmessage = '[diskquota] diskquota is not ready'; + +DROP EXTENSION diskquota; + +\c contrib_regression +DROP DATABASE test_readiness_logged; diff --git a/upgrade_test/CMakeLists.txt b/upgrade_test/CMakeLists.txt index be07de9f..79af2781 100644 --- a/upgrade_test/CMakeLists.txt +++ b/upgrade_test/CMakeLists.txt @@ -16,19 +16,26 @@ regresstarget_add( REGRESS_OPTS --dbname=contrib_regression) +exec_program( + git ${CMAKE_SOURCE_DIR} ARGS + tag | sort --version-sort -r | head -n 1 + OUTPUT_VARIABLE latest_tag +) + # check whether DDL file (*.sql) is modified file(GLOB ddl_files ${CMAKE_SOURCE_DIR}/*.sql) foreach(ddl IN LISTS ddl_files) + cmake_path(GET ddl FILENAME ddl) exec_program( - git ARGS - diff --exit-code ${ddl} + git ${CMAKE_SOURCE_DIR} ARGS + diff ${latest_tag} --exit-code ${ddl} OUTPUT_VARIABLE NULL RETURN_VALUE "${ddl}_modified") if("${${ddl}_modified}") message( NOTICE - "detected DDL file ${ddl} is modified, checking if upgrade test is needed." + "compared to ${latest_tag}, the DDL file ${ddl} is modified, checking if upgrade test is needed." ) set(DISKQUOTA_DDL_MODIFIED TRUE) endif()