From 0f3c40370f2aa988c583cb3aff92ff24354ea3f8 Mon Sep 17 00:00:00 2001 From: NikitaUnisikhin Date: Wed, 6 Mar 2024 12:36:25 +0000 Subject: [PATCH 1/5] add copy test --- docker/Dockerfile | 1 + docker/bin/setup | 8 ++++ docker/copy/config.conf | 86 ++++++++++++++++++++++++++++++++++++++++ docker/copy/copy_test.sh | 21 ++++++++++ docker/entrypoint.sh | 9 ++++- 5 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 docker/copy/config.conf create mode 100755 docker/copy/copy_test.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index b8cea8963..80d8e16b2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -80,6 +80,7 @@ COPY ./docker/lagpolling /lagpolling COPY ./docker/shell-test /shell-test COPY ./docker/tsa /tsa COPY ./docker/xproto /xproto +COPY ./docker/copy /copy COPY ./docker/entrypoint.sh /usr/local/bin/entrypoint.sh diff --git a/docker/bin/setup b/docker/bin/setup index ca9bfeee1..2830675e9 100755 --- a/docker/bin/setup +++ b/docker/bin/setup @@ -146,6 +146,14 @@ do } done +# Create tables +psql -h localhost -p 5432 -U postgres -d db -c "CREATE TABLE copy_test(c1 VARCHAR(50), c2 VARCHAR(50), c3 VARCHAR(50))" >> $SETUP_LOG 2>&1 || { + echo "ERROR: tables creation failed, examine the log" + cat "$SETUP_LOG" + cat "$PG_LOG" + exit 1 +} + # Create databases for ldap_storage_credentials for database_name in ldap_db1 ldap_db2; do sudo -u postgres createdb $database_name >> "$SETUP_LOG" 2>&1 || { diff --git a/docker/copy/config.conf b/docker/copy/config.conf new file mode 100644 index 000000000..a1ed71121 --- /dev/null +++ b/docker/copy/config.conf @@ -0,0 +1,86 @@ +pid_file "/tmp/odyssey.pid" +daemonize yes + +unix_socket_dir "/tmp" +unix_socket_mode "0644" + +log_format "%p %t %l [%i %s] (%c) %m\n" + +log_to_stdout no + +log_syslog no +log_syslog_ident "odyssey" +log_syslog_facility "daemon" + +log_debug no +log_config yes +log_session yes +log_query no +log_stats yes +stats_interval 60 +log_general_stats_prom yes +log_route_stats_prom no +promhttp_server_port 7777 + +workers "auto" +resolvers 1 + +readahead 8192 + +cache_coroutine 0 + +coroutine_stack_size 16 + +nodelay yes + +keepalive 15 +keepalive_keep_interval 75 +keepalive_probes 9 + +keepalive_usr_timeout 0 + +listen { + host "*" + port 6432 + backlog 128 + compression yes + tls "disable" +} + + +storage "postgres_server" { + type "remote" + host "[localhost]:5432,localhost" + port 5550 +} + +database "db" { + user "postgres" { + authentication "none" + storage "postgres_server" + pool "transaction" + pool_discard no + pool_reserve_prepared_statement yes + client_fwd_error yes + } +} + +storage "local" { + type "local" +} + +database "console" { + user default { + authentication "none" + role "admin" + pool "session" + storage "local" + } +} + + +locks_dir "/tmp/odyssey" + +graceful_die_on_errors yes +enable_online_restart no +bindwith_reuseport yes diff --git a/docker/copy/copy_test.sh b/docker/copy/copy_test.sh new file mode 100755 index 000000000..eb80327a5 --- /dev/null +++ b/docker/copy/copy_test.sh @@ -0,0 +1,21 @@ +#!/bin/bash -x + +set -ex + +/usr/bin/odyssey /copy/config.conf + +(for i in {1..1000}; do +echo "run_id_${i},task_id_${i},Some random ${i}th text"; +done | psql postgresql://postgres@localhost:6432/db -c "COPY copy_test FROM STDIN (FORMAT csv);";) > /dev/null 2>&1 || { + echo "ERROR: COPY BUG" + + cat /var/log/odyssey.log + echo " + + " + cat /var/log/postgresql/postgresql-14-main.log + + exit 1 +} + +ody-stop diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index d2a7c20bf..df3e429b5 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -11,6 +11,13 @@ ody-start /xproto/test.sh ody-stop +# copy +/copy/copy_test.sh +if [ $? -eq 1 ] +then + exit 1 +fi + # odyssey rule-address test /rule-address/test.sh if [ $? -eq 1 ] @@ -86,4 +93,4 @@ ody-start /ody-integration-test ody-stop -teardown \ No newline at end of file +teardown From ee1d39294bf656b2f2232f6318255e2e3540ca91 Mon Sep 17 00:00:00 2001 From: NikitaUnisikhin Date: Wed, 6 Mar 2024 13:42:21 +0000 Subject: [PATCH 2/5] refactoring --- docker/copy/copy_test.sh | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/docker/copy/copy_test.sh b/docker/copy/copy_test.sh index eb80327a5..7d54698b4 100755 --- a/docker/copy/copy_test.sh +++ b/docker/copy/copy_test.sh @@ -2,12 +2,25 @@ set -ex +test_successful() { + (for i in {1..1000}; do + echo "run_id_${i},task_id_${i},Some random ${i}th text"; + done | psql postgresql://postgres@localhost:6432/db -c "COPY copy_test FROM STDIN (FORMAT csv);";) > /dev/null 2>&1 || echo 1 + echo 0 +} + /usr/bin/odyssey /copy/config.conf +with_pstmts_test_successful=$(test_successful) +ody-stop -(for i in {1..1000}; do -echo "run_id_${i},task_id_${i},Some random ${i}th text"; -done | psql postgresql://postgres@localhost:6432/db -c "COPY copy_test FROM STDIN (FORMAT csv);";) > /dev/null 2>&1 || { - echo "ERROR: COPY BUG" +sed -i '/pool_reserve_prepared_statement yes/d' /copy/config.conf + +/usr/bin/odyssey /copy/config.conf +without_pstmts_test_successful=$(test_successful) +ody-stop + +if [ $with_pstmts_test_successful -eq 1 -a $without_pstmts_test_successful -eq 0 ]; then { + echo "ERROR: copy bug when pool_reserve_prepared_statement setting to yes" cat /var/log/odyssey.log echo " @@ -16,6 +29,16 @@ done | psql postgresql://postgres@localhost:6432/db -c "COPY copy_test FROM STDI cat /var/log/postgresql/postgresql-14-main.log exit 1 -} +} fi -ody-stop +if [ $with_pstmts_test_successful -eq 1 -o $without_pstmts_test_successful -eq 1 ]; then { + echo "ERROR: copy bug" + + cat /var/log/odyssey.log + echo " + + " + cat /var/log/postgresql/postgresql-14-main.log + + exit 1 +} fi From 0e8a90ad000b8f9cec6cc85ccd81b90c3642184f Mon Sep 17 00:00:00 2001 From: NikitaUnisikhin Date: Wed, 6 Mar 2024 14:47:29 +0000 Subject: [PATCH 3/5] fix test --- docker/copy/copy_test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/copy/copy_test.sh b/docker/copy/copy_test.sh index 7d54698b4..ab50f7566 100755 --- a/docker/copy/copy_test.sh +++ b/docker/copy/copy_test.sh @@ -5,7 +5,10 @@ set -ex test_successful() { (for i in {1..1000}; do echo "run_id_${i},task_id_${i},Some random ${i}th text"; - done | psql postgresql://postgres@localhost:6432/db -c "COPY copy_test FROM STDIN (FORMAT csv);";) > /dev/null 2>&1 || echo 1 + done | psql postgresql://postgres@localhost:6432/db -c "COPY copy_test FROM STDIN (FORMAT csv);";) > /dev/null 2>&1 || { + echo 1 + return + } echo 0 } From 2ecd5bff5f5ba24b4cbc17d534863934144019bd Mon Sep 17 00:00:00 2001 From: NikitaUnisikhin Date: Mon, 11 Mar 2024 13:20:57 +0000 Subject: [PATCH 4/5] fix entrypoint --- docker/entrypoint.sh | 80 -------------------------------------------- 1 file changed, 80 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index df3e429b5..27759d32e 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -6,11 +6,6 @@ cd /test_dir/test && /usr/bin/odyssey_test setup -# proto -ody-start -/xproto/test.sh -ody-stop - # copy /copy/copy_test.sh if [ $? -eq 1 ] @@ -18,79 +13,4 @@ then exit 1 fi -# odyssey rule-address test -/rule-address/test.sh -if [ $? -eq 1 ] -then - exit 1 -fi - -# odyssey target session attrs test -/tsa/tsa.sh -if [ $? -eq 1 ] -then - exit 1 -fi - -ody-start -/config-validation -ody-stop - -#ldap -/ldap/test_ldap.sh -if [ $? -eq 1 ] -then - exit 1 -fi - -# scram -/scram/test_scram.sh -if [ $? -eq 1 ] -then - exit 1 -fi - -# auth query -/auth_query/test_auth_query.sh -if [ $? -eq 1 ] -then - exit 1 -fi - -# odyssey hba test -/hba/test.sh -if [ $? -eq 1 ] -then - exit 1 -fi - -#prepared statements in transaction pooling -/usr/bin/odyssey /etc/odyssey/pstmts.conf -sleep 1 -/pstmts-test - -ody-stop - -# lag polling -/lagpolling/test-lag.sh -if [ $? -eq 1 ] -then - exit 1 -fi - -/usr/bin/odyssey-asan /etc/odyssey/odyssey.conf -ody-stop - -# TODO: rewrite -#/shell-test/test.sh -/shell-test/console_role_test.sh -/shell-test/parse_pg_options_test.sh -/shell-test/override_pg_options_test.sh -/shell-test/pool_size_test.sh -ody-stop - -ody-start -/ody-integration-test -ody-stop - teardown From 8c371e6c8f966e5838e507a2321fe6d19913c5b6 Mon Sep 17 00:00:00 2001 From: NikitaUnisikhin Date: Mon, 18 Mar 2024 12:02:23 +0000 Subject: [PATCH 5/5] rebase master --- docker/entrypoint.sh | 80 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 27759d32e..df3e429b5 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -6,6 +6,11 @@ cd /test_dir/test && /usr/bin/odyssey_test setup +# proto +ody-start +/xproto/test.sh +ody-stop + # copy /copy/copy_test.sh if [ $? -eq 1 ] @@ -13,4 +18,79 @@ then exit 1 fi +# odyssey rule-address test +/rule-address/test.sh +if [ $? -eq 1 ] +then + exit 1 +fi + +# odyssey target session attrs test +/tsa/tsa.sh +if [ $? -eq 1 ] +then + exit 1 +fi + +ody-start +/config-validation +ody-stop + +#ldap +/ldap/test_ldap.sh +if [ $? -eq 1 ] +then + exit 1 +fi + +# scram +/scram/test_scram.sh +if [ $? -eq 1 ] +then + exit 1 +fi + +# auth query +/auth_query/test_auth_query.sh +if [ $? -eq 1 ] +then + exit 1 +fi + +# odyssey hba test +/hba/test.sh +if [ $? -eq 1 ] +then + exit 1 +fi + +#prepared statements in transaction pooling +/usr/bin/odyssey /etc/odyssey/pstmts.conf +sleep 1 +/pstmts-test + +ody-stop + +# lag polling +/lagpolling/test-lag.sh +if [ $? -eq 1 ] +then + exit 1 +fi + +/usr/bin/odyssey-asan /etc/odyssey/odyssey.conf +ody-stop + +# TODO: rewrite +#/shell-test/test.sh +/shell-test/console_role_test.sh +/shell-test/parse_pg_options_test.sh +/shell-test/override_pg_options_test.sh +/shell-test/pool_size_test.sh +ody-stop + +ody-start +/ody-integration-test +ody-stop + teardown