From adbc356f22dd4e4178d72dbac1e66f66a07befb3 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Fri, 15 Nov 2019 23:06:41 +0100 Subject: [PATCH 1/6] Moved unused *.plsql files to askbot/search/postgres/old --- .../postgresql/{ => old}/question_answer_comment_models.plsql | 0 .../postgresql/{ => old}/thread_and_post_models_01022013.plsql | 0 .../postgresql/{ => old}/thread_and_post_models_01162012.plsql | 0 .../postgresql/{ => old}/thread_and_post_models_05222012.plsql | 0 .../postgresql/{ => old}/thread_and_post_models_27112012.plsql | 0 .../postgresql/{ => old}/user_profile_search_02262013.plsql | 0 .../search/postgresql/{ => old}/user_profile_search_051312.plsql | 0 .../postgresql/{ => old}/user_profile_search_08312012.plsql | 0 .../postgresql/{ => old}/user_profile_search_09262015.plsql | 0 .../postgresql/{ => old}/user_profile_search_16102012.plsql | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename askbot/search/postgresql/{ => old}/question_answer_comment_models.plsql (100%) rename askbot/search/postgresql/{ => old}/thread_and_post_models_01022013.plsql (100%) rename askbot/search/postgresql/{ => old}/thread_and_post_models_01162012.plsql (100%) rename askbot/search/postgresql/{ => old}/thread_and_post_models_05222012.plsql (100%) rename askbot/search/postgresql/{ => old}/thread_and_post_models_27112012.plsql (100%) rename askbot/search/postgresql/{ => old}/user_profile_search_02262013.plsql (100%) rename askbot/search/postgresql/{ => old}/user_profile_search_051312.plsql (100%) rename askbot/search/postgresql/{ => old}/user_profile_search_08312012.plsql (100%) rename askbot/search/postgresql/{ => old}/user_profile_search_09262015.plsql (100%) rename askbot/search/postgresql/{ => old}/user_profile_search_16102012.plsql (100%) diff --git a/askbot/search/postgresql/question_answer_comment_models.plsql b/askbot/search/postgresql/old/question_answer_comment_models.plsql similarity index 100% rename from askbot/search/postgresql/question_answer_comment_models.plsql rename to askbot/search/postgresql/old/question_answer_comment_models.plsql diff --git a/askbot/search/postgresql/thread_and_post_models_01022013.plsql b/askbot/search/postgresql/old/thread_and_post_models_01022013.plsql similarity index 100% rename from askbot/search/postgresql/thread_and_post_models_01022013.plsql rename to askbot/search/postgresql/old/thread_and_post_models_01022013.plsql diff --git a/askbot/search/postgresql/thread_and_post_models_01162012.plsql b/askbot/search/postgresql/old/thread_and_post_models_01162012.plsql similarity index 100% rename from askbot/search/postgresql/thread_and_post_models_01162012.plsql rename to askbot/search/postgresql/old/thread_and_post_models_01162012.plsql diff --git a/askbot/search/postgresql/thread_and_post_models_05222012.plsql b/askbot/search/postgresql/old/thread_and_post_models_05222012.plsql similarity index 100% rename from askbot/search/postgresql/thread_and_post_models_05222012.plsql rename to askbot/search/postgresql/old/thread_and_post_models_05222012.plsql diff --git a/askbot/search/postgresql/thread_and_post_models_27112012.plsql b/askbot/search/postgresql/old/thread_and_post_models_27112012.plsql similarity index 100% rename from askbot/search/postgresql/thread_and_post_models_27112012.plsql rename to askbot/search/postgresql/old/thread_and_post_models_27112012.plsql diff --git a/askbot/search/postgresql/user_profile_search_02262013.plsql b/askbot/search/postgresql/old/user_profile_search_02262013.plsql similarity index 100% rename from askbot/search/postgresql/user_profile_search_02262013.plsql rename to askbot/search/postgresql/old/user_profile_search_02262013.plsql diff --git a/askbot/search/postgresql/user_profile_search_051312.plsql b/askbot/search/postgresql/old/user_profile_search_051312.plsql similarity index 100% rename from askbot/search/postgresql/user_profile_search_051312.plsql rename to askbot/search/postgresql/old/user_profile_search_051312.plsql diff --git a/askbot/search/postgresql/user_profile_search_08312012.plsql b/askbot/search/postgresql/old/user_profile_search_08312012.plsql similarity index 100% rename from askbot/search/postgresql/user_profile_search_08312012.plsql rename to askbot/search/postgresql/old/user_profile_search_08312012.plsql diff --git a/askbot/search/postgresql/user_profile_search_09262015.plsql b/askbot/search/postgresql/old/user_profile_search_09262015.plsql similarity index 100% rename from askbot/search/postgresql/user_profile_search_09262015.plsql rename to askbot/search/postgresql/old/user_profile_search_09262015.plsql diff --git a/askbot/search/postgresql/user_profile_search_16102012.plsql b/askbot/search/postgresql/old/user_profile_search_16102012.plsql similarity index 100% rename from askbot/search/postgresql/user_profile_search_16102012.plsql rename to askbot/search/postgresql/old/user_profile_search_16102012.plsql From 0139d46ebfa07dc7b4e8b3d481d0f3e2e7a44f2d Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Sat, 16 Nov 2019 09:39:36 +0100 Subject: [PATCH 2/6] Immediately use most recent pgplsql scripts * Before this commit the migrations used two different versions of thread_and_post_models and two different versions of user_profile_search. - The diff between these scripts is minimal and I cannot see the advantage of first applying the old versions and virtually immediately afterwards the new version. - The difference between the versions appears irrelevant for the migrations that happen between applying the old version and applying the new version. => drop old version and apply new version right away * fixed migration dependencies because two migrations were removed --- askbot/migrations/0004_auto_20151219_0751.py | 4 +- askbot/migrations/0007_auto_20151220_0653.py | 37 ------------------- askbot/migrations/0008_auto_20160101_0951.py | 3 +- askbot/migrations/0009_auto_20160103_1150.py | 28 -------------- ...te_language_code_for_reps_20160108_1052.py | 2 +- 5 files changed, 5 insertions(+), 69 deletions(-) delete mode 100644 askbot/migrations/0007_auto_20151220_0653.py delete mode 100644 askbot/migrations/0009_auto_20160103_1150.py diff --git a/askbot/migrations/0004_auto_20151219_0751.py b/askbot/migrations/0004_auto_20151219_0751.py index 0f7e4b95c2..dd373c9327 100644 --- a/askbot/migrations/0004_auto_20151219_0751.py +++ b/askbot/migrations/0004_auto_20151219_0751.py @@ -13,7 +13,7 @@ def init_postgresql_fts(apps, schema_editor): askbot.get_install_directory(), 'search', 'postgresql', - 'thread_and_post_models_10032013.plsql' + 'thread_and_post_models_03012016.plsql' ) setup_full_text_search(script_path) @@ -21,7 +21,7 @@ def init_postgresql_fts(apps, schema_editor): askbot.get_install_directory(), 'search', 'postgresql', - 'user_profile_search_12192015.plsql' + 'user_profile_search_12202015.plsql' ) setup_full_text_search(script_path) diff --git a/askbot/migrations/0007_auto_20151220_0653.py b/askbot/migrations/0007_auto_20151220_0653.py deleted file mode 100644 index b886432aaf..0000000000 --- a/askbot/migrations/0007_auto_20151220_0653.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- - - -from askbot.search.postgresql import setup_full_text_search -from django.db import models, migrations -import askbot -import os.path - -def init_postgresql_fts(apps, schema_editor): - conn = schema_editor.connection - if hasattr(conn, 'vendor') and conn.vendor == 'postgresql': - script_path = os.path.join( - askbot.get_install_directory(), - 'search', - 'postgresql', - 'thread_and_post_models_10032013.plsql' - ) - setup_full_text_search(script_path) - - script_path = os.path.join( - askbot.get_install_directory(), - 'search', - 'postgresql', - 'user_profile_search_12202015.plsql' - ) - setup_full_text_search(script_path) - - -class Migration(migrations.Migration): - - dependencies = [ - ('askbot', '0006_auto_20151220_0459'), - ] - - operations = [ - migrations.RunPython(init_postgresql_fts) - ] diff --git a/askbot/migrations/0008_auto_20160101_0951.py b/askbot/migrations/0008_auto_20160101_0951.py index 4d869c3a82..8383c7342b 100644 --- a/askbot/migrations/0008_auto_20160101_0951.py +++ b/askbot/migrations/0008_auto_20160101_0951.py @@ -8,7 +8,8 @@ class Migration(migrations.Migration): dependencies = [ - ('askbot', '0007_auto_20151220_0653'), + ('askbot', '0006_auto_20151220_0459'), + ] operations = [ diff --git a/askbot/migrations/0009_auto_20160103_1150.py b/askbot/migrations/0009_auto_20160103_1150.py deleted file mode 100644 index 150a09d6bd..0000000000 --- a/askbot/migrations/0009_auto_20160103_1150.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- - -from askbot.search.postgresql import setup_full_text_search -from django.db import models, migrations -import askbot -import os.path - -def init_postgresql_fts(apps, schema_editor): - conn = schema_editor.connection - if hasattr(conn, 'vendor') and conn.vendor == 'postgresql': - script_path = os.path.join( - askbot.get_install_directory(), - 'search', - 'postgresql', - 'thread_and_post_models_03012016.plsql' - ) - setup_full_text_search(script_path) - -class Migration(migrations.Migration): - - dependencies = [ - ('askbot', '0008_auto_20160101_0951'), - ] - - operations = [ - migrations.RunPython(init_postgresql_fts) - ] - diff --git a/askbot/migrations/0010_populate_language_code_for_reps_20160108_1052.py b/askbot/migrations/0010_populate_language_code_for_reps_20160108_1052.py index 782b9c56ea..b84c641218 100644 --- a/askbot/migrations/0010_populate_language_code_for_reps_20160108_1052.py +++ b/askbot/migrations/0010_populate_language_code_for_reps_20160108_1052.py @@ -76,7 +76,7 @@ def calculate_localized_reps(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('askbot', '0009_auto_20160103_1150'), + ('askbot', '0008_auto_20160101_0951'), ] operations = [ From 9f486c2102bf8d5388be5f3dac3955f69a65b537 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Sat, 16 Nov 2019 09:45:13 +0100 Subject: [PATCH 3/6] Move (now) unused pgplsql scripts to old/ --- .../postgresql/{ => old}/thread_and_post_models_10032013.plsql | 0 .../postgresql/{ => old}/user_profile_search_12192015.plsql | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename askbot/search/postgresql/{ => old}/thread_and_post_models_10032013.plsql (100%) rename askbot/search/postgresql/{ => old}/user_profile_search_12192015.plsql (100%) diff --git a/askbot/search/postgresql/thread_and_post_models_10032013.plsql b/askbot/search/postgresql/old/thread_and_post_models_10032013.plsql similarity index 100% rename from askbot/search/postgresql/thread_and_post_models_10032013.plsql rename to askbot/search/postgresql/old/thread_and_post_models_10032013.plsql diff --git a/askbot/search/postgresql/user_profile_search_12192015.plsql b/askbot/search/postgresql/old/user_profile_search_12192015.plsql similarity index 100% rename from askbot/search/postgresql/user_profile_search_12192015.plsql rename to askbot/search/postgresql/old/user_profile_search_12192015.plsql From 5a2404f8bc5ffcc05444b347698b2f27b38eb469 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Fri, 15 Nov 2019 23:10:10 +0100 Subject: [PATCH 4/6] Update documentation with recent file names --- askbot/doc/source/mysql-to-postgres.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/askbot/doc/source/mysql-to-postgres.rst b/askbot/doc/source/mysql-to-postgres.rst index 1157ff7b20..9b982ff9c2 100644 --- a/askbot/doc/source/mysql-to-postgres.rst +++ b/askbot/doc/source/mysql-to-postgres.rst @@ -82,11 +82,11 @@ Test this by running a search query on the askbot site. .. If you have an issue with the above command, it is possible to run the search setup sql script manually: - 1. Download `thread_and_post_models_10032013.plsql `_ - 2. Download `user_profile_search_08312012.plsql `_ + 1. Download `thread_and_post_models_03012016.plsql `_ + 2. Download `user_profile_search_12202015.plsql `_ 3. Apply the scripts to your postgres database:: psql your_database < thread_and_post_models_10032013.plsql - psql your_database < user_profile_search_08312012.plsql + psql your_database < user_profile_search_12202015.plsql Fixing data types From 5535ec9e9646b7c7dd78dfc25f0e37f46b6b9591 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Sat, 16 Nov 2019 10:02:46 +0100 Subject: [PATCH 5/6] Fix whitespace in *.plsql --- .../thread_and_post_models_03012016.plsql | 22 +++++++++---------- .../user_profile_search_12202015.plsql | 12 +++++----- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/askbot/search/postgresql/thread_and_post_models_03012016.plsql b/askbot/search/postgresql/thread_and_post_models_03012016.plsql index 61951dc880..6a99a19efa 100644 --- a/askbot/search/postgresql/thread_and_post_models_03012016.plsql +++ b/askbot/search/postgresql/thread_and_post_models_03012016.plsql @@ -1,14 +1,14 @@ /* function testing for existence of a column in a table if table does not exists, function will return "false" */ CREATE OR REPLACE FUNCTION column_exists(colname text, tablename text) -RETURNS boolean AS +RETURNS boolean AS $$ DECLARE q text; onerow record; BEGIN - q = 'SELECT attname FROM pg_attribute WHERE attrelid = ( SELECT oid FROM pg_class WHERE relname = '''||tablename||''') AND attname = '''||colname||''''; + q = 'SELECT attname FROM pg_attribute WHERE attrelid = ( SELECT oid FROM pg_class WHERE relname = '''||tablename||''') AND attname = '''||colname||''''; FOR onerow IN EXECUTE q LOOP RETURN true; @@ -181,7 +181,7 @@ DECLARE onerow record; BEGIN query = 'SELECT concat_tsvectors(text_search_vector), count(*) FROM askbot_post' || - ' WHERE parent_id=' || parent_id || + ' WHERE parent_id=' || parent_id || ' AND post_type=''comment'' AND deleted=false'; FOR onerow IN EXECUTE query LOOP IF onerow.count = 0 THEN @@ -259,7 +259,7 @@ BEGIN RETURN new; END; $$ LANGUAGE plpgsql; -CREATE TRIGGER thread_search_vector_update_trigger +CREATE TRIGGER thread_search_vector_update_trigger BEFORE UPDATE ON askbot_thread FOR EACH ROW EXECUTE PROCEDURE thread_update_trigger(); CREATE OR REPLACE FUNCTION thread_insert_trigger() RETURNS trigger AS @@ -283,7 +283,7 @@ BEGIN new.post_type, new.language_code ); - UPDATE askbot_thread SET text_search_vector=text_search_vector || + UPDATE askbot_thread SET text_search_vector=text_search_vector || new.text_search_vector WHERE id=new.thread_id; ELSIF new.post_type = 'comment' THEN new.text_search_vector = get_post_tsv( @@ -297,14 +297,14 @@ BEGIN return new; END; $$ LANGUAGE plpgsql; -CREATE TRIGGER post_search_vector_insert_trigger - BEFORE INSERT ON askbot_post - FOR EACH ROW +CREATE TRIGGER post_search_vector_insert_trigger + BEFORE INSERT ON askbot_post + FOR EACH ROW EXECUTE PROCEDURE post_trigger(); -CREATE TRIGGER post_search_vector_update_trigger - BEFORE UPDATE ON askbot_post - FOR EACH ROW +CREATE TRIGGER post_search_vector_update_trigger + BEFORE UPDATE ON askbot_post + FOR EACH ROW WHEN ( (old.text IS DISTINCT FROM new.text) OR (old.text_search_vector IS DISTINCT FROM new.text_search_vector) diff --git a/askbot/search/postgresql/user_profile_search_12202015.plsql b/askbot/search/postgresql/user_profile_search_12202015.plsql index 6c6bd613e6..5bd44995cf 100644 --- a/askbot/search/postgresql/user_profile_search_12202015.plsql +++ b/askbot/search/postgresql/user_profile_search_12202015.plsql @@ -1,9 +1,9 @@ -/* +/* Script depends on functions defined for general askbot full text search. to_tsvector(), add_tsvector_column() calculates text search vector for the user profile -the searched fields are: +the searched fields are: 1) user name 2) user profile 3) group names - for groups to which user belongs @@ -40,8 +40,8 @@ BEGIN 'INNER JOIN askbot_userprofile AS p ON id=p.auth_user_ptr_id ' || 'WHERE id=' || user_id; FOR onerow in EXECUTE user_query LOOP - tsv = tsv || - to_tsvector(onerow.username) || + tsv = tsv || + to_tsvector(onerow.username) || to_tsvector(onerow.real_name) || to_tsvector(onerow.email); END LOOP; @@ -69,7 +69,7 @@ $$ LANGUAGE plpgsql; DROP TRIGGER IF EXISTS auth_user_tsv_update_trigger ON auth_user; CREATE TRIGGER auth_user_tsv_update_trigger -BEFORE INSERT OR UPDATE ON auth_user +BEFORE INSERT OR UPDATE ON auth_user FOR EACH ROW EXECUTE PROCEDURE auth_user_tsv_update_handler(); /* localized user profile trigger */ @@ -132,7 +132,7 @@ AFTER INSERT OR DELETE ON auth_user_groups FOR EACH ROW EXECUTE PROCEDURE group_membership_tsv_update_handler(); -/* todo: whenever group name changes - also +/* todo: whenever group name changes - also * reindex users belonging to the group */ DROP INDEX IF EXISTS auth_user_search_idx; From 8c3739e2bde4716169c139e126982bd6bf913487 Mon Sep 17 00:00:00 2001 From: Martin BTS Date: Sat, 16 Nov 2019 10:03:36 +0100 Subject: [PATCH 6/6] Fix issue #845 * https://github.com/ASKBOT/askbot-devel/issues/845 * setup_aggregates() used to conditionally DROP AGGREGATE concat_tsvectors(tsvector); - the condition SELECTs a field from an internal table that was removed in Postgresql 11, consequently the function definition, therefore the script, therefore the django migration fails - the problematic code section is a custom implementation of DROP AGGREGATE IF EXISTS - DROP AGGREGATE IF EXSITS has been supported (at least) since Postgres 8.2.23, released 2006 => replaced custom implementation with DROP AGGREGATE IF EXISTS * this patch removes the reference to the changed internal Postgresql table and therefore the issue which makes the script fail using Postgresql 11 --- .../search/postgresql/thread_and_post_models_03012016.plsql | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/askbot/search/postgresql/thread_and_post_models_03012016.plsql b/askbot/search/postgresql/thread_and_post_models_03012016.plsql index 6a99a19efa..14f8be817d 100644 --- a/askbot/search/postgresql/thread_and_post_models_03012016.plsql +++ b/askbot/search/postgresql/thread_and_post_models_03012016.plsql @@ -48,12 +48,8 @@ $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION setup_aggregates() RETURNS boolean AS $$ -DECLARE - onerow record; BEGIN - FOR onerow IN SELECT * FROM pg_proc WHERE proname = 'concat_tsvectors' AND proisagg LOOP - DROP AGGREGATE concat_tsvectors(tsvector); - END LOOP; + DROP AGGREGATE IF EXISTS concat_tsvectors(tsvector); CREATE AGGREGATE concat_tsvectors ( BASETYPE = tsvector, SFUNC = tsv_add,