From d32c9eec30ab60650a6836a55f5265af9bf0c06c Mon Sep 17 00:00:00 2001 From: Sergio Gutierrez Villalba Date: Tue, 1 Oct 2024 14:04:28 +0200 Subject: [PATCH 1/3] refactor(files): drop no longer useful indexes related to the name --- .../20241001112638-drop-name-indexes-files.js | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 migrations/20241001112638-drop-name-indexes-files.js diff --git a/migrations/20241001112638-drop-name-indexes-files.js b/migrations/20241001112638-drop-name-indexes-files.js new file mode 100644 index 00000000..cecbee29 --- /dev/null +++ b/migrations/20241001112638-drop-name-indexes-files.js @@ -0,0 +1,36 @@ +'use strict'; + +const indexName = 'files_status_index'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.sequelize.query( + ` + DROP INDEX IF EXISTS files_status_index; + DROP INDEX IF EXISTS files_name_type_folderid_deleted_unique; + DROP INDEX IF EXISTS files_plainname_type_folderid_deleted_key; + `, + ); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.sequelize.query( + `CREATE INDEX CONCURRENTLY ${indexName} ON files (status)`, + ); + await queryInterface.addIndex('files', ['name', 'type', 'folder_id'], { + name: 'files_name_type_folderid_deleted_unique', + unique: true, + where: { deleted: { [Sequelize.Op.eq]: false } }, + }); + await queryInterface.addIndex( + 'files', + ['plain_name', 'type', 'folder_id'], + { + name: 'files_plainname_type_folderid_deleted_key', + unique: true, + where: { deleted: { [Sequelize.Op.eq]: false } }, + }, + ); + }, +}; From 076e427be3457802e41bd7d4d8b979a36ee65261 Mon Sep 17 00:00:00 2001 From: Sergio Gutierrez Villalba Date: Tue, 1 Oct 2024 14:05:02 +0200 Subject: [PATCH 2/3] feat(files): prepare plain_name field to be indexed by UNIQUE when file.status = 'EXISTS' --- ...t-to-remove-plain-name-duplicates-files.js | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 migrations/20241001115355-script-to-remove-plain-name-duplicates-files.js diff --git a/migrations/20241001115355-script-to-remove-plain-name-duplicates-files.js b/migrations/20241001115355-script-to-remove-plain-name-duplicates-files.js new file mode 100644 index 00000000..2471c4d8 --- /dev/null +++ b/migrations/20241001115355-script-to-remove-plain-name-duplicates-files.js @@ -0,0 +1,104 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.sequelize.query( + ` + DO $$ + DECLARE + outer_rec RECORD; + inner_rec RECORD; + new_name TEXT; + suffix INT; + total_duplicates INT; + renamed_count INT := 0; + BEGIN + -- Find and loop over duplicates + FOR outer_rec IN + SELECT plain_name, COALESCE(type, '') AS type, folder_id, user_id + FROM public.files + WHERE status = 'EXISTS' + AND plain_name IS NOT NULL + GROUP BY plain_name, COALESCE(type, ''), folder_id, user_id + HAVING COUNT(*) > 1 + ORDER BY user_id + LOOP + SELECT COUNT(*) + INTO total_duplicates + FROM public.files + WHERE plain_name = outer_rec.plain_name + AND COALESCE(type, '') = outer_rec.type + AND folder_id = outer_rec.folder_id + AND user_id = outer_rec.user_id + AND status = 'EXISTS' + AND plain_name is not null; + + RAISE NOTICE 'Found % duplicates for plain_name: %, type: %, folder_id: %, user_id: %', + total_duplicates, outer_rec.plain_name, outer_rec.type, outer_rec.folder_id, outer_rec.user_id; + + suffix := 1; + + -- Renaming loop + FOR inner_rec IN + SELECT id, plain_name + FROM public.files + WHERE plain_name = outer_rec.plain_name + AND COALESCE(type, '') = outer_rec.type + AND folder_id = outer_rec.folder_id + AND user_id = outer_rec.user_id + AND status = 'EXISTS' + ORDER BY id + LOOP + -- Prepare the new name for the duplicates + new_name := inner_rec.plain_name; + + -- If it is not the first file, then generate a suffix + IF suffix > 1 THEN + -- Try diff suffixes until finding one which is free + LOOP + new_name := inner_rec.plain_name || ' (' || suffix || ')'; + + -- Is the suffix already taken? + EXIT WHEN NOT EXISTS ( + SELECT 1 + FROM public.files + WHERE plain_name = new_name + AND folder_id = outer_rec.folder_id + AND user_id = outer_rec.user_id + AND status = 'EXISTS' + ); + + -- If it is taken, try with the next one + suffix := suffix + 1; + END LOOP; + + -- Update the row with the new name + UPDATE public.files + SET plain_name = new_name + WHERE id = inner_rec.id; + + renamed_count := renamed_count + 1; -- Count renamed items + RAISE NOTICE 'Renamed plain_name from % to %', inner_rec.plain_name, new_name; + END IF; + + -- Inc suffix for the next duplicate + suffix := suffix + 1; + END LOOP; + + RAISE NOTICE 'Total duplicates found: %, Total renamed: %', + total_duplicates, renamed_count; + END LOOP; + + RAISE NOTICE 'Renaming completed. Total renamed: %', renamed_count; + END $$; + `, + ); + }, + + async down(queryInterface, Sequelize) { + /** + * no op + */ + }, +}; From 080afede35cf280dec3258f4b608d15a6b3575a1 Mon Sep 17 00:00:00 2001 From: Sergio Gutierrez Villalba Date: Tue, 1 Oct 2024 14:06:05 +0200 Subject: [PATCH 3/3] feat(files): create unique plain_name index by (plain_name, type, folder_id) where status = 'EXISTS' --- ...nname-folderid-type-exists-unique-files.js | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 migrations/20241001120044-create-index-plainname-folderid-type-exists-unique-files.js diff --git a/migrations/20241001120044-create-index-plainname-folderid-type-exists-unique-files.js b/migrations/20241001120044-create-index-plainname-folderid-type-exists-unique-files.js new file mode 100644 index 00000000..eaf71d6a --- /dev/null +++ b/migrations/20241001120044-create-index-plainname-folderid-type-exists-unique-files.js @@ -0,0 +1,20 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.sequelize.query( + ` + CREATE UNIQUE INDEX CONCURRENTLY files_plainname_type_folderid_exists_unique + ON files USING btree (plain_name, type, folder_id) + WHERE (status = 'EXISTS'); + `, + ); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.sequelize.query( + `DROP INDEX IF EXISTS files_plainname_type_folderid_exists_unique`, + ); + }, +};