From 7800cfdeb68f2a9424d226be1d5ff887fabfa29e Mon Sep 17 00:00:00 2001 From: Steve Larson <9larsons@gmail.com> Date: Tue, 25 Jun 2024 07:59:04 -0500 Subject: [PATCH 1/6] Improved performance fetching posts ref https://linear.app/tryghost/issue/ONC-111 - added composite index to posts_tags for post_id,tag_id for faster lookup - added composite index to posts for updated_at; this is commonly used by get helpers on the front end to display data like the latest posts In testing, this provided a very dramatic improvement for simple get helper requests like 'filter="id:-{{post.id}}+tag:sampleTag" limit="3"' which are by default sorted by updated_at desc. I'm not entirely clear why when sorting by published_at we do not need a composite index - so far it doesn't seem to be necessary. This should cover the primary cases for get helpers - the latest posts with a given tag or set of tags. --- .../2024-06-25-12-08-20-add-posts-tags-post-tag-index.js | 4 ++++ ...-25-12-08-45-add-posts-type-status-updated-at-index.js | 4 ++++ ghost/core/core/server/data/schema/schema.js | 8 ++++++-- ghost/core/test/unit/server/data/schema/integrity.test.js | 2 +- 4 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-20-add-posts-tags-post-tag-index.js create mode 100644 ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-45-add-posts-type-status-updated-at-index.js diff --git a/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-20-add-posts-tags-post-tag-index.js b/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-20-add-posts-tags-post-tag-index.js new file mode 100644 index 00000000000..85b44ff025f --- /dev/null +++ b/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-20-add-posts-tags-post-tag-index.js @@ -0,0 +1,4 @@ +// For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253 +const {createAddIndexMigration} = require('../../utils'); + +module.exports = createAddIndexMigration('posts_tags',['post_id','tag_id']); \ No newline at end of file diff --git a/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-45-add-posts-type-status-updated-at-index.js b/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-45-add-posts-type-status-updated-at-index.js new file mode 100644 index 00000000000..2415fe1d447 --- /dev/null +++ b/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-45-add-posts-type-status-updated-at-index.js @@ -0,0 +1,4 @@ +// For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253 +const {createAddIndexMigration} = require('../../utils'); + +module.exports = createAddIndexMigration('posts',['type','status','updated_at']); \ No newline at end of file diff --git a/ghost/core/core/server/data/schema/schema.js b/ghost/core/core/server/data/schema/schema.js index d63f74febfb..c871359fc6d 100644 --- a/ghost/core/core/server/data/schema/schema.js +++ b/ghost/core/core/server/data/schema/schema.js @@ -94,7 +94,8 @@ module.exports = { newsletter_id: {type: 'string', maxlength: 24, nullable: true, references: 'newsletters.id'}, show_title_and_feature_image: {type: 'boolean', nullable: false, defaultTo: true}, '@@INDEXES@@': [ - ['type','status'] + ['type','status'], + ['type','status','updated_at'] ], '@@UNIQUE_CONSTRAINTS@@': [ ['slug', 'type'] @@ -298,7 +299,10 @@ module.exports = { id: {type: 'string', maxlength: 24, nullable: false, primary: true}, post_id: {type: 'string', maxlength: 24, nullable: false, references: 'posts.id'}, tag_id: {type: 'string', maxlength: 24, nullable: false, references: 'tags.id'}, - sort_order: {type: 'integer', nullable: false, unsigned: true, defaultTo: 0} + sort_order: {type: 'integer', nullable: false, unsigned: true, defaultTo: 0}, + '@@INDEXES@@': [ + ['post_id','tag_id'] + ] }, invites: { id: {type: 'string', maxlength: 24, nullable: false, primary: true}, diff --git a/ghost/core/test/unit/server/data/schema/integrity.test.js b/ghost/core/test/unit/server/data/schema/integrity.test.js index 9f6feb5d85d..2d4a221d434 100644 --- a/ghost/core/test/unit/server/data/schema/integrity.test.js +++ b/ghost/core/test/unit/server/data/schema/integrity.test.js @@ -35,7 +35,7 @@ const validateRouteSettings = require('../../../../../core/server/services/route */ describe('DB version integrity', function () { // Only these variables should need updating - const currentSchemaHash = '45c8072332176e0fe5a4fdff58fb2113'; + const currentSchemaHash = '450aa437d7f5bba4191467d7ae14b8cc'; const currentFixturesHash = 'a489d615989eab1023d4b8af0ecee7fd'; const currentSettingsHash = '5c957ceb48c4878767d7d3db484c592d'; const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01'; From 89b6801f08431296405f0ea622622e48cf8175fc Mon Sep 17 00:00:00 2001 From: Steve Larson <9larsons@gmail.com> Date: Tue, 25 Jun 2024 08:03:29 -0500 Subject: [PATCH 2/6] Removed redundant type,status index --- .../5.87/2024-06-20-17-02-15-add-posts-type-status-index.js | 4 ---- ghost/core/core/server/data/schema/schema.js | 1 - ghost/core/test/unit/server/data/schema/integrity.test.js | 2 +- 3 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 ghost/core/core/server/data/migrations/versions/5.87/2024-06-20-17-02-15-add-posts-type-status-index.js diff --git a/ghost/core/core/server/data/migrations/versions/5.87/2024-06-20-17-02-15-add-posts-type-status-index.js b/ghost/core/core/server/data/migrations/versions/5.87/2024-06-20-17-02-15-add-posts-type-status-index.js deleted file mode 100644 index 375725294f7..00000000000 --- a/ghost/core/core/server/data/migrations/versions/5.87/2024-06-20-17-02-15-add-posts-type-status-index.js +++ /dev/null @@ -1,4 +0,0 @@ -// For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253 -const {createAddIndexMigration} = require('../../utils'); - -module.exports = createAddIndexMigration('posts',['type','status']); \ No newline at end of file diff --git a/ghost/core/core/server/data/schema/schema.js b/ghost/core/core/server/data/schema/schema.js index c871359fc6d..f2e7052f8ce 100644 --- a/ghost/core/core/server/data/schema/schema.js +++ b/ghost/core/core/server/data/schema/schema.js @@ -94,7 +94,6 @@ module.exports = { newsletter_id: {type: 'string', maxlength: 24, nullable: true, references: 'newsletters.id'}, show_title_and_feature_image: {type: 'boolean', nullable: false, defaultTo: true}, '@@INDEXES@@': [ - ['type','status'], ['type','status','updated_at'] ], '@@UNIQUE_CONSTRAINTS@@': [ diff --git a/ghost/core/test/unit/server/data/schema/integrity.test.js b/ghost/core/test/unit/server/data/schema/integrity.test.js index 2d4a221d434..9e8ea291253 100644 --- a/ghost/core/test/unit/server/data/schema/integrity.test.js +++ b/ghost/core/test/unit/server/data/schema/integrity.test.js @@ -35,7 +35,7 @@ const validateRouteSettings = require('../../../../../core/server/services/route */ describe('DB version integrity', function () { // Only these variables should need updating - const currentSchemaHash = '450aa437d7f5bba4191467d7ae14b8cc'; + const currentSchemaHash = 'ce97eff9bf1b3c215fed1271f9275f83'; const currentFixturesHash = 'a489d615989eab1023d4b8af0ecee7fd'; const currentSettingsHash = '5c957ceb48c4878767d7d3db484c592d'; const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01'; From 06f2db3d68e8c5775ad685ced10745d8274c20be Mon Sep 17 00:00:00 2001 From: Steve Larson <9larsons@gmail.com> Date: Tue, 25 Jun 2024 12:23:31 -0500 Subject: [PATCH 3/6] Updated so rollbacks will maintain an index for the FK --- ghost/core/core/server/data/migrations/utils/schema.js | 10 +++++++++- ...024-06-25-12-08-20-add-posts-tags-post-tag-index.js | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ghost/core/core/server/data/migrations/utils/schema.js b/ghost/core/core/server/data/migrations/utils/schema.js index d72215749af..d3263d07d78 100644 --- a/ghost/core/core/server/data/migrations/utils/schema.js +++ b/ghost/core/core/server/data/migrations/utils/schema.js @@ -99,14 +99,22 @@ function createSetNullableMigration(table, column, options = {}) { /** * @param {string} table * @param {string[]|string} columns One or multiple columns (in case the index should be for multiple columns) + * @param {string[]|string} rollbackIndexColumns One or multiple columns (in case the index should be for multiple columns). + * When rolling back an index that modifies a foreign key index, it needs to be replaced with the original index on the FK. * @returns {Migration} */ -function createAddIndexMigration(table, columns) { +function createAddIndexMigration(table, columns, rollbackIndexColumns = null) { return createTransactionalMigration( async function up(knex) { await commands.addIndex(table, columns, knex); }, async function down(knex) { + console.log(`DOWN`) + console.log(`ROLLBACK INDEX COLUMNS >> `, rollbackIndexColumns) + if (rollbackIndexColumns) { + console.log(`ADDING INDEX FOR ROLLBACK >> `, rollbackIndexColumns); + await commands.addIndex(table, rollbackIndexColumns, knex); + } await commands.dropIndex(table, columns, knex); } ); diff --git a/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-20-add-posts-tags-post-tag-index.js b/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-20-add-posts-tags-post-tag-index.js index 85b44ff025f..92b32e48369 100644 --- a/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-20-add-posts-tags-post-tag-index.js +++ b/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-20-add-posts-tags-post-tag-index.js @@ -1,4 +1,4 @@ // For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253 const {createAddIndexMigration} = require('../../utils'); -module.exports = createAddIndexMigration('posts_tags',['post_id','tag_id']); \ No newline at end of file +module.exports = createAddIndexMigration('posts_tags',['post_id','tag_id'],'post_id'); \ No newline at end of file From 6a843422335a14c812671833505b6e8dfb2903e0 Mon Sep 17 00:00:00 2001 From: Steve Larson <9larsons@gmail.com> Date: Tue, 25 Jun 2024 12:31:22 -0500 Subject: [PATCH 4/6] removed comments --- ghost/core/core/server/data/migrations/utils/schema.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/ghost/core/core/server/data/migrations/utils/schema.js b/ghost/core/core/server/data/migrations/utils/schema.js index d3263d07d78..9630aa7e151 100644 --- a/ghost/core/core/server/data/migrations/utils/schema.js +++ b/ghost/core/core/server/data/migrations/utils/schema.js @@ -109,10 +109,7 @@ function createAddIndexMigration(table, columns, rollbackIndexColumns = null) { await commands.addIndex(table, columns, knex); }, async function down(knex) { - console.log(`DOWN`) - console.log(`ROLLBACK INDEX COLUMNS >> `, rollbackIndexColumns) if (rollbackIndexColumns) { - console.log(`ADDING INDEX FOR ROLLBACK >> `, rollbackIndexColumns); await commands.addIndex(table, rollbackIndexColumns, knex); } await commands.dropIndex(table, columns, knex); From c4ed887ff87006cd4d2472a53b0e4b8282727b84 Mon Sep 17 00:00:00 2001 From: Steve Larson <9larsons@gmail.com> Date: Tue, 25 Jun 2024 12:33:31 -0500 Subject: [PATCH 5/6] update migration with error handling --- ...-12-08-20-add-posts-tags-post-tag-index.js | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-20-add-posts-tags-post-tag-index.js b/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-20-add-posts-tags-post-tag-index.js index 92b32e48369..4aa38673dbf 100644 --- a/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-20-add-posts-tags-post-tag-index.js +++ b/ghost/core/core/server/data/migrations/versions/5.87/2024-06-25-12-08-20-add-posts-tags-post-tag-index.js @@ -1,4 +1,26 @@ -// For information on writing migrations, see https://www.notion.so/ghost/Database-migrations-eb5b78c435d741d2b34a582d57c24253 -const {createAddIndexMigration} = require('../../utils'); +const logging = require('@tryghost/logging'); +const {createNonTransactionalMigration} = require('../../utils'); +const {addIndex, dropIndex} = require('../../../schema/commands'); -module.exports = createAddIndexMigration('posts_tags',['post_id','tag_id'],'post_id'); \ No newline at end of file +module.exports = createNonTransactionalMigration( + async function up(knex) { + await addIndex('posts_tags', ['post_id', 'tag_id'], knex); + }, + async function down(knex) { + try { + await dropIndex('posts_tags', ['post_id', 'tag_id'], knex); + } catch (err) { + if (err.code === 'ER_DROP_INDEX_FK') { + logging.error({ + message: 'Error dropping index over posts_tags(post_id, tag_id), re-adding index for post_id' + }); + + await addIndex('posts_tags', ['post_id'], knex); + await dropIndex('posts_tags', ['post_id', 'tag_id'], knex); + return; + } + + throw err; + } + } +); \ No newline at end of file From 27651bbfc8b021edf053d73e0ebdc1f3be0427da Mon Sep 17 00:00:00 2001 From: Steve Larson <9larsons@gmail.com> Date: Wed, 26 Jun 2024 06:53:36 -0500 Subject: [PATCH 6/6] Removed rollback handling in utils --- ghost/core/core/server/data/migrations/utils/schema.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ghost/core/core/server/data/migrations/utils/schema.js b/ghost/core/core/server/data/migrations/utils/schema.js index 9630aa7e151..d72215749af 100644 --- a/ghost/core/core/server/data/migrations/utils/schema.js +++ b/ghost/core/core/server/data/migrations/utils/schema.js @@ -99,19 +99,14 @@ function createSetNullableMigration(table, column, options = {}) { /** * @param {string} table * @param {string[]|string} columns One or multiple columns (in case the index should be for multiple columns) - * @param {string[]|string} rollbackIndexColumns One or multiple columns (in case the index should be for multiple columns). - * When rolling back an index that modifies a foreign key index, it needs to be replaced with the original index on the FK. * @returns {Migration} */ -function createAddIndexMigration(table, columns, rollbackIndexColumns = null) { +function createAddIndexMigration(table, columns) { return createTransactionalMigration( async function up(knex) { await commands.addIndex(table, columns, knex); }, async function down(knex) { - if (rollbackIndexColumns) { - await commands.addIndex(table, rollbackIndexColumns, knex); - } await commands.dropIndex(table, columns, knex); } );