🐛 Fixed Twitter not rendering in Emails

ghost/core/core/server/services/oembed/TwitterOEmbedProvider.js

@@ -67,7 +67,6 @@ class TwitterOEmbedProvider {
                         Authorization: `Bearer ${this.dependencies.config.bearerToken}`
                     }
                 }).json();
-
                 oembedData.tweet_data = body.data;
                 oembedData.tweet_data.includes = body.includes;
             } catch (err) {

ghost/core/core/server/services/oembed/XEmbedProvider.js

@@ -0,0 +1,131 @@
+const {ValidationError} = require('@tryghost/errors');
+
+const TWITTER_PATH_REGEX = /\/status\/(\d+)/;
+
+/**
+ * @typedef {import('./oembed').ICustomProvider} ICustomProvider
+ * @typedef {import('./oembed').IExternalRequest} IExternalRequest
+ */
+
+/**
+ * @implements ICustomProvider
+ */
+class XEmbedProvider {
+    /**
+     * @param {object} dependencies
+     */
+    constructor(dependencies) {
+        this._dependencies = dependencies;
+    }
+
+    /**
+     * Checks if the provided URL can be supported (i.e., if it is a Twitter or X.com URL).
+     * @param {URL} url
+     * @returns {Promise<boolean>}
+     */
+    async canSupportRequest(url) {
+        return (url.host === 'twitter.com' || url.host === 'x.com') && TWITTER_PATH_REGEX.test(url.pathname);
+    }
+
+    /**
+     * Extracts the Tweet ID from the given URL.
+     * @param {URL} url
+     * @returns {Promise<string>}
+     * @throws {ValidationError} If the URL is invalid.
+     */
+    async getTweetId(url) {
+        const match = TWITTER_PATH_REGEX.exec(url.pathname);
+
+        if (!match) {
+            throw new ValidationError({
+                message: 'Invalid URL'
+            });
+        }
+
+        return match[1];
+    }
+
+    // Maps tweet entity in email template compatible format
+    async mapTweetToTweetData(tweetEntity) {
+        const urls = (tweetEntity.entities?.urls || []).map((url) => {
+            return {
+                url: url,
+                display_url: url.replace(/(^\w+:|^)\/\//, '') // Remove the protocol
+            };
+        });
+
+        const mentionedUsers = (tweetEntity.entities?.mentionedUsers || []).map((mention) => {
+            return {
+                username: mention
+            };
+        });
+        const tweetData = {
+            id: tweetEntity.id,
+            author_id: tweetEntity.tweetBy.id,
+            public_metrics: {
+                retweet_count: tweetEntity.retweetCount,
+                like_count: tweetEntity.likeCount
+            },
+            users: [{
+                id: tweetEntity.tweetBy.id,
+                name: tweetEntity.tweetBy.fullName,
+                username: tweetEntity.tweetBy.userName,
+                profile_image_url: tweetEntity.tweetBy.profileImage
+            }],
+            text: tweetEntity.fullText,
+            created_at: new Date(tweetEntity.createdAt).toISOString(),
+            entities: {
+                urls: urls,
+                hashtags: tweetEntity.entities.hashtags.map((hashtag) => {
+                    return {
+                        tag: hashtag
+                    };
+                }),
+                mentions: mentionedUsers
+            },
+            attachments: {
+                ...(tweetEntity.media?.length > 0 
+                    ? {media_keys: tweetEntity.media.map(media => media.url)} 
+                    : {})
+            },
+            includes: {
+                media: tweetEntity.media
+            },
+            retweetedTweet: tweetEntity.retweetedTweet,
+            source: 'rettiwt'
+        };
+
+        return tweetData;
+    }
+    /**
+     * @param {URL} url
+     */
+
+    /**
+     * Fetches the oEmbed data for the given Twitter URL.
+     * @param {URL} url
+     * @returns {Promise<object>}
+     */
+
+    async getOEmbedData(url) {
+        if (url.host === 'x.com') { // api is still at twitter.com... also not certain how people are getting x urls because twitter currently redirects every x host to twitter
+            url = new URL('https://twitter.com' + url.pathname);
+        }
+
+        const [match, tweetId] = url.pathname.match(TWITTER_PATH_REGEX);
+        if (!match) {
+            return null;
+        }
+        const {extract} = require('@extractus/oembed-extractor');
+        /** @type {object} */
+        const oembedData = await extract(url.href);
+
+        const tweetData = await this._dependencies._fetchTweetEntity(tweetId);
+        oembedData.tweet_data = await this.mapTweetToTweetData(tweetData);
+        oembedData.type = 'twitter';
+
+        return oembedData;
+    }
+}
+
+module.exports = XEmbedProvider;

ghost/core/core/server/services/oembed/service.js

@@ -1,6 +1,7 @@
 const config = require('../../../shared/config');
 const storage = require('../../adapters/storage');
 const externalRequest = require('../../lib/request-external');
+const {Rettiwt} = require('rettiwt-api');
 
 const OEmbed = require('@tryghost/oembed-service');
 const oembed = new OEmbed({config, externalRequest, storage});
@@ -12,10 +13,21 @@ const nft = new NFT({
     }
 });
 
-const Twitter = require('./TwitterOEmbedProvider');
-const twitter = new Twitter({
-    config: {
-        bearerToken: config.get('twitter').privateReadOnlyToken
+//#TODO Clean up
+
+// const Twitter = require('./TwitterOEmbedProvider');
+// const twitter = new Twitter({
+//     config: {
+//         bearerToken: config.get('twitter').privateReadOnlyToken
+//     }
+// });
+
+const X = require('./XEmbedProvider');
+const rettiwt = new Rettiwt();
+const twitter = new X({
+    _fetchTweetEntity: async (tweetId) => {
+        const response = await rettiwt.tweet.details(tweetId);
+        return response;
     }
 });
 

ghost/core/package.json

@@ -215,6 +215,7 @@
     "node-jose": "2.2.0",
     "path-match": "1.2.4",
     "probe-image-size": "7.2.3",
+    "rettiwt-api": "4.1.4",
     "rss": "1.2.2",
     "sanitize-html": "2.14.0",
     "semver": "7.6.3",

ghost/core/test/unit/server/services/oembed/rettiwtEmbed.test.js

@@ -0,0 +1,206 @@
+const assert = require('assert/strict');
+const sinon = require('sinon');
+const XEmbedProvider = require('../../../../../core/server/services/oembed/XEmbedProvider');
+const nock = require('nock');
+const {ValidationError} = require('@tryghost/errors');
+
+describe('XEmbedProvider', function () {
+    let provider;
+    let mockDependencies;
+
+    before(function () {
+        nock.disableNetConnect();
+    });
+
+    beforeEach(function () {
+        mockDependencies = {
+            _fetchTweetEntity: sinon.stub()
+        };
+        provider = new XEmbedProvider(mockDependencies);
+    });
+
+    afterEach(function () {
+        sinon.restore();
+        nock.cleanAll();
+    });
+
+    describe('canSupportRequest', function () {
+        it('should support valid Twitter URLs', async function () {
+            const url = new URL('https://twitter.com/Ghost/status/1630581157568839683');
+            const result = await provider.canSupportRequest(url);
+            assert.equal(result, true);
+        });
+
+        it('should support valid X.com URLs', async function () {
+            const url = new URL('https://x.com/Ghost/status/1630581157568839683');
+            const result = await provider.canSupportRequest(url);
+            assert.equal(result, true);
+        });
+
+        it('should reject unsupported URLs', async function () {
+            const url = new URL('https://example.com/some/path');
+            const result = await provider.canSupportRequest(url);
+            assert.equal(result, false);
+        });
+    });
+
+    describe('getTweetId', function () {
+        it('should extract Tweet ID from valid URL', async function () {
+            const url = new URL('https://twitter.com/Ghost/status/1630581157568839683');
+            const tweetId = await provider.getTweetId(url);
+            assert.equal(tweetId, '1630581157568839683');
+        });
+
+        it('should throw ValidationError for invalid URL', async function () {
+            const url = new URL('https://twitter.com/Ghost/some/invalid/path');
+            await assert.rejects(() => provider.getTweetId(url), ValidationError, 'Invalid URL');
+        });
+    });
+
+    describe('mapTweetToTweetData', function () {
+        it('should map tweet entity correctly', async function () {
+            const tweetEntity = {
+                id: '12345',
+                tweetBy: {
+                    id: '67890',
+                    fullName: 'Ghost',
+                    userName: 'GhostOrg',
+                    profileImage: 'https://example.com/profile.jpg'
+                },
+                retweetCount: 5,
+                likeCount: 20,
+                fullText: 'This is a test tweet.',
+                createdAt: '2024-01-01T12:00:00Z',
+                entities: {
+                    urls: ['https://example.com'],
+                    hashtags: ['example'],
+                    mentionedUsers: ['anotherUser']
+                },
+                media: [{url: 'https://example.com/media.jpg'}],
+                retweetedTweet: null
+            };
+
+            const tweetData = await provider.mapTweetToTweetData(tweetEntity);
+
+            assert.deepEqual(tweetData, {
+                id: '12345',
+                author_id: '67890',
+                public_metrics: {
+                    retweet_count: 5,
+                    like_count: 20
+                },
+                users: [{
+                    id: '67890',
+                    name: 'Ghost',
+                    username: 'GhostOrg',
+                    profile_image_url: 'https://example.com/profile.jpg'
+                }],
+                source: 'rettiwt',
+                text: 'This is a test tweet.',
+                created_at: '2024-01-01T12:00:00.000Z',
+                entities: {
+                    urls: [{url: 'https://example.com', display_url: 'example.com'}],
+                    hashtags: [{
+                        tag: 'example'
+                    }],
+                    mentions: [{
+                        username: 'anotherUser'
+                    }]
+                },
+                attachments: {
+                    media_keys: ['https://example.com/media.jpg']
+                },
+                includes: {
+                    media: [{url: 'https://example.com/media.jpg'}]
+                },
+                retweetedTweet: null
+            });
+        });
+    });
+
+    describe('getOEmbedData', function () {
+        function nockOembedRequest() {
+            nock('https://publish.twitter.com')
+                .get('/oembed')
+                .query(true)
+                .reply(200, {
+                    html: '<blockquote>Test embed</blockquote>'
+                });
+        }
+
+        it('should fetch and map oEmbed data correctly', async function () {
+            const tweetURL = new URL('https://twitter.com/Ghost/status/1630581157568839683');
+
+            mockDependencies._fetchTweetEntity.resolves({
+                id: '12345',
+                tweetBy: {id: '67890', fullName: 'Ghost', userName: 'GhostOrg', profileImage: 'https://example.com/profile.jpg'},
+                retweetCount: 5,
+                likeCount: 20,
+                fullText: 'This is a test tweet.',
+                createdAt: '2024-01-01T12:00:00Z',
+                entities: {
+                    urls: ['https://example.com', 'https://example2.com'],
+                    hashtags: ['example'],
+                    mentions: ['anotherUser']
+                },
+                media: [{url: 'https://example.com/media.jpg'}],
+                retweetedTweet: null
+            });
+
+            nockOembedRequest();
+
+            nock('https://api.twitter.com')
+                .get('/2/tweets/1630581157568839683')
+                .query(true)
+                .reply(200, {
+                    data: {id: '1630581157568839683'}
+                });
+
+            const oembedData = await provider.getOEmbedData(tweetURL);
+
+            assert.equal(oembedData.type, 'twitter');
+            assert.ok(oembedData.html);
+            assert.ok(oembedData.tweet_data);
+        });
+
+        it('should handle errors when fetching oEmbed data', async function () {
+            const tweetURL = new URL('https://twitter.com/Ghost/status/1630581157568839683');
+
+            // Mock a failed oEmbed request
+            nock('https://publish.twitter.com')
+                .get('/oembed')
+                .query(true)
+                .reply(500, 'Internal Server Error');
+
+            await assert.rejects(
+                () => provider.getOEmbedData(tweetURL),
+                {
+                    name: 'Error',
+                    message: 'Request failed with error code 500'
+                }
+            );
+        });
+
+        it('should handle errors when fetching tweet entity', async function () {
+            const tweetURL = new URL('https://twitter.com/Ghost/status/1630581157568839683');
+
+            // Mock successful oEmbed request
+            nock('https://publish.twitter.com')
+                .get('/oembed')
+                .query(true)
+                .reply(200, {
+                    html: '<blockquote>Test embed</blockquote>'
+                });
+
+            // Mock failed tweet entity fetch
+            mockDependencies._fetchTweetEntity.rejects(new Error('Failed to fetch tweet'));
+
+            await assert.rejects(
+                () => provider.getOEmbedData(tweetURL),
+                {
+                    message: 'Failed to fetch tweet'
+                }
+            );
+        });
+    });
+});