Skip to content

Commit

Permalink
Fix Atom feed hashing format. (#901)
Browse files Browse the repository at this point in the history 
* Ensure atom feeds hash and prefix

* Add some fixing code for this.

* Remove feed fixer

* Refactor RSS feed tests a bit

* Add tests to ensure hashes do not change

* changelog
  • Loading branch information
@Half-Shot
Half-Shot authored Feb 21, 2024
1 parent 6093479 commit c09f00e
Show file tree
Hide file tree
Showing 4 changed files with 101 additions and 56 deletions.
1 change: 1 addition & 0 deletions changelog.d/901.bugfix
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix Atom feeds being repeated in rooms once after an upgrade.
4 changes: 3 additions & 1 deletion src/Stores/RedisStorageProvider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,8 +227,10 @@ export class RedisStorageProvider extends RedisStorageContextualProvider impleme

public async hasSeenFeedGuids(url: string, ...guids: string[]): Promise<string[]> {
let multi = this.redis.multi();
const feedKey = `${FEED_GUIDS}${url}`;

for (const guid of guids) {
multi = multi.lpos(`${FEED_GUIDS}${url}`, guid);
multi = multi.lpos(feedKey, guid);
}
const res = await multi.exec();
if (res === null) {
Expand Down
2 changes: 1 addition & 1 deletion src/feeds/parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ fn parse_feed_to_js_result(feed: &Feed) -> JsRssChannel {
.map(|date| date.to_rfc2822()),
summary: item.summary().map(|v| v.value.clone()),
author: authors_to_string(item.authors()),
hash_id: hash_id(item.id.clone()).ok(),
hash_id: hash_id(item.id.clone()).ok().map(|f| format!("md5:{}", f)),
})
.collect(),
}
Expand Down
150 changes: 96 additions & 54 deletions tests/FeedReader.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,18 +62,22 @@ async function constructFeedReader(feedResponse: () => {headers: Record<string,s
});
const cm = new MockConnectionManager([{ feedUrl } as unknown as IConnection]) as unknown as ConnectionManager
const mq = new MockMessageQueue();
const events: MessageQueueMessage<FeedEntry>[] = [];
mq.on('pushed', (data) => { if (data.eventName === 'feed.entry') {events.push(data);} });

const storage = new MemoryStorageProvider();
// Ensure we don't initial sync by storing a guid.
await storage.storeFeedGuids(feedUrl, '-test-guid-');
const feedReader = new FeedReader(
config, cm, mq, storage,
);
return {config, cm, mq, feedReader, feedUrl, httpServer};
after(() => httpServer.close());
return {config, cm, events, feedReader, feedUrl, httpServer, storage};
}

describe("FeedReader", () => {
it("should correctly handle empty titles", async () => {
const { mq, feedReader, httpServer } = await constructFeedReader(() => ({
const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
headers: {}, data: `
<?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
Expand All @@ -89,18 +93,15 @@ describe("FeedReader", () => {
`
}));

after(() => httpServer.close());

const event: any = await new Promise((resolve) => {
mq.on('pushed', (data) => { resolve(data); feedReader.stop() });
});
await feedReader.pollFeed(feedUrl);
feedReader.stop();
expect(events).to.have.lengthOf(1);

expect(event.eventName).to.equal('feed.entry');
expect(event.data.feed.title).to.equal(null);
expect(event.data.title).to.equal(null);
expect(events[0].data.feed.title).to.equal(null);
expect(events[0].data.title).to.equal(null);
});
it("should handle RSS 2.0 feeds", async () => {
const { mq, feedReader, httpServer } = await constructFeedReader(() => ({
const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
headers: {}, data: `
<?xml version="1.0" encoding="UTF-8" ?>
<rss version="2.0">
Expand All @@ -125,22 +126,19 @@ describe("FeedReader", () => {
`
}));

after(() => httpServer.close());

const event: MessageQueueMessage<FeedEntry> = await new Promise((resolve) => {
mq.on('pushed', (data) => { resolve(data); feedReader.stop() });
});
await feedReader.pollFeed(feedUrl);
feedReader.stop();
expect(events).to.have.lengthOf(1);

expect(event.eventName).to.equal('feed.entry');
expect(event.data.feed.title).to.equal('RSS Title');
expect(event.data.author).to.equal('John Doe');
expect(event.data.title).to.equal('Example entry');
expect(event.data.summary).to.equal('Here is some text containing an interesting description.');
expect(event.data.link).to.equal('http://www.example.com/blog/post/1');
expect(event.data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000');
expect(events[0].data.feed.title).to.equal('RSS Title');
expect(events[0].data.author).to.equal('John Doe');
expect(events[0].data.title).to.equal('Example entry');
expect(events[0].data.summary).to.equal('Here is some text containing an interesting description.');
expect(events[0].data.link).to.equal('http://www.example.com/blog/post/1');
expect(events[0].data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000');
});
it("should handle RSS feeds with a permalink url", async () => {
const { mq, feedReader, httpServer } = await constructFeedReader(() => ({
const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
headers: {}, data: `
<?xml version="1.0" encoding="UTF-8" ?>
<rss version="2.0">
Expand All @@ -164,22 +162,19 @@ describe("FeedReader", () => {
`
}));

after(() => httpServer.close());

const event: MessageQueueMessage<FeedEntry> = await new Promise((resolve) => {
mq.on('pushed', (data) => { resolve(data); feedReader.stop() });
});
await feedReader.pollFeed(feedUrl);
feedReader.stop();
expect(events).to.have.lengthOf(1);

expect(event.eventName).to.equal('feed.entry');
expect(event.data.feed.title).to.equal('RSS Title');
expect(event.data.author).to.equal('John Doe');
expect(event.data.title).to.equal('Example entry');
expect(event.data.summary).to.equal('Here is some text containing an interesting description.');
expect(event.data.link).to.equal('http://www.example.com/blog/post/1');
expect(event.data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000');
expect(events[0].data.feed.title).to.equal('RSS Title');
expect(events[0].data.author).to.equal('John Doe');
expect(events[0].data.title).to.equal('Example entry');
expect(events[0].data.summary).to.equal('Here is some text containing an interesting description.');
expect(events[0].data.link).to.equal('http://www.example.com/blog/post/1');
expect(events[0].data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000');
});
it("should handle Atom feeds", async () => {
const { mq, feedReader, httpServer } = await constructFeedReader(() => ({
const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
headers: {}, data: `
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
Expand Down Expand Up @@ -207,22 +202,19 @@ describe("FeedReader", () => {
`
}));

after(() => httpServer.close());

const event: MessageQueueMessage<FeedEntry> = await new Promise((resolve) => {
mq.on('pushed', (data) => { resolve(data); feedReader.stop() });
});
await feedReader.pollFeed(feedUrl);
feedReader.stop();
expect(events).to.have.lengthOf(1);

expect(event.eventName).to.equal('feed.entry');
expect(event.data.feed.title).to.equal('Example Feed');
expect(event.data.title).to.equal('Atom-Powered Robots Run Amok');
expect(event.data.author).to.equal('John Doe');
expect(event.data.summary).to.equal('Some text.');
expect(event.data.link).to.equal('http://example.org/2003/12/13/atom03');
expect(event.data.pubdate).to.equal('Sat, 13 Dec 2003 18:30:02 +0000');
expect(events[0].data.feed.title).to.equal('Example Feed');
expect(events[0].data.title).to.equal('Atom-Powered Robots Run Amok');
expect(events[0].data.author).to.equal('John Doe');
expect(events[0].data.summary).to.equal('Some text.');
expect(events[0].data.link).to.equal('http://example.org/2003/12/13/atom03');
expect(events[0].data.pubdate).to.equal('Sat, 13 Dec 2003 18:30:02 +0000');
});
it("should not duplicate feed entries", async () => {
const { mq, feedReader, httpServer, feedUrl } = await constructFeedReader(() => ({
const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
headers: {}, data: `
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
Expand All @@ -240,14 +232,64 @@ describe("FeedReader", () => {
`
}));

after(() => httpServer.close());

const events: MessageQueueMessage<FeedEntry>[] = [];
mq.on('pushed', (data) => { if (data.eventName === 'feed.entry') {events.push(data);} });
await feedReader.pollFeed(feedUrl);
await feedReader.pollFeed(feedUrl);
await feedReader.pollFeed(feedUrl);
feedReader.stop();
expect(events).to.have.lengthOf(1);
});
it("should always hash to the same value for Atom feeds", async () => {
const expectedHash = ['md5:d41d8cd98f00b204e9800998ecf8427e'];
const { feedReader, feedUrl, storage } = await constructFeedReader(() => ({
headers: {}, data: `
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<entry>
<title>Atom-Powered Robots Run Amok</title>
<link href="http://example.com/test/123"/>
<guid isPermaLink="true">http://example.com/test/123</guid>
</entry>
</feed>
`
}));

await feedReader.pollFeed(feedUrl);
feedReader.stop();
const items = await storage.hasSeenFeedGuids(feedUrl, ...expectedHash);
expect(items).to.deep.equal(expectedHash);
});
it("should always hash to the same value for RSS feeds", async () => {
const expectedHash = [
'md5:98bafde155b931e656ad7c137cd7711e', // guid
'md5:72eec3c0d59ff91a80f0073ee4f8511a', // link
'md5:7c5dd7e5988ff388ab2a402ce7feb2f0', // title
];
const { feedReader, feedUrl, storage } = await constructFeedReader(() => ({
headers: {}, data: `
<?xml version="1.0" encoding="UTF-8" ?>
<rss version="2.0">
<channel>
<title>RSS Title</title>
<description>This is an example of an RSS feed</description>
<item>
<title>Example entry</title>
<guid isPermaLink="true">http://www.example.com/blog/post/1</guid>
</item>
<item>
<title>Example entry</title>
<link>http://www.example.com/blog/post/2</link>
</item>
<item>
<title>Example entry 3</title>
</item>
</channel>
</rss>
`
}));

await feedReader.pollFeed(feedUrl);
feedReader.stop();
const items = await storage.hasSeenFeedGuids(feedUrl, ...expectedHash);
expect(items).to.deep.equal(expectedHash);
});
});

0 comments on commit c09f00e

Please sign in to comment.