nodejs/hookshot
nodejs/hookshot
1
0
Fork 0
mirror of https://github.com/matrix-org/matrix-hookshot.git synced 2025-03-10 21:19:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Atom feed hashing format. (#901)

Browse Source 
* Ensure atom feeds hash and prefix

* Add some fixing code for this.

* Remove feed fixer

* Refactor RSS feed tests a bit

* Add tests to ensure hashes do not change

* changelog
This commit is contained in:
Will Hunt 2024-02-21 16:13:36 +00:00 committed by GitHub
parent 609347966e
commit c09f00e2a3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 101 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/901.bugfix Normal file
View File

@ -0,0 +1 @@
Fix Atom feeds being repeated in rooms once after an upgrade.

4
src/Stores/RedisStorageProvider.ts
View File

@ -227,8 +227,10 @@ export class RedisStorageProvider extends RedisStorageContextualProvider impleme
public async hasSeenFeedGuids(url: string, ...guids: string[]): Promise<string[]> { public async hasSeenFeedGuids(url: string, ...guids: string[]): Promise<string[]> {
let multi = this.redis.multi(); let multi = this.redis.multi();
const feedKey = `${FEED_GUIDS}${url}`;
for (const guid of guids) { for (const guid of guids) {
multi = multi.lpos(`${FEED_GUIDS}${url}`, guid); multi = multi.lpos(feedKey, guid);
} }
const res = await multi.exec(); const res = await multi.exec();
if (res === null) { if (res === null) {

2
src/feeds/parser.rs
View File

@ -118,7 +118,7 @@ fn parse_feed_to_js_result(feed: &Feed) -> JsRssChannel {
.map(|date| date.to_rfc2822()), .map(|date| date.to_rfc2822()),
summary: item.summary().map(|v| v.value.clone()), summary: item.summary().map(|v| v.value.clone()),
author: authors_to_string(item.authors()), author: authors_to_string(item.authors()),
hash_id: hash_id(item.id.clone()).ok(), hash_id: hash_id(item.id.clone()).ok().map(|f| format!("md5:{}", f)),
}) })
.collect(), .collect(),
} }

150
tests/FeedReader.spec.ts
View File

@ -62,18 +62,22 @@ async function constructFeedReader(feedResponse: () => {headers: Record<string,s
}); });
const cm = new MockConnectionManager([{ feedUrl } as unknown as IConnection]) as unknown as ConnectionManager const cm = new MockConnectionManager([{ feedUrl } as unknown as IConnection]) as unknown as ConnectionManager
const mq = new MockMessageQueue(); const mq = new MockMessageQueue();
const events: MessageQueueMessage<FeedEntry>[] = [];
mq.on('pushed', (data) => { if (data.eventName === 'feed.entry') {events.push(data);} });
const storage = new MemoryStorageProvider(); const storage = new MemoryStorageProvider();
// Ensure we don't initial sync by storing a guid. // Ensure we don't initial sync by storing a guid.
await storage.storeFeedGuids(feedUrl, '-test-guid-'); await storage.storeFeedGuids(feedUrl, '-test-guid-');
const feedReader = new FeedReader( const feedReader = new FeedReader(
config, cm, mq, storage, config, cm, mq, storage,
); );
return {config, cm, mq, feedReader, feedUrl, httpServer}; after(() => httpServer.close());
return {config, cm, events, feedReader, feedUrl, httpServer, storage};
} }
describe("FeedReader", () => { describe("FeedReader", () => {
it("should correctly handle empty titles", async () => { it("should correctly handle empty titles", async () => {
const { mq, feedReader, httpServer } = await constructFeedReader(() => ({ const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
headers: {}, data: ` headers: {}, data: `
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0"> <rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
@ -89,18 +93,15 @@ describe("FeedReader", () => {
` `
})); }));
after(() => httpServer.close()); await feedReader.pollFeed(feedUrl);
feedReader.stop();
expect(events).to.have.lengthOf(1);
const event: any = await new Promise((resolve) => { expect(events[0].data.feed.title).to.equal(null);
mq.on('pushed', (data) => { resolve(data); feedReader.stop() }); expect(events[0].data.title).to.equal(null);
});
expect(event.eventName).to.equal('feed.entry');
expect(event.data.feed.title).to.equal(null);
expect(event.data.title).to.equal(null);
}); });
it("should handle RSS 2.0 feeds", async () => { it("should handle RSS 2.0 feeds", async () => {
const { mq, feedReader, httpServer } = await constructFeedReader(() => ({ const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
headers: {}, data: ` headers: {}, data: `
<?xml version="1.0" encoding="UTF-8" ?> <?xml version="1.0" encoding="UTF-8" ?>
<rss version="2.0"> <rss version="2.0">
@ -125,22 +126,19 @@ describe("FeedReader", () => {
` `
})); }));
after(() => httpServer.close()); await feedReader.pollFeed(feedUrl);
feedReader.stop();
expect(events).to.have.lengthOf(1);
const event: MessageQueueMessage<FeedEntry> = await new Promise((resolve) => { expect(events[0].data.feed.title).to.equal('RSS Title');
mq.on('pushed', (data) => { resolve(data); feedReader.stop() }); expect(events[0].data.author).to.equal('John Doe');
}); expect(events[0].data.title).to.equal('Example entry');
expect(events[0].data.summary).to.equal('Here is some text containing an interesting description.');
expect(event.eventName).to.equal('feed.entry'); expect(events[0].data.link).to.equal('http://www.example.com/blog/post/1');
expect(event.data.feed.title).to.equal('RSS Title'); expect(events[0].data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000');
expect(event.data.author).to.equal('John Doe');
expect(event.data.title).to.equal('Example entry');
expect(event.data.summary).to.equal('Here is some text containing an interesting description.');
expect(event.data.link).to.equal('http://www.example.com/blog/post/1');
expect(event.data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000');
}); });
it("should handle RSS feeds with a permalink url", async () => { it("should handle RSS feeds with a permalink url", async () => {
const { mq, feedReader, httpServer } = await constructFeedReader(() => ({ const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
headers: {}, data: ` headers: {}, data: `
<?xml version="1.0" encoding="UTF-8" ?> <?xml version="1.0" encoding="UTF-8" ?>
<rss version="2.0"> <rss version="2.0">
@ -164,22 +162,19 @@ describe("FeedReader", () => {
` `
})); }));
after(() => httpServer.close()); await feedReader.pollFeed(feedUrl);
feedReader.stop();
expect(events).to.have.lengthOf(1);
const event: MessageQueueMessage<FeedEntry> = await new Promise((resolve) => { expect(events[0].data.feed.title).to.equal('RSS Title');
mq.on('pushed', (data) => { resolve(data); feedReader.stop() }); expect(events[0].data.author).to.equal('John Doe');
}); expect(events[0].data.title).to.equal('Example entry');
expect(events[0].data.summary).to.equal('Here is some text containing an interesting description.');
expect(event.eventName).to.equal('feed.entry'); expect(events[0].data.link).to.equal('http://www.example.com/blog/post/1');
expect(event.data.feed.title).to.equal('RSS Title'); expect(events[0].data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000');
expect(event.data.author).to.equal('John Doe');
expect(event.data.title).to.equal('Example entry');
expect(event.data.summary).to.equal('Here is some text containing an interesting description.');
expect(event.data.link).to.equal('http://www.example.com/blog/post/1');
expect(event.data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000');
}); });
it("should handle Atom feeds", async () => { it("should handle Atom feeds", async () => {
const { mq, feedReader, httpServer } = await constructFeedReader(() => ({ const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
headers: {}, data: ` headers: {}, data: `
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom"> <feed xmlns="http://www.w3.org/2005/Atom">
@ -207,22 +202,19 @@ describe("FeedReader", () => {
` `
})); }));
after(() => httpServer.close()); await feedReader.pollFeed(feedUrl);
feedReader.stop();
expect(events).to.have.lengthOf(1);
const event: MessageQueueMessage<FeedEntry> = await new Promise((resolve) => { expect(events[0].data.feed.title).to.equal('Example Feed');
mq.on('pushed', (data) => { resolve(data); feedReader.stop() }); expect(events[0].data.title).to.equal('Atom-Powered Robots Run Amok');
}); expect(events[0].data.author).to.equal('John Doe');
expect(events[0].data.summary).to.equal('Some text.');
expect(event.eventName).to.equal('feed.entry'); expect(events[0].data.link).to.equal('http://example.org/2003/12/13/atom03');
expect(event.data.feed.title).to.equal('Example Feed'); expect(events[0].data.pubdate).to.equal('Sat, 13 Dec 2003 18:30:02 +0000');
expect(event.data.title).to.equal('Atom-Powered Robots Run Amok');
expect(event.data.author).to.equal('John Doe');
expect(event.data.summary).to.equal('Some text.');
expect(event.data.link).to.equal('http://example.org/2003/12/13/atom03');
expect(event.data.pubdate).to.equal('Sat, 13 Dec 2003 18:30:02 +0000');
}); });
it("should not duplicate feed entries", async () => { it("should not duplicate feed entries", async () => {
const { mq, feedReader, httpServer, feedUrl } = await constructFeedReader(() => ({ const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
headers: {}, data: ` headers: {}, data: `
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom"> <feed xmlns="http://www.w3.org/2005/Atom">
@ -240,14 +232,64 @@ describe("FeedReader", () => {
` `
})); }));
after(() => httpServer.close());
const events: MessageQueueMessage<FeedEntry>[] = [];
mq.on('pushed', (data) => { if (data.eventName === 'feed.entry') {events.push(data);} });
await feedReader.pollFeed(feedUrl); await feedReader.pollFeed(feedUrl);
await feedReader.pollFeed(feedUrl); await feedReader.pollFeed(feedUrl);
await feedReader.pollFeed(feedUrl); await feedReader.pollFeed(feedUrl);
feedReader.stop(); feedReader.stop();
expect(events).to.have.lengthOf(1); expect(events).to.have.lengthOf(1);
}); });
it("should always hash to the same value for Atom feeds", async () => {
const expectedHash = ['md5:d41d8cd98f00b204e9800998ecf8427e'];
const { feedReader, feedUrl, storage } = await constructFeedReader(() => ({
headers: {}, data: `
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<entry>
<title>Atom-Powered Robots Run Amok</title>
<link href="http://example.com/test/123"/>
<guid isPermaLink="true">http://example.com/test/123</guid>
</entry>
</feed>
`
}));
await feedReader.pollFeed(feedUrl);
feedReader.stop();
const items = await storage.hasSeenFeedGuids(feedUrl, ...expectedHash);
expect(items).to.deep.equal(expectedHash);
});
it("should always hash to the same value for RSS feeds", async () => {
const expectedHash = [
'md5:98bafde155b931e656ad7c137cd7711e', // guid
'md5:72eec3c0d59ff91a80f0073ee4f8511a', // link
'md5:7c5dd7e5988ff388ab2a402ce7feb2f0', // title
];
const { feedReader, feedUrl, storage } = await constructFeedReader(() => ({
headers: {}, data: `
<?xml version="1.0" encoding="UTF-8" ?>
<rss version="2.0">
<channel>
<title>RSS Title</title>
<description>This is an example of an RSS feed</description>
<item>
<title>Example entry</title>
<guid isPermaLink="true">http://www.example.com/blog/post/1</guid>
</item>
<item>
<title>Example entry</title>
<link>http://www.example.com/blog/post/2</link>
</item>
<item>
<title>Example entry 3</title>
</item>
</channel>
</rss>
`
}));
await feedReader.pollFeed(feedUrl);
feedReader.stop();
const items = await storage.hasSeenFeedGuids(feedUrl, ...expectedHash);
expect(items).to.deep.equal(expectedHash);
});
}); });