mirror of
https://github.com/matrix-org/matrix-hookshot.git
synced 2025-03-10 21:19:13 +00:00
c09f00e2a3
* Ensure atom feeds hash and prefix * Add some fixing code for this. * Remove feed fixer * Refactor RSS feed tests a bit * Add tests to ensure hashes do not change * changelog
296 lines
12 KiB
TypeScript
296 lines
12 KiB
TypeScript
import { expect } from "chai";
|
|
import EventEmitter from "events";
|
|
import { BridgeConfigFeeds } from "../src/config/Config";
|
|
import { ConnectionManager } from "../src/ConnectionManager";
|
|
import { IConnection } from "../src/Connections";
|
|
import { FeedEntry, FeedReader } from "../src/feeds/FeedReader";
|
|
import { MessageQueue, MessageQueueMessage } from "../src/MessageQueue";
|
|
import { MemoryStorageProvider } from "../src/Stores/MemoryStorageProvider";
|
|
import { Server, createServer } from 'http';
|
|
import { AddressInfo } from "net";
|
|
|
|
class MockConnectionManager extends EventEmitter {
|
|
constructor(
|
|
public connections: IConnection[]
|
|
) {
|
|
super();
|
|
}
|
|
|
|
getAllConnectionsOfType() {
|
|
return this.connections;
|
|
}
|
|
}
|
|
|
|
class MockMessageQueue extends EventEmitter implements MessageQueue {
|
|
subscribe(eventGlob: string): void {
|
|
this.emit('subscribed', eventGlob);
|
|
}
|
|
|
|
unsubscribe(eventGlob: string): void {
|
|
this.emit('unsubscribed', eventGlob);
|
|
}
|
|
|
|
async push(data: MessageQueueMessage<unknown>, single?: boolean): Promise<void> {
|
|
this.emit('pushed', data, single);
|
|
}
|
|
|
|
async pushWait<X>(): Promise<X> {
|
|
throw new Error('Not yet implemented');
|
|
}
|
|
}
|
|
|
|
async function constructFeedReader(feedResponse: () => {headers: Record<string,string>, data: string}) {
|
|
const httpServer = await new Promise<Server>(resolve => {
|
|
const srv = createServer((_req, res) => {
|
|
res.writeHead(200);
|
|
const { headers, data } = feedResponse();
|
|
Object.entries(headers).forEach(([key,value]) => {
|
|
res.setHeader(key, value);
|
|
});
|
|
res.write(data);
|
|
res.end();
|
|
}).listen(0, '127.0.0.1', () => {
|
|
resolve(srv);
|
|
});
|
|
});
|
|
const address = httpServer.address() as AddressInfo;
|
|
const feedUrl = `http://127.0.0.1:${address.port}/`
|
|
const config = new BridgeConfigFeeds({
|
|
enabled: true,
|
|
pollIntervalSeconds: 1,
|
|
pollTimeoutSeconds: 1,
|
|
});
|
|
const cm = new MockConnectionManager([{ feedUrl } as unknown as IConnection]) as unknown as ConnectionManager
|
|
const mq = new MockMessageQueue();
|
|
const events: MessageQueueMessage<FeedEntry>[] = [];
|
|
mq.on('pushed', (data) => { if (data.eventName === 'feed.entry') {events.push(data);} });
|
|
|
|
const storage = new MemoryStorageProvider();
|
|
// Ensure we don't initial sync by storing a guid.
|
|
await storage.storeFeedGuids(feedUrl, '-test-guid-');
|
|
const feedReader = new FeedReader(
|
|
config, cm, mq, storage,
|
|
);
|
|
after(() => httpServer.close());
|
|
return {config, cm, events, feedReader, feedUrl, httpServer, storage};
|
|
}
|
|
|
|
describe("FeedReader", () => {
|
|
it("should correctly handle empty titles", async () => {
|
|
const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
|
|
headers: {}, data: `
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
|
|
<channel><title type='text'></title><description>test feed</description><link>http://test/</link>
|
|
<pubDate>Wed, 12 Apr 2023 09:53:00 GMT</pubDate>
|
|
<item>
|
|
<title type='text'></title><description>test item</description>
|
|
<link>http://example.com/test/1681293180</link>
|
|
<guid isPermaLink="true">http://example.com/test/1681293180</guid>
|
|
<pubDate>Wed, 12 Apr 2023 09:53:00 GMT</pubDate>
|
|
</item>
|
|
</channel></rss>
|
|
`
|
|
}));
|
|
|
|
await feedReader.pollFeed(feedUrl);
|
|
feedReader.stop();
|
|
expect(events).to.have.lengthOf(1);
|
|
|
|
expect(events[0].data.feed.title).to.equal(null);
|
|
expect(events[0].data.title).to.equal(null);
|
|
});
|
|
it("should handle RSS 2.0 feeds", async () => {
|
|
const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
|
|
headers: {}, data: `
|
|
<?xml version="1.0" encoding="UTF-8" ?>
|
|
<rss version="2.0">
|
|
<channel>
|
|
<title>RSS Title</title>
|
|
<description>This is an example of an RSS feed</description>
|
|
<link>http://www.example.com/main.html</link>
|
|
<copyright>2020 Example.com All rights reserved</copyright>
|
|
<lastBuildDate>Mon, 6 Sep 2010 00:01:00 +0000</lastBuildDate>
|
|
<pubDate>Sun, 6 Sep 2009 16:20:00 +0000</pubDate>
|
|
<ttl>1800</ttl>
|
|
<item>
|
|
<title>Example entry</title>
|
|
<author>John Doe</author>
|
|
<description>Here is some text containing an interesting description.</description>
|
|
<link>http://www.example.com/blog/post/1</link>
|
|
<guid isPermaLink="false">7bd204c6-1655-4c27-aeee-53f933c5395f</guid>
|
|
<pubDate>Sun, 6 Sep 2009 16:20:00 +0000</pubDate>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
`
|
|
}));
|
|
|
|
await feedReader.pollFeed(feedUrl);
|
|
feedReader.stop();
|
|
expect(events).to.have.lengthOf(1);
|
|
|
|
expect(events[0].data.feed.title).to.equal('RSS Title');
|
|
expect(events[0].data.author).to.equal('John Doe');
|
|
expect(events[0].data.title).to.equal('Example entry');
|
|
expect(events[0].data.summary).to.equal('Here is some text containing an interesting description.');
|
|
expect(events[0].data.link).to.equal('http://www.example.com/blog/post/1');
|
|
expect(events[0].data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000');
|
|
});
|
|
it("should handle RSS feeds with a permalink url", async () => {
|
|
const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
|
|
headers: {}, data: `
|
|
<?xml version="1.0" encoding="UTF-8" ?>
|
|
<rss version="2.0">
|
|
<channel>
|
|
<title>RSS Title</title>
|
|
<description>This is an example of an RSS feed</description>
|
|
<link>http://www.example.com/main.html</link>
|
|
<copyright>2020 Example.com All rights reserved</copyright>
|
|
<lastBuildDate>Mon, 6 Sep 2010 00:01:00 +0000</lastBuildDate>
|
|
<pubDate>Sun, 6 Sep 2009 16:20:00 +0000</pubDate>
|
|
<ttl>1800</ttl>
|
|
<item>
|
|
<title>Example entry</title>
|
|
<author>John Doe</author>
|
|
<description>Here is some text containing an interesting description.</description>
|
|
<guid isPermaLink="true">http://www.example.com/blog/post/1</guid>
|
|
<pubDate>Sun, 6 Sep 2009 16:20:00 +0000</pubDate>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
`
|
|
}));
|
|
|
|
await feedReader.pollFeed(feedUrl);
|
|
feedReader.stop();
|
|
expect(events).to.have.lengthOf(1);
|
|
|
|
expect(events[0].data.feed.title).to.equal('RSS Title');
|
|
expect(events[0].data.author).to.equal('John Doe');
|
|
expect(events[0].data.title).to.equal('Example entry');
|
|
expect(events[0].data.summary).to.equal('Here is some text containing an interesting description.');
|
|
expect(events[0].data.link).to.equal('http://www.example.com/blog/post/1');
|
|
expect(events[0].data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000');
|
|
});
|
|
it("should handle Atom feeds", async () => {
|
|
const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
|
|
headers: {}, data: `
|
|
<?xml version="1.0" encoding="utf-8"?>
|
|
<feed xmlns="http://www.w3.org/2005/Atom">
|
|
|
|
<title>Example Feed</title>
|
|
<link href="http://example.org/"/>
|
|
<updated>2003-12-13T18:30:02Z</updated>
|
|
<author>
|
|
<name>John Doe</name>
|
|
</author>
|
|
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
|
|
|
|
<entry>
|
|
<author>
|
|
<name>John Doe</name>
|
|
</author>
|
|
<title>Atom-Powered Robots Run Amok</title>
|
|
<link href="http://example.org/2003/12/13/atom03"/>
|
|
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
|
|
<updated>2003-12-13T18:30:02Z</updated>
|
|
<summary>Some text.</summary>
|
|
</entry>
|
|
|
|
</feed>
|
|
`
|
|
}));
|
|
|
|
await feedReader.pollFeed(feedUrl);
|
|
feedReader.stop();
|
|
expect(events).to.have.lengthOf(1);
|
|
|
|
expect(events[0].data.feed.title).to.equal('Example Feed');
|
|
expect(events[0].data.title).to.equal('Atom-Powered Robots Run Amok');
|
|
expect(events[0].data.author).to.equal('John Doe');
|
|
expect(events[0].data.summary).to.equal('Some text.');
|
|
expect(events[0].data.link).to.equal('http://example.org/2003/12/13/atom03');
|
|
expect(events[0].data.pubdate).to.equal('Sat, 13 Dec 2003 18:30:02 +0000');
|
|
});
|
|
it("should not duplicate feed entries", async () => {
|
|
const { events, feedReader, feedUrl } = await constructFeedReader(() => ({
|
|
headers: {}, data: `
|
|
<?xml version="1.0" encoding="utf-8"?>
|
|
<feed xmlns="http://www.w3.org/2005/Atom">
|
|
<entry>
|
|
<author>
|
|
<name>John Doe</name>
|
|
</author>
|
|
<title>Atom-Powered Robots Run Amok</title>
|
|
<link href="http://example.org/2003/12/13/atom03"/>
|
|
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
|
|
<updated>2003-12-13T18:30:02Z</updated>
|
|
<summary>Some text.</summary>
|
|
</entry>
|
|
</feed>
|
|
`
|
|
}));
|
|
|
|
await feedReader.pollFeed(feedUrl);
|
|
await feedReader.pollFeed(feedUrl);
|
|
await feedReader.pollFeed(feedUrl);
|
|
feedReader.stop();
|
|
expect(events).to.have.lengthOf(1);
|
|
});
|
|
it("should always hash to the same value for Atom feeds", async () => {
|
|
const expectedHash = ['md5:d41d8cd98f00b204e9800998ecf8427e'];
|
|
const { feedReader, feedUrl, storage } = await constructFeedReader(() => ({
|
|
headers: {}, data: `
|
|
<?xml version="1.0" encoding="utf-8"?>
|
|
<feed xmlns="http://www.w3.org/2005/Atom">
|
|
<entry>
|
|
<title>Atom-Powered Robots Run Amok</title>
|
|
<link href="http://example.com/test/123"/>
|
|
<guid isPermaLink="true">http://example.com/test/123</guid>
|
|
</entry>
|
|
</feed>
|
|
`
|
|
}));
|
|
|
|
await feedReader.pollFeed(feedUrl);
|
|
feedReader.stop();
|
|
const items = await storage.hasSeenFeedGuids(feedUrl, ...expectedHash);
|
|
expect(items).to.deep.equal(expectedHash);
|
|
});
|
|
it("should always hash to the same value for RSS feeds", async () => {
|
|
const expectedHash = [
|
|
'md5:98bafde155b931e656ad7c137cd7711e', // guid
|
|
'md5:72eec3c0d59ff91a80f0073ee4f8511a', // link
|
|
'md5:7c5dd7e5988ff388ab2a402ce7feb2f0', // title
|
|
];
|
|
const { feedReader, feedUrl, storage } = await constructFeedReader(() => ({
|
|
headers: {}, data: `
|
|
<?xml version="1.0" encoding="UTF-8" ?>
|
|
<rss version="2.0">
|
|
<channel>
|
|
<title>RSS Title</title>
|
|
<description>This is an example of an RSS feed</description>
|
|
<item>
|
|
<title>Example entry</title>
|
|
<guid isPermaLink="true">http://www.example.com/blog/post/1</guid>
|
|
</item>
|
|
<item>
|
|
<title>Example entry</title>
|
|
<link>http://www.example.com/blog/post/2</link>
|
|
</item>
|
|
<item>
|
|
<title>Example entry 3</title>
|
|
</item>
|
|
</channel>
|
|
</rss>
|
|
`
|
|
}));
|
|
|
|
await feedReader.pollFeed(feedUrl);
|
|
feedReader.stop();
|
|
const items = await storage.hasSeenFeedGuids(feedUrl, ...expectedHash);
|
|
expect(items).to.deep.equal(expectedHash);
|
|
});
|
|
});
|