import { expect } from "chai"; import EventEmitter from "events"; import { BridgeConfigFeeds } from "../src/config/Config"; import { ConnectionManager } from "../src/ConnectionManager"; import { IConnection } from "../src/Connections"; import { FeedEntry, FeedReader } from "../src/feeds/FeedReader"; import { MessageQueue, MessageQueueMessage } from "../src/MessageQueue"; import { MemoryStorageProvider } from "../src/Stores/MemoryStorageProvider"; import { Server, createServer } from 'http'; import { AddressInfo } from "net"; class MockConnectionManager extends EventEmitter { constructor( public connections: IConnection[] ) { super(); } getAllConnectionsOfType() { return this.connections; } } class MockMessageQueue extends EventEmitter implements MessageQueue { subscribe(eventGlob: string): void { this.emit('subscribed', eventGlob); } unsubscribe(eventGlob: string): void { this.emit('unsubscribed', eventGlob); } async push(data: MessageQueueMessage, single?: boolean): Promise { this.emit('pushed', data, single); } async pushWait(): Promise { throw new Error('Not yet implemented'); } } async function constructFeedReader(feedResponse: () => {headers: Record, data: string}) { const httpServer = await new Promise(resolve => { const srv = createServer((_req, res) => { res.writeHead(200); const { headers, data } = feedResponse(); Object.entries(headers).forEach(([key,value]) => { res.setHeader(key, value); }); res.write(data); res.end(); }).listen(0, '127.0.0.1', () => { resolve(srv); }); }); const address = httpServer.address() as AddressInfo; const feedUrl = `http://127.0.0.1:${address.port}/` const config = new BridgeConfigFeeds({ enabled: true, pollIntervalSeconds: 1, pollTimeoutSeconds: 1, }); const cm = new MockConnectionManager([{ feedUrl } as unknown as IConnection]) as unknown as ConnectionManager const mq = new MockMessageQueue(); const events: MessageQueueMessage[] = []; mq.on('pushed', (data) => { if (data.eventName === 'feed.entry') {events.push(data);} }); const storage = new MemoryStorageProvider(); // Ensure we don't initial sync by storing a guid. await storage.storeFeedGuids(feedUrl, '-test-guid-'); const feedReader = new FeedReader( config, cm, mq, storage, ); // eslint-disable-next-line mocha/no-top-level-hooks after(() => httpServer.close()); return {config, cm, events, feedReader, feedUrl, httpServer, storage}; } describe("FeedReader", () => { it("should correctly handle empty titles", async () => { const { events, feedReader, feedUrl } = await constructFeedReader(() => ({ headers: {}, data: ` test feedhttp://test/ Wed, 12 Apr 2023 09:53:00 GMT test item http://example.com/test/1681293180 http://example.com/test/1681293180 Wed, 12 Apr 2023 09:53:00 GMT ` })); await feedReader.pollFeed(feedUrl); feedReader.stop(); expect(events).to.have.lengthOf(1); expect(events[0].data.feed.title).to.equal(null); expect(events[0].data.title).to.equal(null); }); it("should handle RSS 2.0 feeds", async () => { const { events, feedReader, feedUrl } = await constructFeedReader(() => ({ headers: {}, data: ` RSS Title This is an example of an RSS feed http://www.example.com/main.html 2020 Example.com All rights reserved Mon, 6 Sep 2010 00:01:00 +0000 Sun, 6 Sep 2009 16:20:00 +0000 1800 Example entry John Doe Here is some text containing an interesting description. http://www.example.com/blog/post/1 7bd204c6-1655-4c27-aeee-53f933c5395f Sun, 6 Sep 2009 16:20:00 +0000 ` })); await feedReader.pollFeed(feedUrl); feedReader.stop(); expect(events).to.have.lengthOf(1); expect(events[0].data.feed.title).to.equal('RSS Title'); expect(events[0].data.author).to.equal('John Doe'); expect(events[0].data.title).to.equal('Example entry'); expect(events[0].data.summary).to.equal('Here is some text containing an interesting description.'); expect(events[0].data.link).to.equal('http://www.example.com/blog/post/1'); expect(events[0].data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000'); }); it("should handle RSS feeds with a permalink url", async () => { const { events, feedReader, feedUrl } = await constructFeedReader(() => ({ headers: {}, data: ` RSS Title This is an example of an RSS feed http://www.example.com/main.html 2020 Example.com All rights reserved Mon, 6 Sep 2010 00:01:00 +0000 Sun, 6 Sep 2009 16:20:00 +0000 1800 Example entry John Doe Here is some text containing an interesting description. http://www.example.com/blog/post/1 Sun, 6 Sep 2009 16:20:00 +0000 ` })); await feedReader.pollFeed(feedUrl); feedReader.stop(); expect(events).to.have.lengthOf(1); expect(events[0].data.feed.title).to.equal('RSS Title'); expect(events[0].data.author).to.equal('John Doe'); expect(events[0].data.title).to.equal('Example entry'); expect(events[0].data.summary).to.equal('Here is some text containing an interesting description.'); expect(events[0].data.link).to.equal('http://www.example.com/blog/post/1'); expect(events[0].data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000'); }); it("should handle Atom feeds", async () => { const { events, feedReader, feedUrl } = await constructFeedReader(() => ({ headers: {}, data: ` Example Feed 2003-12-13T18:30:02Z John Doe urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6 John Doe Atom-Powered Robots Run Amok urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a 2003-12-13T18:30:02Z Some text. ` })); await feedReader.pollFeed(feedUrl); feedReader.stop(); expect(events).to.have.lengthOf(1); expect(events[0].data.feed.title).to.equal('Example Feed'); expect(events[0].data.title).to.equal('Atom-Powered Robots Run Amok'); expect(events[0].data.author).to.equal('John Doe'); expect(events[0].data.summary).to.equal('Some text.'); expect(events[0].data.link).to.equal('http://example.org/2003/12/13/atom03'); expect(events[0].data.pubdate).to.equal('Sat, 13 Dec 2003 18:30:02 +0000'); }); it("should not duplicate feed entries", async () => { const { events, feedReader, feedUrl } = await constructFeedReader(() => ({ headers: {}, data: ` John Doe Atom-Powered Robots Run Amok urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a 2003-12-13T18:30:02Z Some text. ` })); await feedReader.pollFeed(feedUrl); await feedReader.pollFeed(feedUrl); await feedReader.pollFeed(feedUrl); feedReader.stop(); expect(events).to.have.lengthOf(1); }); it("should always hash to the same value for Atom feeds", async () => { const expectedHash = ['md5:d41d8cd98f00b204e9800998ecf8427e']; const { feedReader, feedUrl, storage } = await constructFeedReader(() => ({ headers: {}, data: ` Atom-Powered Robots Run Amok http://example.com/test/123 ` })); await feedReader.pollFeed(feedUrl); feedReader.stop(); const items = await storage.hasSeenFeedGuids(feedUrl, ...expectedHash); expect(items).to.deep.equal(expectedHash); }); it("should always hash to the same value for RSS feeds", async () => { const expectedHash = [ 'md5:98bafde155b931e656ad7c137cd7711e', // guid 'md5:72eec3c0d59ff91a80f0073ee4f8511a', // link 'md5:7c5dd7e5988ff388ab2a402ce7feb2f0', // title ]; const { feedReader, feedUrl, storage } = await constructFeedReader(() => ({ headers: {}, data: ` RSS Title This is an example of an RSS feed Example entry http://www.example.com/blog/post/1 Example entry http://www.example.com/blog/post/2 Example entry 3 ` })); await feedReader.pollFeed(feedUrl); feedReader.stop(); const items = await storage.hasSeenFeedGuids(feedUrl, ...expectedHash); expect(items).to.deep.equal(expectedHash); }); });