Support removing feed connections, deduplicate feed URLs

src/ConnectionManager.ts

@@ -426,6 +426,7 @@ export class ConnectionManager extends EventEmitter {
         }
         this.connections.splice(connectionIndex, 1);
         Metrics.connections.set(this.connections.length);
+        this.emit('connection-removed', connection);
     }
 
     /**

src/Connections/FeedConnection.ts

@@ -3,7 +3,6 @@ import { IConnection, IConnectionState } from ".";
 import { BridgeConfigFeeds } from "../Config/Config";
 import { FeedEntry, FeedError} from "../feeds/FeedReader";
 import LogWrapper from "../LogWrapper";
-import { GetConnectionsResponseItem } from "../provisioning/api";
 import { IBridgeStorageProvider } from "../Stores/StorageProvider";
 import { BaseConnection } from "./BaseConnection";
 import markdown from "markdown-it";
@@ -33,11 +32,11 @@ export class FeedConnection extends BaseConnection implements IConnection {
         private readonly storage: IBridgeStorageProvider
     ) {
         super(roomId, stateKey, FeedConnection.CanonicalEventType)
-        log.info(`FeedConnection created for ${roomId}, ${JSON.stringify(state)}`);
+        log.info(`Connection ${this.connectionId} created for ${roomId}, ${JSON.stringify(state)}`);
     }
 
     public isInterestedInStateEvent(eventType: string, stateKey: string): boolean {
-        return false;
+        return !!FeedConnection.EventTypes.find(e => e === eventType) && stateKey === this.feedUrl;
     }
 
     public async handleFeedEntry(entry: FeedEntry): Promise<void> {
@@ -62,6 +61,11 @@ export class FeedConnection extends BaseConnection implements IConnection {
         }
     }
 
+    // needed to ensure that the connection is removable
+    public async onRemove(): Promise<void> {
+        log.info(`Removing connection ${this.connectionId}`);
+    }
+
     toString(): string {
         return `FeedConnection ${this.state.url}`;
     }

src/feeds/FeedReader.ts

@@ -50,8 +50,16 @@ function stripHtml(input: string): string {
     return input.replace(/<[^>]*?>/g, '');
 }
 
+function normalizeUrl(input: string): string {
+    const url = new URL(input);
+    url.hash = '';
+    return url.toString();
+}
+
 export class FeedReader {
-    private observedFeedUrls: string[];
+    private connections: FeedConnection[];
+    // ts should notice that we do in fact initialize it in constructor, but it doesn't (in this version)
+    private observedFeedUrls: Set<string> = new Set();
     private seenEntries: Map<string, string[]> = new Map();
     static readonly seenEntriesEventType = "uk.half-shot.matrix-hookshot.feed.reader.seenEntries";
 
@@ -61,12 +69,21 @@ export class FeedReader {
         private queue: MessageQueue,
         private matrixClient: MatrixClient,
     ) {
-        const feedConnections = this.connectionManager.getAllConnectionsOfType(FeedConnection);
+        this.connections = this.connectionManager.getAllConnectionsOfType(FeedConnection);
-        this.observedFeedUrls = feedConnections.map(c => c.feedUrl);
+        this.calculateFeedUrls();
         connectionManager.on('new-connection', c => {
             if (c instanceof FeedConnection) {
-                log.info('New connection tracked:', c.feedUrl);
+                log.info('New connection tracked:', c.connectionId);
-                this.observedFeedUrls.push(c.feedUrl);
+                this.connections.push(c);
+                this.calculateFeedUrls();
+            }
+        });
+        connectionManager.on('connection-removed', removed => {
+            if (removed instanceof FeedConnection) {
+                log.info('Connections before removal:', this.connections.map(c => c.connectionId));
+                this.connections = this.connections.filter(c => c.connectionId !== removed.connectionId);
+                log.info('Connections after removal:', this.connections.map(c => c.connectionId));
+                this.calculateFeedUrls();
             }
         });
 
@@ -77,6 +94,19 @@ export class FeedReader {
         });
     }
 
+    private calculateFeedUrls(): void {
+        // just in case we got an invalid URL somehow
+        const normalizedUrls = [];
+        for (const conn of this.connections) {
+            try {
+                normalizedUrls.push(normalizeUrl(conn.feedUrl));
+            } catch (err: unknown) {
+                log.error(`Invalid feedUrl for connection ${conn.connectionId}: ${conn.feedUrl}. It will not be tracked`);
+            }
+        }
+        this.observedFeedUrls = new Set(normalizedUrls);
+    }
+
     private async loadSeenEntries(): Promise<void> {
         try {
             const accountData = await this.matrixClient.getAccountData<any>(FeedReader.seenEntriesEventType).catch((err: any) => {
@@ -108,13 +138,13 @@ export class FeedReader {
     }
 
     private async pollFeeds(): Promise<void> {
-        log.debug(`Checking for updates in ${this.observedFeedUrls.length} RSS/Atom feeds`);
+        log.debug(`Checking for updates in ${this.observedFeedUrls.size} RSS/Atom feeds`);
 
         let seenEntriesChanged = false;
 
         const fetchingStarted = (new Date()).getTime();
 
-        for (const url of this.observedFeedUrls) {
+        for (const url of this.observedFeedUrls.values()) {
             try {
                 const res = await axios.get(url.toString());
                 const feed = await (new Parser()).parseString(res.data);