Merge pull request #315 from matrix-org/tadzik/rss

Add RSS/Atom feed support

changelog.d/315.feature

@@ -0,0 +1 @@
+Add RSS/Atom feed support

config.sample.yml

@@ -80,6 +80,11 @@ generic:
   userIdPrefix: _webhooks_
   allowJsTransformationFunctions: false
   waitForComplete: false
+feeds:
+  # (Optional) Configure this to enable RSS/Atom feed support
+  #
+  enabled: false
+  pollIntervalSeconds: 600
 provisioning:
   # (Optional) Provisioning API for integration managers
   #

docs/SUMMARY.md

@@ -3,6 +3,7 @@
 - [ℹ️ Hookshot](./hookshot.md)
 - [⚙️ Setup](./setup.md)
   - [📃 Sample Configuration](./setup/sample-configuration.md)
+  - [Feed](./setup/feeds.md)
   - [Figma](./setup/figma.md)
   - [GitHub](./setup/github.md)
   - [GitLab](./setup/gitlab.md)

docs/_site/style.css

@@ -29,21 +29,25 @@
 /* icons for headers */
 
 .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(2) strong:after {
-    content: url('/matrix-hookshot/latest/icons/figma.png')
+    content: url('/matrix-hookshot/latest/icons/feeds.png')
 }
 
 .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(3) strong:after {
-    content: url('/matrix-hookshot/latest/icons/github.png')
+    content: url('/matrix-hookshot/latest/icons/figma.png')
 }
 
 .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(4) strong:after {
-    content: url('/matrix-hookshot/latest/icons/gitlab.png')
+    content: url('/matrix-hookshot/latest/icons/github.png')
 }
 
 .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(5) strong:after {
-    content: url('/matrix-hookshot/latest/icons/jira.png')
+    content: url('/matrix-hookshot/latest/icons/gitlab.png')
 }
 
 .chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(6) strong:after {
+    content: url('/matrix-hookshot/latest/icons/jira.png')
+}
+
+.chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(7) strong:after {
     content: url('/matrix-hookshot/latest/icons/webhooks.png')
-}
+}

docs/metrics.md

@@ -30,6 +30,11 @@ Below is the generated list of Prometheus metrics for Hookshot.
 | matrix_api_calls | The number of Matrix client API calls made | method |
 | matrix_api_calls_failed | The number of Matrix client API calls which failed | method |
 | matrix_appservice_events | The number of events sent over the AS API |  |
+## feed
+| Metric | Help | Labels |
+|--------|------|--------|
+| feed_count | The number of RSS feeds that hookshot is subscribed to |  |
+| feed_fetch_ms | The time taken for hookshot to fetch all feeds |  |
 ## process
 | Metric | Help | Labels |
 |--------|------|--------|

docs/setup.md

@@ -103,6 +103,7 @@ Each permission set can have a services. The `service` field can be:
 - `github`
 - `gitlab`
 - `jira`
+- `feed`
 - `figma`
 - `webhooks`
 - `*`, for any service.
@@ -204,6 +205,7 @@ in the upstream library. See <a href="https://github.com/turt2live/matrix-bot-sd
 
 You will need to configure some services. Each service has its own documentation file inside the setup subdirectory.
 
+- [Feeds](./setup/feeds.md)
 - [Figma](./setup/figma.md)
 - [GitHub](./setup/github.md)
 - [GitLab](./setup/gitlab.md)

docs/setup/feeds.md

@@ -0,0 +1,39 @@
+# Feeds
+
+You can configure hookshot to bridge RSS/Atom feeds into Matrix.
+
+## Configuration
+
+```yaml
+feeds:
+  # (Optional) Configure this to enable RSS/Atom feed support
+  #
+  enabled: true
+  pollIntervalSeconds: 600
+```
+
+`pollIntervalSeconds` specifies how often each feed will be checked for updates.
+It may be checked less often if under exceptional load, but it will never be checked more often than every `pollIntervalSeconds`.
+
+Each feed will only be checked once, regardless of the number of rooms to which it's bridged.
+
+No entries will be bridged upon the “initial sync” -- all entries that exist at the moment of setup will be considered to be already seen.
+
+## Usage
+
+### Adding new feeds
+
+To add a feed to your room:
+
+  - Invite the bot user to the room.
+  - Make sure the bot able to send state events (usually the Moderator power level in clients)
+  - Say `!hookshot feed <URL>` where `<URL>` links to an RSS/Atom feed you want to subscribe to.
+
+### Listing feeds
+
+You can list all feeds that a room you're in is currently subscribed to with `!hookshot feed list`.
+It requires no special permissions from the user issuing the command.
+
+### Removing feeds
+
+To remove a feed from a room, say `!hookshot feed remove <URL>`, with the URL specifying which feed you want to unsubscribe from.

package.json

@@ -41,6 +41,7 @@
     "@octokit/rest": "^18.10.0",
     "@octokit/webhooks": "^9.1.2",
     "@uiw/react-codemirror": "^4.5.3",
+    "ajv": "^8.11.0",
     "axios": "^0.24.0",
     "cors": "^2.8.5",
     "express": "^4.17.1",
@@ -56,6 +57,7 @@
     "node-emoji": "^1.11.0",
     "prom-client": "^14.0.1",
     "reflect-metadata": "^0.1.13",
+    "rss-parser": "^3.12.0",
     "source-map-support": "^0.5.21",
     "string-argv": "^0.3.1",
     "uuid": "^8.3.2",
@@ -69,6 +71,7 @@
     "@prefresh/snowpack": "^3.1.2",
     "@snowpack/plugin-sass": "^1.4.0",
     "@snowpack/plugin-typescript": "^1.2.1",
+    "@types/ajv": "^1.0.0",
     "@types/chai": "^4.2.22",
     "@types/cors": "^2.8.12",
     "@types/express": "^4.17.13",

src/Bridge.ts

@@ -10,7 +10,7 @@ import { GetIssueResponse, GetIssueOpts } from "./Gitlab/Types"
 import { GithubInstance } from "./Github/GithubInstance";
 import { IBridgeStorageProvider } from "./Stores/StorageProvider";
 import { IConnection, GitHubDiscussionSpace, GitHubDiscussionConnection, GitHubUserSpace, JiraProjectConnection, GitLabRepoConnection,
-    GitHubIssueConnection, GitHubProjectConnection, GitHubRepoConnection, GitLabIssueConnection, FigmaFileConnection } from "./Connections";
+    GitHubIssueConnection, GitHubProjectConnection, GitHubRepoConnection, GitLabIssueConnection, FigmaFileConnection, FeedConnection } from "./Connections";
 import { IGitLabWebhookIssueStateEvent, IGitLabWebhookMREvent, IGitLabWebhookNoteEvent, IGitLabWebhookPushEvent, IGitLabWebhookReleaseEvent, IGitLabWebhookTagPushEvent, IGitLabWebhookWikiPageEvent } from "./Gitlab/WebhookTypes";
 import { JiraIssueEvent, JiraIssueUpdatedEvent } from "./Jira/WebhookTypes";
 import { JiraOAuthResult } from "./Jira/Types";
@@ -42,6 +42,7 @@ import { JiraOAuthRequestCloud, JiraOAuthRequestOnPrem, JiraOAuthRequestResult }
 import { CLOUD_INSTANCE } from "./Jira/Client";
 import { GenericWebhookEvent, GenericWebhookEventResult } from "./generic/types";
 import { SetupWidget } from "./Widgets/SetupWidget";
+import { FeedEntry, FeedError, FeedReader } from "./feeds/FeedReader";
 const log = new LogWrapper("Bridge");
 
 export class Bridge {
@@ -137,6 +138,16 @@ export class Bridge {
         await this.tokenStore.load();
         const connManager = this.connectionManager = new ConnectionManager(this.as,
             this.config, this.tokenStore, this.commentProcessor, this.messageClient, this.storage, this.github);
+
+        if (this.config.feeds?.enabled) {
+            new FeedReader(
+                this.config.feeds,
+                this.connectionManager,
+                this.queue,
+                this.as.botClient,
+            );
+        }
+
     
         if (this.config.provisioning) {
             const routers = [];
@@ -196,6 +207,7 @@ export class Bridge {
         this.queue.subscribe("gitlab.*");
         this.queue.subscribe("jira.*");
         this.queue.subscribe("figma.*");
+        this.queue.subscribe("feed.*");
 
         const validateRepoIssue = (data: GitHubWebhookTypes.IssuesEvent|GitHubWebhookTypes.IssueCommentEvent) => {
             if (!data.repository || !data.issue) {
@@ -596,6 +608,17 @@ export class Bridge {
             (c, data) => c.handleNewComment(data.payload),
         )
 
+        this.bindHandlerToQueue<FeedEntry, FeedConnection>(
+            "feed.entry",
+            (data) => connManager.getConnectionsForFeedUrl(data.feed.url),
+            (c, data) => c.handleFeedEntry(data),
+        );
+        this.bindHandlerToQueue<FeedError, FeedConnection>(
+            "feed.error",
+            (data) => connManager.getConnectionsForFeedUrl(data.url),
+            (c, data) => c.handleFeedError(data),
+        );
+
         // Set the name and avatar of the bot
         if (this.config.bot) {
             // Ensure we are registered before we set a profile
@@ -1152,4 +1175,4 @@ export class Bridge {
         log.debug(`Set up ${roomId} as an admin room for ${adminRoom.userId}`);
         return adminRoom;
     }
-}
+}

src/Config/Config.ts

@@ -184,6 +184,11 @@ export class BridgeConfigGitLab {
     }
 }
 
+export interface BridgeConfigFeeds {
+    enabled: boolean;
+    pollIntervalSeconds: number;
+}
+
 export interface BridgeConfigFigma {
     publicUrl: string;
     overrideUserId?: string;
@@ -323,6 +328,7 @@ export interface BridgeConfigRoot {
     bot?: BridgeConfigBot;
     bridge: BridgeConfigBridge;
     figma?: BridgeConfigFigma;
+    feeds?: BridgeConfigFeeds;
     generic?: BridgeGenericWebhooksConfigYAML;
     github?: BridgeConfigGitHub;
     gitlab?: BridgeConfigGitLabYAML;
@@ -362,6 +368,8 @@ export class BridgeConfig {
     public readonly generic?: BridgeConfigGenericWebhooks;
     @configKey("Configure this to enable Figma support", true)
     public readonly figma?: BridgeConfigFigma;
+    @configKey("Configure this to enable RSS/Atom feed support", true)
+    public readonly feeds?: BridgeConfigFeeds;
     @configKey("Define profile information for the bot user", true)
     public readonly bot?: BridgeConfigBot;
     @configKey("EXPERIMENTAL support for complimentary widgets", true)
@@ -395,6 +403,7 @@ export class BridgeConfig {
         this.figma = configData.figma;
         this.jira = configData.jira && new BridgeConfigJira(configData.jira);
         this.generic = configData.generic && new BridgeConfigGenericWebhooks(configData.generic);
+        this.feeds = configData.feeds;
         this.provisioning = configData.provisioning;
         this.passFile = configData.passFile;
         this.bot = configData.bot;
@@ -428,8 +437,8 @@ export class BridgeConfig {
             log.warn(`You have not configured any permissions for the bridge, which by default means all users on ${this.bridge.domain} have admin levels of control. Please adjust your config.`);
         }
 
-        if (!this.github && !this.gitlab && !this.jira && !this.generic && !this.figma) {
+        if (!this.github && !this.gitlab && !this.jira && !this.generic && !this.figma && !this.feeds) {
-            throw Error("Config is not valid: At least one of GitHub, GitLab, JIRA, Figma or generic hooks must be configured");
+            throw Error("Config is not valid: At least one of GitHub, GitLab, JIRA, Figma, feeds or generic hooks must be configured");
         }
 
         // TODO: Formalize env support

src/Config/Defaults.ts

@@ -106,6 +106,10 @@ export const DefaultConfig = new BridgeConfig({
             }
         }
     },
+    feeds: {
+        enabled: false,
+        pollIntervalSeconds: 600,
+    },
     provisioning: {
         secret: "!secretToken"
     },

src/ConnectionManager.ts

@@ -18,13 +18,14 @@ import { GetConnectionTypeResponseItem } from "./provisioning/api";
 import { ApiError, ErrCode } from "./api";
 import { UserTokenStore } from "./UserTokenStore";
 import {v4 as uuid} from "uuid";
-import { FigmaFileConnection } from "./Connections/FigmaFileConnection";
+import { FigmaFileConnection, FeedConnection } from "./Connections";
 import { IBridgeStorageProvider } from "./Stores/StorageProvider";
 import Metrics from "./Metrics";
+import EventEmitter from "events";
 
 const log = new LogWrapper("ConnectionManager");
 
-export class ConnectionManager {
+export class ConnectionManager extends EventEmitter {
     private connections: IConnection[] = [];
     public readonly enabledForProvisioning: Record<string, GetConnectionTypeResponseItem> = {};
 
@@ -39,8 +40,9 @@ export class ConnectionManager {
         private readonly commentProcessor: CommentProcessor,
         private readonly messageClient: MessageSenderClient,
         private readonly storage: IBridgeStorageProvider,
-        private readonly github?: GithubInstance) {
+        private readonly github?: GithubInstance
-
+    ) {
+        super();
     }
 
     /**
@@ -54,6 +56,7 @@ export class ConnectionManager {
         for (const connection of connections) {
             if (!this.connections.find(c => c.connectionId === connection.connectionId)) {
                 this.connections.push(connection);
+                this.emit('new-connection', connection);
             }
         }
         Metrics.connections.set(this.connections.length);
@@ -125,7 +128,7 @@ export class ConnectionManager {
         throw new ApiError(`Connection type not known`);
     }
 
-    private assertStateAllowed(state: StateEvent<any>, serviceType: "github"|"gitlab"|"jira"|"figma"|"webhooks") {
+    private assertStateAllowed(state: StateEvent<any>, serviceType: "github"|"gitlab"|"jira"|"figma"|"webhooks"|"feed") {
         if (state.sender === this.as.botUserId) {
             return;
         }
@@ -245,6 +248,14 @@ export class ConnectionManager {
             return new FigmaFileConnection(roomId, state.stateKey, state.content, this.config.figma, this.as, this.storage);
         }
 
+        if (FeedConnection.EventTypes.includes(state.type)) {
+            if (!this.config.feeds?.enabled) {
+                throw Error('RSS/Atom feeds are not configured');
+            }
+            this.assertStateAllowed(state, "feed");
+            return new FeedConnection(roomId, state.stateKey, state.content, this.config.feeds, this.as, this.storage);
+        }
+
         if (GenericHookConnection.EventTypes.includes(state.type) && this.config.generic?.enabled) {
             if (!this.config.generic) {
                 throw Error('Generic webhooks are not configured');
@@ -374,6 +385,10 @@ export class ConnectionManager {
     public getForFigmaFile(fileKey: string, instanceName: string): FigmaFileConnection[] {
         return this.connections.filter((c) => (c instanceof FigmaFileConnection && (c.fileId === fileKey || c.instanceName === instanceName))) as FigmaFileConnection[];
     }
+
+    public getConnectionsForFeedUrl(url: string): FeedConnection[] {
+        return this.connections.filter(c => c instanceof FeedConnection && c.feedUrl === url) as FeedConnection[];
+    }
     
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     public getAllConnectionsOfType<T extends IConnection>(typeT: new (...params : any[]) => T): T[] {
@@ -411,6 +426,7 @@ export class ConnectionManager {
         }
         this.connections.splice(connectionIndex, 1);
         Metrics.connections.set(this.connections.length);
+        this.emit('connection-removed', connection);
     }
 
     /**

src/Connections/FeedConnection.ts

@@ -0,0 +1,84 @@
+import {Appservice} from "matrix-bot-sdk";
+import { IConnection, IConnectionState } from ".";
+import { BridgeConfigFeeds } from "../Config/Config";
+import { FeedEntry, FeedError} from "../feeds/FeedReader";
+import LogWrapper from "../LogWrapper";
+import { IBridgeStorageProvider } from "../Stores/StorageProvider";
+import { BaseConnection } from "./BaseConnection";
+import markdown from "markdown-it";
+
+const log = new LogWrapper("FeedConnection");
+const md = new markdown();
+
+export interface FeedConnectionState extends IConnectionState {
+    url: string;
+}
+
+export class FeedConnection extends BaseConnection implements IConnection {
+    static readonly CanonicalEventType = "uk.half-shot.matrix-hookshot.feed";
+    static readonly EventTypes = [ FeedConnection.CanonicalEventType ];
+    private hasError = false;
+
+    public get feedUrl(): string {
+        return this.state.url;
+    }
+
+    constructor(
+        roomId: string,
+        stateKey: string,
+        private state: FeedConnectionState,
+        private readonly config: BridgeConfigFeeds,
+        private readonly as: Appservice,
+        private readonly storage: IBridgeStorageProvider
+    ) {
+        super(roomId, stateKey, FeedConnection.CanonicalEventType)
+        log.info(`Connection ${this.connectionId} created for ${roomId}, ${JSON.stringify(state)}`);
+    }
+
+    public isInterestedInStateEvent(eventType: string, stateKey: string): boolean {
+        return !!FeedConnection.EventTypes.find(e => e === eventType) && stateKey === this.feedUrl;
+    }
+
+    public async handleFeedEntry(entry: FeedEntry): Promise<void> {
+        this.hasError = false;
+
+        let entryDetails;
+        if (entry.title && entry.link) {
+            entryDetails = `[${entry.title}](${entry.link})`;
+        } else {
+            entryDetails = entry.title || entry.link;
+        }
+
+        let message = `New post in ${entry.feed.title || entry.feed.url}`;
+        if (entryDetails) {
+            message += `: ${entryDetails}`;
+        }
+
+        await this.as.botIntent.sendEvent(this.roomId, {
+            msgtype: 'm.notice',
+            format: "org.matrix.custom.html",
+            formatted_body: md.renderInline(message),
+            body: message,
+        });
+    }
+
+    public async handleFeedError(error: FeedError): Promise<void> {
+        if (!this.hasError) {
+            await this.as.botIntent.sendEvent(this.roomId, {
+                msgtype: 'm.notice',
+                format: 'm.text',
+                body: `Error fetching ${this.feedUrl}: ${error.cause.message}`
+            });
+            this.hasError = true;
+        }
+    }
+
+    // needed to ensure that the connection is removable
+    public async onRemove(): Promise<void> {
+        log.info(`Removing connection ${this.connectionId}`);
+    }
+
+    toString(): string {
+        return `FeedConnection ${this.state.url}`;
+    }
+}

src/Connections/IConnection.ts

@@ -64,7 +64,7 @@ export interface IConnection {
      * If supported, this is sent when a user attempts to remove the connection from a room. The connection
      *  state should be removed and any resources should be cleaned away.
      */
-    onRemove?: () => void;
+    onRemove?: () => Promise<void>;
 
     toString(): string;
-}
+}

src/Connections/SetupConnection.ts

@@ -10,6 +10,7 @@ import { v4 as uuid } from "uuid";
 import { BridgeConfig, BridgePermissionLevel } from "../Config/Config";
 import markdown from "markdown-it";
 import { FigmaFileConnection } from "./FigmaFileConnection";
+import { FeedConnection } from "./FeedConnection";
 import { URL } from "url";
 import { SetupWidget } from "../Widgets/SetupWidget";
 import { AdminRoom } from "../AdminRoom";
@@ -45,6 +46,7 @@ export class SetupConnection extends CommandConnection {
                 this.config.figma ? "figma": "",
                 this.config.jira ? "jira": "",
                 this.config.generic?.enabled ? "webhook": "",
+                this.config.feeds?.enabled ? "feed" : "",
                 this.config.widgets?.roomSetupWidget ? "widget" : "",
             ];
             this.includeTitlesInHelp = false;
@@ -55,15 +57,9 @@ export class SetupConnection extends CommandConnection {
         if (!this.githubInstance || !this.config.github) {
             throw new CommandError("not-configured", "The bridge is not configured to support GitHub.");
         }
-        if (!this.config.checkPermission(userId, "github", BridgePermissionLevel.manageConnections)) {
+
-            throw new CommandError('You are not permitted to provision connections for GitHub.');
+        await this.checkUserPermissions(userId, "github", GitHubRepoConnection.CanonicalEventType);
-        }
+
-        if (!await this.as.botClient.userHasPowerLevelFor(userId, this.roomId, "", true)) {
-            throw new CommandError("not-configured", "You must be able to set state in a room ('Change settings') in order to set up new integrations.");
-        }
-        if (!await this.as.botClient.userHasPowerLevelFor(this.as.botUserId, this.roomId, GitHubRepoConnection.CanonicalEventType, true)) {
-            throw new CommandError("Bot lacks power level to set room state", "I do not have permission to set up a bridge in this room. Please promote me to an Admin/Moderator.");
-        }
         const octokit = await this.tokenStore.getOctokitForUser(userId);
         if (!octokit) {
             throw new CommandError("User not logged in", "You are not logged into GitHub. Start a DM with this bot and use the command `github login`.");
@@ -84,15 +80,9 @@ export class SetupConnection extends CommandConnection {
         if (!this.config.jira) {
             throw new CommandError("not-configured", "The bridge is not configured to support Jira.");
         }
-        if (!this.config.checkPermission(userId, "jira", BridgePermissionLevel.manageConnections)) {
+
-            throw new CommandError('You are not permitted to provision connections for Jira.');
+        await this.checkUserPermissions(userId, "jira", JiraProjectConnection.CanonicalEventType);
-        }
+
-        if (!await this.as.botClient.userHasPowerLevelFor(userId, this.roomId, "", true)) {
-            throw new CommandError("not-configured", "You must be able to set state in a room ('Change settings') in order to set up new integrations.");
-        }
-        if (!await this.as.botClient.userHasPowerLevelFor(this.as.botUserId, this.roomId, GitHubRepoConnection.CanonicalEventType, true)) {
-            throw new CommandError("Bot lacks power level to set room state", "I do not have permission to set up a bridge in this room. Please promote me to an Admin/Moderator.");
-        }
         const jiraClient = await this.tokenStore.getJiraForUser(userId, urlStr);
         if (!jiraClient) {
             throw new CommandError("User not logged in", "You are not logged into Jira. Start a DM with this bot and use the command `jira login`.");
@@ -113,15 +103,9 @@ export class SetupConnection extends CommandConnection {
         if (!this.config.generic?.enabled) {
             throw new CommandError("not-configured", "The bridge is not configured to support webhooks.");
         }
-        if (!this.config.checkPermission(userId, "webhooks", BridgePermissionLevel.manageConnections)) {
+
-            throw new CommandError('You are not permitted to provision connections for generic webhooks.');
+        await this.checkUserPermissions(userId, "webhooks", GitHubRepoConnection.CanonicalEventType);
-        }
+
-        if (!await this.as.botClient.userHasPowerLevelFor(userId, this.roomId, "", true)) {
-            throw new CommandError("not-configured", "You must be able to set state in a room ('Change settings') in order to set up new integrations.");
-        }
-        if (!await this.as.botClient.userHasPowerLevelFor(this.as.botUserId, this.roomId, GitHubRepoConnection.CanonicalEventType, true)) {
-            throw new CommandError("Bot lacks power level to set room state", "I do not have permission to set up a bridge in this room. Please promote me to an Admin/Moderator.");
-        }
         if (!name || name.length < 3 || name.length > 64) {
             throw new CommandError("Bad webhook name", "A webhook name must be between 3-64 characters.");
         }
@@ -139,15 +123,9 @@ export class SetupConnection extends CommandConnection {
         if (!this.config.figma) {
             throw new CommandError("not-configured", "The bridge is not configured to support Figma.");
         }
-        if (!this.config.checkPermission(userId, "figma", BridgePermissionLevel.manageConnections)) {
+
-            throw new CommandError('You are not permitted to provision connections for Figma.');
+        await this.checkUserPermissions(userId, "figma", FigmaFileConnection.CanonicalEventType);
-        }
+
-        if (!await this.as.botClient.userHasPowerLevelFor(userId, this.roomId, "", true)) {
-            throw new CommandError("not-configured", "You must be able to set state in a room ('Change settings') in order to set up new integrations.");
-        }
-        if (!await this.as.botClient.userHasPowerLevelFor(this.as.botUserId, this.roomId, GitHubRepoConnection.CanonicalEventType, true)) {
-            throw new CommandError("Bot lacks power level to set room state", "I do not have permission to set up a bridge in this room. Please promote me to an Admin/Moderator.");
-        }
         const res = /https:\/\/www\.figma\.com\/file\/(\w+).+/.exec(url);
         if (!res) {
             throw new CommandError("Invalid Figma url", "The Figma file url you entered was not valid. It should be in the format of `https://figma.com/file/FILEID/...`.");
@@ -157,6 +135,63 @@ export class SetupConnection extends CommandConnection {
         return this.as.botClient.sendHtmlNotice(this.roomId, md.renderInline(`Room configured to bridge Figma file.`));
     }
 
+    @botCommand("feed", { help: "Bridge an RSS/Atom feed to the room.", requiredArgs: ["url"], includeUserId: true, category: "feed"})
+    public async onFeed(userId: string, url: string) {
+        if (!this.config.feeds?.enabled) {
+            throw new CommandError("not-configured", "The bridge is not configured to support feeds.");
+        }
+
+        await this.checkUserPermissions(userId, "feed", FeedConnection.CanonicalEventType);
+
+        try {
+            new URL(url);
+            // TODO: fetch and check content-type?
+        } catch {
+            throw new CommandError("Invalid URL", `${url} doesn't look like a valid feed URL`);
+        }
+
+        await this.as.botClient.sendStateEvent(this.roomId, FeedConnection.CanonicalEventType, url, {url});
+        return this.as.botClient.sendHtmlNotice(this.roomId, md.renderInline(`Room configured to bridge \`${url}\``));
+    }
+
+    @botCommand("feed list", { help: "Show feeds currently subscribed to.", category: "feed"})
+    public async onFeedList() {
+        const urls = await this.as.botClient.getRoomState(this.roomId).catch((err: any) => {
+            if (err.body.errcode === 'M_NOT_FOUND') {
+                return []; // not an error to us
+            }
+            throw err;
+        }).then(events =>
+            events.filter(
+                (ev: any) => ev.type === FeedConnection.CanonicalEventType && ev.content.url
+            ).map(ev => ev.content.url)
+        );
+
+        if (urls.length === 0) {
+            return this.as.botClient.sendHtmlNotice(this.roomId, md.renderInline('Not subscribed to any feeds'));
+        } else {
+            return this.as.botClient.sendHtmlNotice(this.roomId, md.render(`Currently subscribed to these feeds:\n\n${urls.map(url => ' * ' + url + '\n')}`));
+        }
+    }
+
+    @botCommand("feed remove", { help: "Unsubscribe from an RSS/Atom.", requiredArgs: ["url"], includeUserId: true, category: "feed"})
+    public async onFeedRemove(userId: string, url: string) {
+        await this.checkUserPermissions(userId, "feed", FeedConnection.CanonicalEventType);
+
+        const event = await this.as.botClient.getRoomStateEvent(this.roomId, FeedConnection.CanonicalEventType, url).catch((err: any) => {
+            if (err.body.errcode === 'M_NOT_FOUND') {
+                return null; // not an error to us
+            }
+            throw err;
+        });
+        if (!event || Object.keys(event).length === 0) {
+            throw new CommandError("Invalid feed URL", `Feed "${url}" is not currently bridged to this room`);
+        }
+
+        await this.as.botClient.sendStateEvent(this.roomId, FeedConnection.CanonicalEventType, url, {});
+        return this.as.botClient.sendHtmlNotice(this.roomId, md.renderInline(`Unsubscribed from \`${url}\``));
+    }
+
     @botCommand("setup-widget", {category: "widget", help: "Open the setup widget in the room"})
     public async onSetupWidget() {
         if (!this.config.widgets?.roomSetupWidget) {
@@ -166,6 +201,18 @@ export class SetupConnection extends CommandConnection {
             await this.as.botClient.sendNotice(this.roomId, `This room already has a setup widget, please open the "Hookshot Configuration" widget.`);
         }
     }
+
+    private async checkUserPermissions(userId: string, service: string, stateEventType: string): Promise<void> {
+        if (!this.config.checkPermission(userId, service, BridgePermissionLevel.manageConnections)) {
+            throw new CommandError(`You are not permitted to provision connections for ${service}.`);
+        }
+        if (!await this.as.botClient.userHasPowerLevelFor(userId, this.roomId, "", true)) {
+            throw new CommandError("not-configured", "You must be able to set state in a room ('Change settings') in order to set up new integrations.");
+        }
+        if (!await this.as.botClient.userHasPowerLevelFor(this.as.botUserId, this.roomId, stateEventType, true)) {
+            throw new CommandError("Bot lacks power level to set room state", "I do not have permission to set up a bridge in this room. Please promote me to an Admin/Moderator.");
+        }
+    }
 }
 
 // Typescript doesn't understand Prototypes very well yet.

src/Connections/index.ts

@@ -9,4 +9,5 @@ export * from "./GitlabIssue";
 export * from "./GitlabRepo";
 export * from "./IConnection";
 export * from "./JiraProject";
-export * from "./FigmaFileConnection";
+export * from "./FigmaFileConnection";
+export * from "./FeedConnection";

src/Jira/oauth/OnPremOAuth.ts

@@ -132,7 +132,7 @@ export class JiraOnPremOAuth implements JiraOAuth {
 
     private prepareParameters(oauthToken: string|null, method: Method, urlStr: string, extraParams: Record<string, string> = {}) {
         const oauthParameters: Record<string, string> = {
-            oauth_timestamp: Math.floor( (new Date()).getTime() / 1000 ).toString(),
+            oauth_timestamp: Math.floor( Date.now() / 1000 ).toString(),
             oauth_nonce: JiraOnPremOAuth.nonce(),
             oauth_version: "1.0",
             oauth_signature_method: "RSA-SHA1",

src/Metrics.ts

@@ -23,6 +23,10 @@ export class Metrics {
 
     public readonly matrixAppserviceEvents = new Counter({ name: "matrix_appservice_events", help: "The number of events sent over the AS API", labelNames: [], registers: [this.registry]});
 
+    public readonly feedsCount = new Gauge({ name: "feed_count", help: "The number of RSS feeds that hookshot is subscribed to", labelNames: [], registers: [this.registry]});
+    public readonly feedFetchMs = new Gauge({ name: "feed_fetch_ms", help: "The time taken for hookshot to fetch all feeds", labelNames: [], registers: [this.registry]});
+
+
     constructor(private registry: Registry = register) {
         this.expressRouter.get('/metrics', this.metricsFunc.bind(this));
         collectDefaultMetrics({

src/feeds/FeedReader.ts

@@ -0,0 +1,229 @@
+import { MatrixClient } from "matrix-bot-sdk";
+import { BridgeConfigFeeds } from "../Config/Config";
+import { ConnectionManager } from "../ConnectionManager";
+import { FeedConnection } from "../Connections";
+import LogWrapper from "../LogWrapper";
+import { MessageQueue } from "../MessageQueue";
+
+import Ajv from "ajv";
+import axios from "axios";
+import Parser from "rss-parser";
+import Metrics from "../Metrics";
+
+const log = new LogWrapper("FeedReader");
+
+export class FeedError extends Error {
+    constructor(
+        public url: string,
+        public cause: Error,
+    ) {
+        super(`Error fetching feed ${url}: ${cause.message}`);
+    }
+}
+
+export interface FeedEntry {
+    feed: {
+        title: string|null,
+        url:   string,
+    },
+    title: string|null,
+    link:  string|null,
+}
+
+interface AccountData {
+    [url: string]: string[],
+}
+
+const accountDataSchema = {
+    type: 'object',
+    patternProperties: {
+        "https?://.+": {
+            type: 'array',
+            items: { type: 'string' },
+        }
+    },
+    additionalProperties: false,
+};
+const ajv = new Ajv();
+const validateAccountData = ajv.compile<AccountData>(accountDataSchema);
+
+function stripHtml(input: string): string {
+    return input.replace(/<[^>]*?>/g, '');
+}
+
+function normalizeUrl(input: string): string {
+    const url = new URL(input);
+    url.hash = '';
+    return url.toString();
+}
+
+export class FeedReader {
+    private connections: FeedConnection[];
+    // ts should notice that we do in fact initialize it in constructor, but it doesn't (in this version)
+    private observedFeedUrls: Set<string> = new Set();
+    private seenEntries: Map<string, string[]> = new Map();
+    static readonly seenEntriesEventType = "uk.half-shot.matrix-hookshot.feed.reader.seenEntries";
+
+    constructor(
+        private config: BridgeConfigFeeds,
+        private connectionManager: ConnectionManager,
+        private queue: MessageQueue,
+        private matrixClient: MatrixClient,
+    ) {
+        this.connections = this.connectionManager.getAllConnectionsOfType(FeedConnection);
+        this.calculateFeedUrls();
+        connectionManager.on('new-connection', c => {
+            if (c instanceof FeedConnection) {
+                log.info('New connection tracked:', c.connectionId);
+                this.connections.push(c);
+                this.calculateFeedUrls();
+            }
+        });
+        connectionManager.on('connection-removed', removed => {
+            if (removed instanceof FeedConnection) {
+                log.info('Connections before removal:', this.connections.map(c => c.connectionId));
+                this.connections = this.connections.filter(c => c.connectionId !== removed.connectionId);
+                log.info('Connections after removal:', this.connections.map(c => c.connectionId));
+                this.calculateFeedUrls();
+            }
+        });
+
+        log.info('Loaded feed URLs:', this.observedFeedUrls);
+
+        void this.loadSeenEntries().then(() => {
+            return this.pollFeeds();
+        });
+    }
+
+    private calculateFeedUrls(): void {
+        // just in case we got an invalid URL somehow
+        const normalizedUrls = [];
+        for (const conn of this.connections) {
+            try {
+                normalizedUrls.push(normalizeUrl(conn.feedUrl));
+            } catch (err: unknown) {
+                log.error(`Invalid feedUrl for connection ${conn.connectionId}: ${conn.feedUrl}. It will not be tracked`);
+            }
+        }
+        this.observedFeedUrls = new Set(normalizedUrls);
+        Metrics.feedsCount.set(this.observedFeedUrls.size);
+    }
+
+    private async loadSeenEntries(): Promise<void> {
+        try {
+            const accountData = await this.matrixClient.getAccountData<any>(FeedReader.seenEntriesEventType).catch((err: any) => {
+                if (err.statusCode === 404) {
+                    return {};
+                } else {
+                    throw err;
+                }
+            });
+            if (!validateAccountData(accountData)) {
+                const errors = validateAccountData.errors!.map(e => `${e.instancePath} ${e.message}`);
+                throw new Error(`Invalid account data: ${errors.join(', ')}`);
+            }
+            for (const url in accountData) {
+                this.seenEntries.set(url, accountData[url]);
+            }
+        } catch (err: unknown) {
+            log.error(`Failed to load seen feed entries from accountData: ${err}. This may result in skipped entries`);
+            // no need to wipe it manually, next saveSeenEntries() will make it right
+        }
+    }
+
+    private async saveSeenEntries(): Promise<void> {
+        const accountData: AccountData = {};
+        for (const [url, guids] of this.seenEntries.entries()) {
+            accountData[url.toString()] = guids;
+        }
+        await this.matrixClient.setAccountData(FeedReader.seenEntriesEventType, accountData);
+    }
+
+    private async pollFeeds(): Promise<void> {
+        log.debug(`Checking for updates in ${this.observedFeedUrls.size} RSS/Atom feeds`);
+
+        let seenEntriesChanged = false;
+
+        const fetchingStarted = Date.now();
+
+        for (const url of this.observedFeedUrls.values()) {
+            try {
+                const res = await axios.get(url.toString());
+                const feed = await (new Parser()).parseString(res.data);
+                let initialSync = false;
+                let seenGuids = this.seenEntries.get(url);
+                if (!seenGuids) {
+                    initialSync = true;
+                    seenGuids = [];
+                    seenEntriesChanged = true; // to ensure we only treat it as an initialSync once
+                }
+                const seenGuidsSet = new Set(seenGuids);
+                const newGuids = [];
+                log.debug(`Found ${feed.items.length} entries in ${url}`);
+                for (const item of feed.items) {
+                    const guid = item.guid || item.id || item.link || item.title;
+                    if (!guid) {
+                        log.error(`Could not determine guid for entry in ${url}, skipping`);
+                        continue;
+                    }
+                    newGuids.push(guid);
+
+                    if (initialSync) {
+                        log.debug(`Skipping entry ${guid} since we're performing an initial sync`);
+                        continue;
+                    }
+                    if (seenGuidsSet.has(guid)) {
+                        log.debug('Skipping already seen entry', guid);
+                        continue;
+                    }
+
+                    const entry = {
+                        feed: {
+                            title: feed.title ? stripHtml(feed.title) : null,
+                            url: url.toString()
+                        },
+                        title: item.title ? stripHtml(item.title) : null,
+                        link: item.link || null,
+                    };
+
+                    log.debug('New entry:', entry);
+                    seenEntriesChanged = true;
+
+                    this.queue.push<FeedEntry>({ eventName: 'feed.entry', sender: 'FeedReader', data: entry });
+                }
+
+                if (seenEntriesChanged) {
+                    // Some RSS feeds can return a very small number of items then bounce
+                    // back to their "normal" size, so we cannot just clobber the recent GUID list per request or else we'll
+                    // forget what we sent and resend it. Instead, we'll keep 2x the max number of items that we've ever
+                    // seen from this feed, up to a max of 10,000. 
+                    // Adopted from https://github.com/matrix-org/go-neb/blob/babb74fa729882d7265ff507b09080e732d060ae/services/rssbot/rssbot.go#L304
+                    const maxGuids = Math.min(Math.max(2 * newGuids.length, seenGuids.length), 10_000);
+                    const newSeenItems = Array.from(new Set([ ...newGuids, ...seenGuids ]).values()).slice(0, maxGuids);
+                    this.seenEntries.set(url, newSeenItems);
+                }
+            } catch (err: any) {
+                const error = new FeedError(url.toString(), err);
+                log.error(error.message);
+                this.queue.push<FeedError>({ eventName: 'feed.error', sender: 'FeedReader', data: error });
+            }
+        }
+        if (seenEntriesChanged) await this.saveSeenEntries();
+
+        const elapsed = Date.now() - fetchingStarted;
+        Metrics.feedFetchMs.set(elapsed);
+
+        let sleepFor: number;
+        if (elapsed > this.config.pollIntervalSeconds * 1000) {
+            log.warn(`It tooks us longer to update the feeds than the configured pool interval (${elapsed / 1000}s)`);
+            sleepFor = 0;
+        } else {
+            sleepFor = this.config.pollIntervalSeconds * 1000 - elapsed;
+            log.debug(`Feed fetching took ${elapsed / 1000}s, sleeping for ${sleepFor / 1000}s`);
+        }
+
+        setTimeout(() => {
+            void this.pollFeeds();
+        }, sleepFor);
+    }
+}

yarn.lock

@@ -1288,6 +1288,13 @@
     "@napi-rs/cli" "^2.2.0"
     shelljs "^0.8.4"
 
+"@types/ajv@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@types/ajv/-/ajv-1.0.0.tgz#4fb2440742f2f6c30e7fb0797b839fc6f696682a"
+  integrity sha1-T7JEB0Ly9sMOf7B5e4OfxvaWaCo=
+  dependencies:
+    ajv "*"
+
 "@types/body-parser@*":
   version "1.19.2"
   resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.2.tgz#aea2059e28b7658639081347ac4fab3de166e6f0"
@@ -1737,6 +1744,16 @@ aggregate-error@^3.0.0, aggregate-error@^3.1.0:
     clean-stack "^2.0.0"
     indent-string "^4.0.0"
 
+ajv@*, ajv@^8.11.0:
+  version "8.11.0"
+  resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.11.0.tgz#977e91dd96ca669f54a11e23e378e33b884a565f"
+  integrity sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==
+  dependencies:
+    fast-deep-equal "^3.1.1"
+    json-schema-traverse "^1.0.0"
+    require-from-string "^2.0.2"
+    uri-js "^4.2.2"
+
 ajv@^6.10.0, ajv@^6.12.3, ajv@^6.12.4:
   version "6.12.6"
   resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
@@ -2866,7 +2883,7 @@ enquirer@^2.3.5:
   dependencies:
     ansi-colors "^4.1.1"
 
-entities@^2.0.0:
+entities@^2.0.0, entities@^2.0.3:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55"
   integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==
@@ -4331,6 +4348,11 @@ json-schema-traverse@^0.4.1:
   resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660"
   integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==
 
+json-schema-traverse@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2"
+  integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==
+
 json-schema@0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5"
@@ -6055,6 +6077,11 @@ require-directory@^2.1.1:
   resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
   integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I=
 
+require-from-string@^2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909"
+  integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==
+
 resolve-alpn@^1.0.0:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.2.1.tgz#b7adbdac3546aaaec20b45e7d8265927072726f9"
@@ -6150,6 +6177,14 @@ rollup@~2.37.1:
   optionalDependencies:
     fsevents "~2.1.2"
 
+rss-parser@^3.12.0:
+  version "3.12.0"
+  resolved "https://registry.yarnpkg.com/rss-parser/-/rss-parser-3.12.0.tgz#b8888699ea46304a74363fbd8144671b2997984c"
+  integrity sha512-aqD3E8iavcCdkhVxNDIdg1nkBI17jgqF+9OqPS1orwNaOgySdpvq6B+DoONLhzjzwV8mWg37sb60e4bmLK117A==
+  dependencies:
+    entities "^2.0.3"
+    xml2js "^0.4.19"
+
 run-parallel@^1.1.9:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee"
@@ -6210,6 +6245,11 @@ sass@^1.3.0:
     immutable "^4.0.0"
     source-map-js ">=0.6.2 <2.0.0"
 
+sax@>=0.6.0:
+  version "1.2.4"
+  resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
+  integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
+
 selderee@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.6.0.tgz#f3bee66cfebcb6f33df98e4a1df77388b42a96f7"
@@ -7087,6 +7127,19 @@ ws@^7.3.0:
   resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.6.tgz#e59fc509fb15ddfb65487ee9765c5a51dec5fe7b"
   integrity sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==
 
+xml2js@^0.4.19:
+  version "0.4.23"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
+  integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==
+  dependencies:
+    sax ">=0.6.0"
+    xmlbuilder "~11.0.0"
+
+xmlbuilder@~11.0.0:
+  version "11.0.1"
+  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
+  integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
+
 xtend@^4.0.0:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"