nodejs/hookshot
nodejs/hookshot
1
0
Fork 0
mirror of https://github.com/matrix-org/matrix-hookshot.git synced 2025-03-10 21:19:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #315 from matrix-org/tadzik/rss

Browse Source 
Add RSS/Atom feed support
This commit is contained in:
Tadeusz Sośnierz 2022-04-25 15:26:37 +02:00 committed by GitHub
commit c02951552d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 585 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/315.feature Normal file
View File

@ -0,0 +1 @@
Add RSS/Atom feed support

5
config.sample.yml
View File

@ -80,6 +80,11 @@ generic:
userIdPrefix: _webhooks_
allowJsTransformationFunctions: false
waitForComplete: false
feeds:
# (Optional) Configure this to enable RSS/Atom feed support
#
enabled: false
pollIntervalSeconds: 600
provisioning:
# (Optional) Provisioning API for integration managers
#

1
docs/SUMMARY.md
View File

@ -3,6 +3,7 @@
- [ℹ️ Hookshot](./hookshot.md)
- [⚙️ Setup](./setup.md)
- [📃 Sample Configuration](./setup/sample-configuration.md)
- [Feed](./setup/feeds.md)
- [Figma](./setup/figma.md)
- [GitHub](./setup/github.md)
- [GitLab](./setup/gitlab.md)

14
docs/_site/style.css
View File

@ -29,21 +29,25 @@
/* icons for headers */
.chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(2) strong:after {
content: url('/matrix-hookshot/latest/icons/figma.png')
content: url('/matrix-hookshot/latest/icons/feeds.png')
}
.chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(3) strong:after {
content: url('/matrix-hookshot/latest/icons/github.png')
content: url('/matrix-hookshot/latest/icons/figma.png')
}
.chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(4) strong:after {
content: url('/matrix-hookshot/latest/icons/gitlab.png')
content: url('/matrix-hookshot/latest/icons/github.png')
}
.chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(5) strong:after {
content: url('/matrix-hookshot/latest/icons/jira.png')
content: url('/matrix-hookshot/latest/icons/gitlab.png')
}
.chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(6) strong:after {
content: url('/matrix-hookshot/latest/icons/jira.png')
}
.chapter > li:nth-child(3) > ol:nth-child(1) > li:nth-child(7) strong:after {
content: url('/matrix-hookshot/latest/icons/webhooks.png')
}
}

BIN
docs/icons/feeds.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 760 B

5
docs/metrics.md
View File

@ -30,6 +30,11 @@ Below is the generated list of Prometheus metrics for Hookshot.
| matrix_api_calls | The number of Matrix client API calls made | method |
| matrix_api_calls_failed | The number of Matrix client API calls which failed | method |
| matrix_appservice_events | The number of events sent over the AS API | |
## feed
| Metric | Help | Labels |
|--------|------|--------|
| feed_count | The number of RSS feeds that hookshot is subscribed to | |
| feed_fetch_ms | The time taken for hookshot to fetch all feeds | |
## process
| Metric | Help | Labels |
|--------|------|--------|

2
docs/setup.md
View File

@ -103,6 +103,7 @@ Each permission set can have a services. The `service` field can be:
- `github`
- `gitlab`
- `jira`
- `feed`
- `figma`
- `webhooks`
- `*`, for any service.
@ -204,6 +205,7 @@ in the upstream library. See <a href="https://github.com/turt2live/matrix-bot-sd
You will need to configure some services. Each service has its own documentation file inside the setup subdirectory.
- [Feeds](./setup/feeds.md)
- [Figma](./setup/figma.md)
- [GitHub](./setup/github.md)
- [GitLab](./setup/gitlab.md)

39
docs/setup/feeds.md Normal file
View File

@ -0,0 +1,39 @@
# Feeds
You can configure hookshot to bridge RSS/Atom feeds into Matrix.
## Configuration
```yaml
feeds:
# (Optional) Configure this to enable RSS/Atom feed support
#
enabled: true
pollIntervalSeconds: 600
```
`pollIntervalSeconds` specifies how often each feed will be checked for updates.
It may be checked less often if under exceptional load, but it will never be checked more often than every `pollIntervalSeconds`.
Each feed will only be checked once, regardless of the number of rooms to which it's bridged.
No entries will be bridged upon the “initial sync” -- all entries that exist at the moment of setup will be considered to be already seen.
## Usage
### Adding new feeds
To add a feed to your room:
- Invite the bot user to the room.
- Make sure the bot able to send state events (usually the Moderator power level in clients)
- Say `!hookshot feed <URL>` where `<URL>` links to an RSS/Atom feed you want to subscribe to.
### Listing feeds
You can list all feeds that a room you're in is currently subscribed to with `!hookshot feed list`.
It requires no special permissions from the user issuing the command.
### Removing feeds
To remove a feed from a room, say `!hookshot feed remove <URL>`, with the URL specifying which feed you want to unsubscribe from.

3
package.json
View File

@ -41,6 +41,7 @@
"@octokit/rest": "^18.10.0",
"@octokit/webhooks": "^9.1.2",
"@uiw/react-codemirror": "^4.5.3",
"ajv": "^8.11.0",
"axios": "^0.24.0",
"cors": "^2.8.5",
"express": "^4.17.1",
@ -56,6 +57,7 @@
"node-emoji": "^1.11.0",
"prom-client": "^14.0.1",
"reflect-metadata": "^0.1.13",
"rss-parser": "^3.12.0",
"source-map-support": "^0.5.21",
"string-argv": "^0.3.1",
"uuid": "^8.3.2",
@ -69,6 +71,7 @@
"@prefresh/snowpack": "^3.1.2",
"@snowpack/plugin-sass": "^1.4.0",
"@snowpack/plugin-typescript": "^1.2.1",
"@types/ajv": "^1.0.0",
"@types/chai": "^4.2.22",
"@types/cors": "^2.8.12",
"@types/express": "^4.17.13",

27
src/Bridge.ts
View File

@ -10,7 +10,7 @@ import { GetIssueResponse, GetIssueOpts } from "./Gitlab/Types"
import { GithubInstance } from "./Github/GithubInstance";
import { IBridgeStorageProvider } from "./Stores/StorageProvider";
import { IConnection, GitHubDiscussionSpace, GitHubDiscussionConnection, GitHubUserSpace, JiraProjectConnection, GitLabRepoConnection,
GitHubIssueConnection, GitHubProjectConnection, GitHubRepoConnection, GitLabIssueConnection, FigmaFileConnection } from "./Connections";
GitHubIssueConnection, GitHubProjectConnection, GitHubRepoConnection, GitLabIssueConnection, FigmaFileConnection, FeedConnection } from "./Connections";
import { IGitLabWebhookIssueStateEvent, IGitLabWebhookMREvent, IGitLabWebhookNoteEvent, IGitLabWebhookPushEvent, IGitLabWebhookReleaseEvent, IGitLabWebhookTagPushEvent, IGitLabWebhookWikiPageEvent } from "./Gitlab/WebhookTypes";
import { JiraIssueEvent, JiraIssueUpdatedEvent } from "./Jira/WebhookTypes";
import { JiraOAuthResult } from "./Jira/Types";
@ -42,6 +42,7 @@ import { JiraOAuthRequestCloud, JiraOAuthRequestOnPrem, JiraOAuthRequestResult }
import { CLOUD_INSTANCE } from "./Jira/Client";
import { GenericWebhookEvent, GenericWebhookEventResult } from "./generic/types";
import { SetupWidget } from "./Widgets/SetupWidget";
import { FeedEntry, FeedError, FeedReader } from "./feeds/FeedReader";
const log = new LogWrapper("Bridge");
export class Bridge {
@ -137,6 +138,16 @@ export class Bridge {
await this.tokenStore.load();
const connManager = this.connectionManager = new ConnectionManager(this.as,
this.config, this.tokenStore, this.commentProcessor, this.messageClient, this.storage, this.github);
if (this.config.feeds?.enabled) {
new FeedReader(
this.config.feeds,
this.connectionManager,
this.queue,
this.as.botClient,
);
}
if (this.config.provisioning) {
const routers = [];
@ -196,6 +207,7 @@ export class Bridge {
this.queue.subscribe("gitlab.*");
this.queue.subscribe("jira.*");
this.queue.subscribe("figma.*");
this.queue.subscribe("feed.*");
const validateRepoIssue = (data: GitHubWebhookTypes.IssuesEvent|GitHubWebhookTypes.IssueCommentEvent) => {
if (!data.repository || !data.issue) {
@ -596,6 +608,17 @@ export class Bridge {
(c, data) => c.handleNewComment(data.payload),
)
this.bindHandlerToQueue<FeedEntry, FeedConnection>(
"feed.entry",
(data) => connManager.getConnectionsForFeedUrl(data.feed.url),
(c, data) => c.handleFeedEntry(data),
);
this.bindHandlerToQueue<FeedError, FeedConnection>(
"feed.error",
(data) => connManager.getConnectionsForFeedUrl(data.url),
(c, data) => c.handleFeedError(data),
);
// Set the name and avatar of the bot
if (this.config.bot) {
// Ensure we are registered before we set a profile
@ -1152,4 +1175,4 @@ export class Bridge {
log.debug(`Set up ${roomId} as an admin room for ${adminRoom.userId}`);
return adminRoom;
}
}
}

13
src/Config/Config.ts
View File

@ -184,6 +184,11 @@ export class BridgeConfigGitLab {
}
}
export interface BridgeConfigFeeds {
enabled: boolean;
pollIntervalSeconds: number;
}
export interface BridgeConfigFigma {
publicUrl: string;
overrideUserId?: string;
@ -323,6 +328,7 @@ export interface BridgeConfigRoot {
bot?: BridgeConfigBot;
bridge: BridgeConfigBridge;
figma?: BridgeConfigFigma;
feeds?: BridgeConfigFeeds;
generic?: BridgeGenericWebhooksConfigYAML;
github?: BridgeConfigGitHub;
gitlab?: BridgeConfigGitLabYAML;
@ -362,6 +368,8 @@ export class BridgeConfig {
public readonly generic?: BridgeConfigGenericWebhooks;
@configKey("Configure this to enable Figma support", true)
public readonly figma?: BridgeConfigFigma;
@configKey("Configure this to enable RSS/Atom feed support", true)
public readonly feeds?: BridgeConfigFeeds;
@configKey("Define profile information for the bot user", true)
public readonly bot?: BridgeConfigBot;
@configKey("EXPERIMENTAL support for complimentary widgets", true)
@ -395,6 +403,7 @@ export class BridgeConfig {
this.figma = configData.figma;
this.jira = configData.jira && new BridgeConfigJira(configData.jira);
this.generic = configData.generic && new BridgeConfigGenericWebhooks(configData.generic);
this.feeds = configData.feeds;
this.provisioning = configData.provisioning;
this.passFile = configData.passFile;
this.bot = configData.bot;
@ -428,8 +437,8 @@ export class BridgeConfig {
log.warn(`You have not configured any permissions for the bridge, which by default means all users on ${this.bridge.domain} have admin levels of control. Please adjust your config.`);
}
if (!this.github && !this.gitlab && !this.jira && !this.generic && !this.figma) {
throw Error("Config is not valid: At least one of GitHub, GitLab, JIRA, Figma or generic hooks must be configured");
if (!this.github && !this.gitlab && !this.jira && !this.generic && !this.figma && !this.feeds) {
throw Error("Config is not valid: At least one of GitHub, GitLab, JIRA, Figma, feeds or generic hooks must be configured");
}
// TODO: Formalize env support

4
src/Config/Defaults.ts
View File

@ -106,6 +106,10 @@ export const DefaultConfig = new BridgeConfig({
}
}
},
feeds: {
enabled: false,
pollIntervalSeconds: 600,
},
provisioning: {
secret: "!secretToken"
},

26
src/ConnectionManager.ts
View File

@ -18,13 +18,14 @@ import { GetConnectionTypeResponseItem } from "./provisioning/api";
import { ApiError, ErrCode } from "./api";
import { UserTokenStore } from "./UserTokenStore";
import {v4 as uuid} from "uuid";
import { FigmaFileConnection } from "./Connections/FigmaFileConnection";
import { FigmaFileConnection, FeedConnection } from "./Connections";
import { IBridgeStorageProvider } from "./Stores/StorageProvider";
import Metrics from "./Metrics";
import EventEmitter from "events";
const log = new LogWrapper("ConnectionManager");
export class ConnectionManager {
export class ConnectionManager extends EventEmitter {
private connections: IConnection[] = [];
public readonly enabledForProvisioning: Record<string, GetConnectionTypeResponseItem> = {};
@ -39,8 +40,9 @@ export class ConnectionManager {
private readonly commentProcessor: CommentProcessor,
private readonly messageClient: MessageSenderClient,
private readonly storage: IBridgeStorageProvider,
private readonly github?: GithubInstance) {
private readonly github?: GithubInstance
) {
super();
}
/**
@ -54,6 +56,7 @@ export class ConnectionManager {
for (const connection of connections) {
if (!this.connections.find(c => c.connectionId === connection.connectionId)) {
this.connections.push(connection);
this.emit('new-connection', connection);
}
}
Metrics.connections.set(this.connections.length);
@ -125,7 +128,7 @@ export class ConnectionManager {
throw new ApiError(`Connection type not known`);
}
private assertStateAllowed(state: StateEvent<any>, serviceType: "github"|"gitlab"|"jira"|"figma"|"webhooks") {
private assertStateAllowed(state: StateEvent<any>, serviceType: "github"|"gitlab"|"jira"|"figma"|"webhooks"|"feed") {
if (state.sender === this.as.botUserId) {
return;
}
@ -245,6 +248,14 @@ export class ConnectionManager {
return new FigmaFileConnection(roomId, state.stateKey, state.content, this.config.figma, this.as, this.storage);
}
if (FeedConnection.EventTypes.includes(state.type)) {
if (!this.config.feeds?.enabled) {
throw Error('RSS/Atom feeds are not configured');
}
this.assertStateAllowed(state, "feed");
return new FeedConnection(roomId, state.stateKey, state.content, this.config.feeds, this.as, this.storage);
}
if (GenericHookConnection.EventTypes.includes(state.type) && this.config.generic?.enabled) {
if (!this.config.generic) {
throw Error('Generic webhooks are not configured');
@ -374,6 +385,10 @@ export class ConnectionManager {
public getForFigmaFile(fileKey: string, instanceName: string): FigmaFileConnection[] {
return this.connections.filter((c) => (c instanceof FigmaFileConnection && (c.fileId === fileKey || c.instanceName === instanceName))) as FigmaFileConnection[];
}
public getConnectionsForFeedUrl(url: string): FeedConnection[] {
return this.connections.filter(c => c instanceof FeedConnection && c.feedUrl === url) as FeedConnection[];
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
public getAllConnectionsOfType<T extends IConnection>(typeT: new (...params : any[]) => T): T[] {
@ -411,6 +426,7 @@ export class ConnectionManager {
}
this.connections.splice(connectionIndex, 1);
Metrics.connections.set(this.connections.length);
this.emit('connection-removed', connection);
}
/**

84
src/Connections/FeedConnection.ts Normal file
View File

@ -0,0 +1,84 @@
import {Appservice} from "matrix-bot-sdk";
import { IConnection, IConnectionState } from ".";
import { BridgeConfigFeeds } from "../Config/Config";
import { FeedEntry, FeedError} from "../feeds/FeedReader";
import LogWrapper from "../LogWrapper";
import { IBridgeStorageProvider } from "../Stores/StorageProvider";
import { BaseConnection } from "./BaseConnection";
import markdown from "markdown-it";
const log = new LogWrapper("FeedConnection");
const md = new markdown();
export interface FeedConnectionState extends IConnectionState {
url: string;
}
export class FeedConnection extends BaseConnection implements IConnection {
static readonly CanonicalEventType = "uk.half-shot.matrix-hookshot.feed";
static readonly EventTypes = [ FeedConnection.CanonicalEventType ];
private hasError = false;
public get feedUrl(): string {
return this.state.url;
}
constructor(
roomId: string,
stateKey: string,
private state: FeedConnectionState,
private readonly config: BridgeConfigFeeds,
private readonly as: Appservice,
private readonly storage: IBridgeStorageProvider
) {
super(roomId, stateKey, FeedConnection.CanonicalEventType)
log.info(`Connection ${this.connectionId} created for ${roomId}, ${JSON.stringify(state)}`);
}
public isInterestedInStateEvent(eventType: string, stateKey: string): boolean {
return !!FeedConnection.EventTypes.find(e => e === eventType) && stateKey === this.feedUrl;
}
public async handleFeedEntry(entry: FeedEntry): Promise<void> {
this.hasError = false;
let entryDetails;
if (entry.title && entry.link) {
entryDetails = `[${entry.title}](${entry.link})`;
} else {
entryDetails = entry.title || entry.link;
}
let message = `New post in ${entry.feed.title || entry.feed.url}`;
if (entryDetails) {
message += `: ${entryDetails}`;
}
await this.as.botIntent.sendEvent(this.roomId, {
msgtype: 'm.notice',
format: "org.matrix.custom.html",
formatted_body: md.renderInline(message),
body: message,
});
}
public async handleFeedError(error: FeedError): Promise<void> {
if (!this.hasError) {
await this.as.botIntent.sendEvent(this.roomId, {
msgtype: 'm.notice',
format: 'm.text',
body: `Error fetching ${this.feedUrl}: ${error.cause.message}`
});
this.hasError = true;
}
}
// needed to ensure that the connection is removable
public async onRemove(): Promise<void> {
log.info(`Removing connection ${this.connectionId}`);
}
toString(): string {
return `FeedConnection ${this.state.url}`;
}
}

4
src/Connections/IConnection.ts
View File

@ -64,7 +64,7 @@ export interface IConnection {
* If supported, this is sent when a user attempts to remove the connection from a room. The connection
* state should be removed and any resources should be cleaned away.
*/
onRemove?: () => void;
onRemove?: () => Promise<void>;
toString(): string;
}
}

119
src/Connections/SetupConnection.ts
View File

@ -10,6 +10,7 @@ import { v4 as uuid } from "uuid";
import { BridgeConfig, BridgePermissionLevel } from "../Config/Config";
import markdown from "markdown-it";
import { FigmaFileConnection } from "./FigmaFileConnection";
import { FeedConnection } from "./FeedConnection";
import { URL } from "url";
import { SetupWidget } from "../Widgets/SetupWidget";
import { AdminRoom } from "../AdminRoom";
@ -45,6 +46,7 @@ export class SetupConnection extends CommandConnection {
this.config.figma ? "figma": "",
this.config.jira ? "jira": "",
this.config.generic?.enabled ? "webhook": "",
this.config.feeds?.enabled ? "feed" : "",
this.config.widgets?.roomSetupWidget ? "widget" : "",
];
this.includeTitlesInHelp = false;
@ -55,15 +57,9 @@ export class SetupConnection extends CommandConnection {
if (!this.githubInstance || !this.config.github) {
throw new CommandError("not-configured", "The bridge is not configured to support GitHub.");
}
if (!this.config.checkPermission(userId, "github", BridgePermissionLevel.manageConnections)) {
throw new CommandError('You are not permitted to provision connections for GitHub.');
}
if (!await this.as.botClient.userHasPowerLevelFor(userId, this.roomId, "", true)) {
throw new CommandError("not-configured", "You must be able to set state in a room ('Change settings') in order to set up new integrations.");
}
if (!await this.as.botClient.userHasPowerLevelFor(this.as.botUserId, this.roomId, GitHubRepoConnection.CanonicalEventType, true)) {
throw new CommandError("Bot lacks power level to set room state", "I do not have permission to set up a bridge in this room. Please promote me to an Admin/Moderator.");
}
await this.checkUserPermissions(userId, "github", GitHubRepoConnection.CanonicalEventType);
const octokit = await this.tokenStore.getOctokitForUser(userId);
if (!octokit) {
throw new CommandError("User not logged in", "You are not logged into GitHub. Start a DM with this bot and use the command `github login`.");
@ -84,15 +80,9 @@ export class SetupConnection extends CommandConnection {
if (!this.config.jira) {
throw new CommandError("not-configured", "The bridge is not configured to support Jira.");
}
if (!this.config.checkPermission(userId, "jira", BridgePermissionLevel.manageConnections)) {
throw new CommandError('You are not permitted to provision connections for Jira.');
}
if (!await this.as.botClient.userHasPowerLevelFor(userId, this.roomId, "", true)) {
throw new CommandError("not-configured", "You must be able to set state in a room ('Change settings') in order to set up new integrations.");
}
if (!await this.as.botClient.userHasPowerLevelFor(this.as.botUserId, this.roomId, GitHubRepoConnection.CanonicalEventType, true)) {
throw new CommandError("Bot lacks power level to set room state", "I do not have permission to set up a bridge in this room. Please promote me to an Admin/Moderator.");
}
await this.checkUserPermissions(userId, "jira", JiraProjectConnection.CanonicalEventType);
const jiraClient = await this.tokenStore.getJiraForUser(userId, urlStr);
if (!jiraClient) {
throw new CommandError("User not logged in", "You are not logged into Jira. Start a DM with this bot and use the command `jira login`.");
@ -113,15 +103,9 @@ export class SetupConnection extends CommandConnection {
if (!this.config.generic?.enabled) {
throw new CommandError("not-configured", "The bridge is not configured to support webhooks.");
}
if (!this.config.checkPermission(userId, "webhooks", BridgePermissionLevel.manageConnections)) {
throw new CommandError('You are not permitted to provision connections for generic webhooks.');
}
if (!await this.as.botClient.userHasPowerLevelFor(userId, this.roomId, "", true)) {
throw new CommandError("not-configured", "You must be able to set state in a room ('Change settings') in order to set up new integrations.");
}
if (!await this.as.botClient.userHasPowerLevelFor(this.as.botUserId, this.roomId, GitHubRepoConnection.CanonicalEventType, true)) {
throw new CommandError("Bot lacks power level to set room state", "I do not have permission to set up a bridge in this room. Please promote me to an Admin/Moderator.");
}
await this.checkUserPermissions(userId, "webhooks", GitHubRepoConnection.CanonicalEventType);
if (!name || name.length < 3 || name.length > 64) {
throw new CommandError("Bad webhook name", "A webhook name must be between 3-64 characters.");
}
@ -139,15 +123,9 @@ export class SetupConnection extends CommandConnection {
if (!this.config.figma) {
throw new CommandError("not-configured", "The bridge is not configured to support Figma.");
}
if (!this.config.checkPermission(userId, "figma", BridgePermissionLevel.manageConnections)) {
throw new CommandError('You are not permitted to provision connections for Figma.');
}
if (!await this.as.botClient.userHasPowerLevelFor(userId, this.roomId, "", true)) {
throw new CommandError("not-configured", "You must be able to set state in a room ('Change settings') in order to set up new integrations.");
}
if (!await this.as.botClient.userHasPowerLevelFor(this.as.botUserId, this.roomId, GitHubRepoConnection.CanonicalEventType, true)) {
throw new CommandError("Bot lacks power level to set room state", "I do not have permission to set up a bridge in this room. Please promote me to an Admin/Moderator.");
}
await this.checkUserPermissions(userId, "figma", FigmaFileConnection.CanonicalEventType);
const res = /https:\/\/www\.figma\.com\/file\/(\w+).+/.exec(url);
if (!res) {
throw new CommandError("Invalid Figma url", "The Figma file url you entered was not valid. It should be in the format of `https://figma.com/file/FILEID/...`.");
@ -157,6 +135,63 @@ export class SetupConnection extends CommandConnection {
return this.as.botClient.sendHtmlNotice(this.roomId, md.renderInline(`Room configured to bridge Figma file.`));
}
@botCommand("feed", { help: "Bridge an RSS/Atom feed to the room.", requiredArgs: ["url"], includeUserId: true, category: "feed"})
public async onFeed(userId: string, url: string) {
if (!this.config.feeds?.enabled) {
throw new CommandError("not-configured", "The bridge is not configured to support feeds.");
}
await this.checkUserPermissions(userId, "feed", FeedConnection.CanonicalEventType);
try {
new URL(url);
// TODO: fetch and check content-type?
} catch {
throw new CommandError("Invalid URL", `${url} doesn't look like a valid feed URL`);
}
await this.as.botClient.sendStateEvent(this.roomId, FeedConnection.CanonicalEventType, url, {url});
return this.as.botClient.sendHtmlNotice(this.roomId, md.renderInline(`Room configured to bridge \`${url}\``));
}
@botCommand("feed list", { help: "Show feeds currently subscribed to.", category: "feed"})
public async onFeedList() {
const urls = await this.as.botClient.getRoomState(this.roomId).catch((err: any) => {
if (err.body.errcode === 'M_NOT_FOUND') {
return []; // not an error to us
}
throw err;
}).then(events =>
events.filter(
(ev: any) => ev.type === FeedConnection.CanonicalEventType && ev.content.url
).map(ev => ev.content.url)
);
if (urls.length === 0) {
return this.as.botClient.sendHtmlNotice(this.roomId, md.renderInline('Not subscribed to any feeds'));
} else {
return this.as.botClient.sendHtmlNotice(this.roomId, md.render(`Currently subscribed to these feeds:\n\n${urls.map(url => ' * ' + url + '\n')}`));
}
}
@botCommand("feed remove", { help: "Unsubscribe from an RSS/Atom.", requiredArgs: ["url"], includeUserId: true, category: "feed"})
public async onFeedRemove(userId: string, url: string) {
await this.checkUserPermissions(userId, "feed", FeedConnection.CanonicalEventType);
const event = await this.as.botClient.getRoomStateEvent(this.roomId, FeedConnection.CanonicalEventType, url).catch((err: any) => {
if (err.body.errcode === 'M_NOT_FOUND') {
return null; // not an error to us
}
throw err;
});
if (!event || Object.keys(event).length === 0) {
throw new CommandError("Invalid feed URL", `Feed "${url}" is not currently bridged to this room`);
}
await this.as.botClient.sendStateEvent(this.roomId, FeedConnection.CanonicalEventType, url, {});
return this.as.botClient.sendHtmlNotice(this.roomId, md.renderInline(`Unsubscribed from \`${url}\``));
}
@botCommand("setup-widget", {category: "widget", help: "Open the setup widget in the room"})
public async onSetupWidget() {
if (!this.config.widgets?.roomSetupWidget) {
@ -166,6 +201,18 @@ export class SetupConnection extends CommandConnection {
await this.as.botClient.sendNotice(this.roomId, `This room already has a setup widget, please open the "Hookshot Configuration" widget.`);
}
}
private async checkUserPermissions(userId: string, service: string, stateEventType: string): Promise<void> {
if (!this.config.checkPermission(userId, service, BridgePermissionLevel.manageConnections)) {
throw new CommandError(`You are not permitted to provision connections for ${service}.`);
}
if (!await this.as.botClient.userHasPowerLevelFor(userId, this.roomId, "", true)) {
throw new CommandError("not-configured", "You must be able to set state in a room ('Change settings') in order to set up new integrations.");
}
if (!await this.as.botClient.userHasPowerLevelFor(this.as.botUserId, this.roomId, stateEventType, true)) {
throw new CommandError("Bot lacks power level to set room state", "I do not have permission to set up a bridge in this room. Please promote me to an Admin/Moderator.");
}
}
}
// Typescript doesn't understand Prototypes very well yet.

3
src/Connections/index.ts
View File

@ -9,4 +9,5 @@ export * from "./GitlabIssue";
export * from "./GitlabRepo";
export * from "./IConnection";
export * from "./JiraProject";
export * from "./FigmaFileConnection";
export * from "./FigmaFileConnection";
export * from "./FeedConnection";

2
src/Jira/oauth/OnPremOAuth.ts
View File

@ -132,7 +132,7 @@ export class JiraOnPremOAuth implements JiraOAuth {
private prepareParameters(oauthToken: string|null, method: Method, urlStr: string, extraParams: Record<string, string> = {}) {
const oauthParameters: Record<string, string> = {
oauth_timestamp: Math.floor( (new Date()).getTime() / 1000 ).toString(),
oauth_timestamp: Math.floor( Date.now() / 1000 ).toString(),
oauth_nonce: JiraOnPremOAuth.nonce(),
oauth_version: "1.0",
oauth_signature_method: "RSA-SHA1",

4
src/Metrics.ts
View File

@ -23,6 +23,10 @@ export class Metrics {
public readonly matrixAppserviceEvents = new Counter({ name: "matrix_appservice_events", help: "The number of events sent over the AS API", labelNames: [], registers: [this.registry]});
public readonly feedsCount = new Gauge({ name: "feed_count", help: "The number of RSS feeds that hookshot is subscribed to", labelNames: [], registers: [this.registry]});
public readonly feedFetchMs = new Gauge({ name: "feed_fetch_ms", help: "The time taken for hookshot to fetch all feeds", labelNames: [], registers: [this.registry]});
constructor(private registry: Registry = register) {
this.expressRouter.get('/metrics', this.metricsFunc.bind(this));
collectDefaultMetrics({

229
src/feeds/FeedReader.ts Normal file
View File

@ -0,0 +1,229 @@
import { MatrixClient } from "matrix-bot-sdk";
import { BridgeConfigFeeds } from "../Config/Config";
import { ConnectionManager } from "../ConnectionManager";
import { FeedConnection } from "../Connections";
import LogWrapper from "../LogWrapper";
import { MessageQueue } from "../MessageQueue";
import Ajv from "ajv";
import axios from "axios";
import Parser from "rss-parser";
import Metrics from "../Metrics";
const log = new LogWrapper("FeedReader");
export class FeedError extends Error {
constructor(
public url: string,
public cause: Error,
) {
super(`Error fetching feed ${url}: ${cause.message}`);
}
}
export interface FeedEntry {
feed: {
title: string|null,
url: string,
},
title: string|null,
link: string|null,
}
interface AccountData {
[url: string]: string[],
}
const accountDataSchema = {
type: 'object',
patternProperties: {
"https?://.+": {
type: 'array',
items: { type: 'string' },
}
},
additionalProperties: false,
};
const ajv = new Ajv();
const validateAccountData = ajv.compile<AccountData>(accountDataSchema);
function stripHtml(input: string): string {
return input.replace(/<[^>]*?>/g, '');
}
function normalizeUrl(input: string): string {
const url = new URL(input);
url.hash = '';
return url.toString();
}
export class FeedReader {
private connections: FeedConnection[];
// ts should notice that we do in fact initialize it in constructor, but it doesn't (in this version)
private observedFeedUrls: Set<string> = new Set();
private seenEntries: Map<string, string[]> = new Map();
static readonly seenEntriesEventType = "uk.half-shot.matrix-hookshot.feed.reader.seenEntries";
constructor(
private config: BridgeConfigFeeds,
private connectionManager: ConnectionManager,
private queue: MessageQueue,
private matrixClient: MatrixClient,
) {
this.connections = this.connectionManager.getAllConnectionsOfType(FeedConnection);
this.calculateFeedUrls();
connectionManager.on('new-connection', c => {
if (c instanceof FeedConnection) {
log.info('New connection tracked:', c.connectionId);
this.connections.push(c);
this.calculateFeedUrls();
}
});
connectionManager.on('connection-removed', removed => {
if (removed instanceof FeedConnection) {
log.info('Connections before removal:', this.connections.map(c => c.connectionId));
this.connections = this.connections.filter(c => c.connectionId !== removed.connectionId);
log.info('Connections after removal:', this.connections.map(c => c.connectionId));
this.calculateFeedUrls();
}
});
log.info('Loaded feed URLs:', this.observedFeedUrls);
void this.loadSeenEntries().then(() => {
return this.pollFeeds();
});
}
private calculateFeedUrls(): void {
// just in case we got an invalid URL somehow
const normalizedUrls = [];
for (const conn of this.connections) {
try {
normalizedUrls.push(normalizeUrl(conn.feedUrl));
} catch (err: unknown) {
log.error(`Invalid feedUrl for connection ${conn.connectionId}: ${conn.feedUrl}. It will not be tracked`);
}
}
this.observedFeedUrls = new Set(normalizedUrls);
Metrics.feedsCount.set(this.observedFeedUrls.size);
}
private async loadSeenEntries(): Promise<void> {
try {
const accountData = await this.matrixClient.getAccountData<any>(FeedReader.seenEntriesEventType).catch((err: any) => {
if (err.statusCode === 404) {
return {};
} else {
throw err;
}
});
if (!validateAccountData(accountData)) {
const errors = validateAccountData.errors!.map(e => `${e.instancePath} ${e.message}`);
throw new Error(`Invalid account data: ${errors.join(', ')}`);
}
for (const url in accountData) {
this.seenEntries.set(url, accountData[url]);
}
} catch (err: unknown) {
log.error(`Failed to load seen feed entries from accountData: ${err}. This may result in skipped entries`);
// no need to wipe it manually, next saveSeenEntries() will make it right
}
}
private async saveSeenEntries(): Promise<void> {
const accountData: AccountData = {};
for (const [url, guids] of this.seenEntries.entries()) {
accountData[url.toString()] = guids;
}
await this.matrixClient.setAccountData(FeedReader.seenEntriesEventType, accountData);
}
private async pollFeeds(): Promise<void> {
log.debug(`Checking for updates in ${this.observedFeedUrls.size} RSS/Atom feeds`);
let seenEntriesChanged = false;
const fetchingStarted = Date.now();
for (const url of this.observedFeedUrls.values()) {
try {
const res = await axios.get(url.toString());
const feed = await (new Parser()).parseString(res.data);
let initialSync = false;
let seenGuids = this.seenEntries.get(url);
if (!seenGuids) {
initialSync = true;
seenGuids = [];
seenEntriesChanged = true; // to ensure we only treat it as an initialSync once
}
const seenGuidsSet = new Set(seenGuids);
const newGuids = [];
log.debug(`Found ${feed.items.length} entries in ${url}`);
for (const item of feed.items) {
const guid = item.guid || item.id || item.link || item.title;
if (!guid) {
log.error(`Could not determine guid for entry in ${url}, skipping`);
continue;
}
newGuids.push(guid);
if (initialSync) {
log.debug(`Skipping entry ${guid} since we're performing an initial sync`);
continue;
}
if (seenGuidsSet.has(guid)) {
log.debug('Skipping already seen entry', guid);
continue;
}
const entry = {
feed: {
title: feed.title ? stripHtml(feed.title) : null,
url: url.toString()
},
title: item.title ? stripHtml(item.title) : null,
link: item.link || null,
};
log.debug('New entry:', entry);
seenEntriesChanged = true;
this.queue.push<FeedEntry>({ eventName: 'feed.entry', sender: 'FeedReader', data: entry });
}
if (seenEntriesChanged) {
// Some RSS feeds can return a very small number of items then bounce
// back to their "normal" size, so we cannot just clobber the recent GUID list per request or else we'll
// forget what we sent and resend it. Instead, we'll keep 2x the max number of items that we've ever
// seen from this feed, up to a max of 10,000.
// Adopted from https://github.com/matrix-org/go-neb/blob/babb74fa729882d7265ff507b09080e732d060ae/services/rssbot/rssbot.go#L304
const maxGuids = Math.min(Math.max(2 * newGuids.length, seenGuids.length), 10_000);
const newSeenItems = Array.from(new Set([ ...newGuids, ...seenGuids ]).values()).slice(0, maxGuids);
this.seenEntries.set(url, newSeenItems);
}
} catch (err: any) {
const error = new FeedError(url.toString(), err);
log.error(error.message);
this.queue.push<FeedError>({ eventName: 'feed.error', sender: 'FeedReader', data: error });
}
}
if (seenEntriesChanged) await this.saveSeenEntries();
const elapsed = Date.now() - fetchingStarted;
Metrics.feedFetchMs.set(elapsed);
let sleepFor: number;
if (elapsed > this.config.pollIntervalSeconds * 1000) {
log.warn(`It tooks us longer to update the feeds than the configured pool interval (${elapsed / 1000}s)`);
sleepFor = 0;
} else {
sleepFor = this.config.pollIntervalSeconds * 1000 - elapsed;
log.debug(`Feed fetching took ${elapsed / 1000}s, sleeping for ${sleepFor / 1000}s`);
}
setTimeout(() => {
void this.pollFeeds();
}, sleepFor);
}
}

55
yarn.lock
View File

@ -1288,6 +1288,13 @@
"@napi-rs/cli" "^2.2.0"
shelljs "^0.8.4"
"@types/ajv@^1.0.0":
version "1.0.0"
resolved "https://registry.yarnpkg.com/@types/ajv/-/ajv-1.0.0.tgz#4fb2440742f2f6c30e7fb0797b839fc6f696682a"
integrity sha1-T7JEB0Ly9sMOf7B5e4OfxvaWaCo=
dependencies:
ajv "*"
"@types/body-parser@*":
version "1.19.2"
resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.2.tgz#aea2059e28b7658639081347ac4fab3de166e6f0"
@ -1737,6 +1744,16 @@ aggregate-error@^3.0.0, aggregate-error@^3.1.0:
clean-stack "^2.0.0"
indent-string "^4.0.0"
ajv@*, ajv@^8.11.0:
version "8.11.0"
resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.11.0.tgz#977e91dd96ca669f54a11e23e378e33b884a565f"
integrity sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==
dependencies:
fast-deep-equal "^3.1.1"
json-schema-traverse "^1.0.0"
require-from-string "^2.0.2"
uri-js "^4.2.2"
ajv@^6.10.0, ajv@^6.12.3, ajv@^6.12.4:
version "6.12.6"
resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"
@ -2866,7 +2883,7 @@ enquirer@^2.3.5:
dependencies:
ansi-colors "^4.1.1"
entities@^2.0.0:
entities@^2.0.0, entities@^2.0.3:
version "2.2.0"
resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55"
integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==
@ -4331,6 +4348,11 @@ json-schema-traverse@^0.4.1:
resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660"
integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==
json-schema-traverse@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2"
integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==
json-schema@0.4.0:
version "0.4.0"
resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5"
@ -6055,6 +6077,11 @@ require-directory@^2.1.1:
resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I=
require-from-string@^2.0.2:
version "2.0.2"
resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909"
integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==
resolve-alpn@^1.0.0:
version "1.2.1"
resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.2.1.tgz#b7adbdac3546aaaec20b45e7d8265927072726f9"
@ -6150,6 +6177,14 @@ rollup@~2.37.1:
optionalDependencies:
fsevents "~2.1.2"
rss-parser@^3.12.0:
version "3.12.0"
resolved "https://registry.yarnpkg.com/rss-parser/-/rss-parser-3.12.0.tgz#b8888699ea46304a74363fbd8144671b2997984c"
integrity sha512-aqD3E8iavcCdkhVxNDIdg1nkBI17jgqF+9OqPS1orwNaOgySdpvq6B+DoONLhzjzwV8mWg37sb60e4bmLK117A==
dependencies:
entities "^2.0.3"
xml2js "^0.4.19"
run-parallel@^1.1.9:
version "1.2.0"
resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee"
@ -6210,6 +6245,11 @@ sass@^1.3.0:
immutable "^4.0.0"
source-map-js ">=0.6.2 <2.0.0"
sax@>=0.6.0:
version "1.2.4"
resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
selderee@^0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.6.0.tgz#f3bee66cfebcb6f33df98e4a1df77388b42a96f7"
@ -7087,6 +7127,19 @@ ws@^7.3.0:
resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.6.tgz#e59fc509fb15ddfb65487ee9765c5a51dec5fe7b"
integrity sha512-6GLgCqo2cy2A2rjCNFlxQS6ZljG/coZfZXclldI8FB/1G3CCI36Zd8xy2HrFVACi8tfk5XrgLQEk+P0Tnz9UcA==
xml2js@^0.4.19:
version "0.4.23"
resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==
dependencies:
sax ">=0.6.0"
xmlbuilder "~11.0.0"
xmlbuilder@~11.0.0:
version "11.0.1"
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
xtend@^4.0.0:
version "4.0.2"
resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"