Add support for templating feed messages (#702)

* Add support for specifying a template for feeds

* Remove unused params

* use the correct url

* make URL visible.

It was annoying me

* Update src/Connections/FeedConnection.ts

Co-authored-by: Tadeusz Sośnierz <tadzik@tadzik.net>

* Support migrations

* Hopefully support migrations now

* lint

---------

Co-authored-by: Tadeusz Sośnierz <tadzik@tadzik.net>

changelog.d/702.feature

@@ -0,0 +1 @@
+Add support for specifying custom templates for feeds.

docs/setup/feeds.md

@@ -37,3 +37,22 @@ It requires no special permissions from the user issuing the command.
 ### Removing feeds
 
 To remove a feed from a room, say `!hookshot feed remove <URL>`, with the URL specifying which feed you want to unsubscribe from.
+
+
+### Feed templates
+
+You can optionally give a feed a specific template to use when sending a message into a room. A template
+may include any of the following tokens:
+
+|Token     |Description                                 |
+|----------|--------------------------------------------|
+|$FEEDNAME | Either the label, title or url of the feed.|
+|$FEEDURL  | The URL of the feed.                       |
+|$FEEDTITLE| The title of the feed.                     |
+|$TITLE    | The title of the feed entry.               |
+|$LINK     | The link of the feed entry.                |
+|$AUTHOR   | The author of the feed entry.              |
+|$DATE     | The publish date (`pubDate`) of the entry. |
+|$SUMMARY  | The summary of the entry.                  |
+
+If not specified, the default template is `New post in $FEEDNAME: $LINK`.

src/Connections/FeedConnection.ts

@@ -1,10 +1,8 @@
-import {Appservice, Intent, StateEvent} from "matrix-bot-sdk";
+import {Intent, StateEvent} from "matrix-bot-sdk";
 import { IConnection, IConnectionState, InstantiateConnectionOpts } from ".";
 import { ApiError, ErrCode } from "../api";
-import { BridgeConfigFeeds } from "../Config/Config";
 import { FeedEntry, FeedError, FeedReader} from "../feeds/FeedReader";
 import { Logger } from "matrix-appservice-bridge";
-import { IBridgeStorageProvider } from "../Stores/StorageProvider";
 import { BaseConnection } from "./BaseConnection";
 import markdown from "markdown-it";
 import { Connection, ProvisionConnectionOpts } from "./IConnection";
@@ -26,6 +24,7 @@ export interface LastResultFail {
 export interface FeedConnectionState extends IConnectionState {
     url:    string;
     label?: string;
+    template?: string;
 }
 
 export interface FeedConnectionSecrets {
@@ -36,6 +35,11 @@ export type FeedResponseItem = GetConnectionsResponseItem<FeedConnectionState, F
 
 const MAX_LAST_RESULT_ITEMS = 5;
 const VALIDATION_FETCH_TIMEOUT_MS = 5000;
+const MAX_SUMMARY_LENGTH = 512;
+const MAX_TEMPLATE_LENGTH = 1024;
+
+const DEFAULT_TEMPLATE = "New post in $FEEDNAME";
+const DEFAULT_TEMPLATE_WITH_CONTENT = "New post in $FEEDNAME: $LINK"
 
 @Connection
 export class FeedConnection extends BaseConnection implements IConnection {
@@ -43,11 +47,12 @@ export class FeedConnection extends BaseConnection implements IConnection {
     static readonly EventTypes = [ FeedConnection.CanonicalEventType ];
     static readonly ServiceCategory = "feeds";
     
-    public static createConnectionForState(roomId: string, event: StateEvent<any>, {config, as, intent, storage}: InstantiateConnectionOpts) {
+
+    public static createConnectionForState(roomId: string, event: StateEvent<any>, {config, intent}: InstantiateConnectionOpts) {
         if (!config.feeds?.enabled) {
             throw Error('RSS/Atom feeds are not configured');
         }
-        return new FeedConnection(roomId, event.stateKey, event.content, config.feeds, as, intent, storage);
+        return new FeedConnection(roomId, event.stateKey, event.content, intent);
     }
 
     static async validateUrl(url: string): Promise<void> {
@@ -64,24 +69,38 @@ export class FeedConnection extends BaseConnection implements IConnection {
         }
     }
 
-    static async provisionConnection(roomId: string, _userId: string, data: Record<string, unknown> = {}, {as, intent, config, storage}: ProvisionConnectionOpts) {
+    static validateState(data: Record<string, unknown> = {}): FeedConnectionState {
-        if (!config.feeds?.enabled) {
-            throw new ApiError('RSS/Atom feeds are not configured', ErrCode.DisabledFeature);
-        }
-
         const url = data.url;
         if (typeof url !== 'string') {
             throw new ApiError('No URL specified', ErrCode.BadValue);
         }
-        await FeedConnection.validateUrl(url);
         if (typeof data.label !== 'undefined' && typeof data.label !== 'string') {
             throw new ApiError('Label must be a string', ErrCode.BadValue);
         }
 
-        const state = { url, label: data.label };
+        if (typeof data.template !== 'undefined') {
+            if (typeof data.template !== 'string') {
+                throw new ApiError('Template must be a string', ErrCode.BadValue);
+            }
+            // Sanity to prevent slowing hookshot down with massive templates.
+            if (data.template.length > MAX_TEMPLATE_LENGTH) {
+                throw new ApiError(`Template should not be longer than ${MAX_TEMPLATE_LENGTH} characters`, ErrCode.BadValue);
+            }
+        }
 
-        const connection = new FeedConnection(roomId, url, state, config.feeds, as, intent, storage);
+
-        await intent.underlyingClient.sendStateEvent(roomId, FeedConnection.CanonicalEventType, url, state);
+        return { url, label: data.label, template: data.template };
+    }
+
+    static async provisionConnection(roomId: string, _userId: string, data: Record<string, unknown> = {}, { intent, config }: ProvisionConnectionOpts) {
+        if (!config.feeds?.enabled) {
+            throw new ApiError('RSS/Atom feeds are not configured', ErrCode.DisabledFeature);
+        }
+
+        const state = this.validateState(data);
+        await FeedConnection.validateUrl(state.url);
+        const connection = new FeedConnection(roomId, state.url, state, intent);
+        await intent.underlyingClient.sendStateEvent(roomId, FeedConnection.CanonicalEventType, state.url, state);
 
         return {
             connection,
@@ -112,6 +131,31 @@ export class FeedConnection extends BaseConnection implements IConnection {
         }
     }
 
+    public templateFeedEntry(template: string, entry: FeedEntry) {
+        return template.replace(/(\$[A-Z]+)/g, (token: string) => {
+            switch(token) {
+                case "$FEEDNAME":
+                    return this.state.label || entry.feed.title || entry.feed.url || "";
+                case "$FEEDURL":
+                    return entry.feed.url || "";
+                case "$FEEDTITLE":
+                    return entry.feed.title || "";
+                case "$TITLE":
+                    return entry.title || "";
+                case "$LINK":
+                    return entry.link ? `[${entry.title ?? entry.link}](${entry.link})` : "";
+                case "$AUTHOR":
+                    return entry.author || "";
+                case "$DATE":
+                    return entry.pubdate || "";
+                case "$SUMMARY":
+                    return entry.summary || "";
+                default:
+                    return token;
+            }
+        });
+    }
+
     private hasError = false;
     private readonly lastResults = new Array<LastResultOk|LastResultFail>();
 
@@ -123,10 +167,7 @@ export class FeedConnection extends BaseConnection implements IConnection {
         roomId: string,
         stateKey: string,
         private state: FeedConnectionState,
-        private readonly config: BridgeConfigFeeds,
-        private readonly as: Appservice,
         private readonly intent: Intent,
-        private readonly storage: IBridgeStorageProvider
     ) {
         super(roomId, stateKey, FeedConnection.CanonicalEventType)
         log.info(`Connection ${this.connectionId} created for ${roomId}, ${JSON.stringify(state)}`);
@@ -138,16 +179,19 @@ export class FeedConnection extends BaseConnection implements IConnection {
 
     public async handleFeedEntry(entry: FeedEntry): Promise<void> {
 
-        let entryDetails;
+        let message;
-        if (entry.title && entry.link) {
+        if (this.state.template) {
-            entryDetails = `[${entry.title}](${entry.link})`;
+            message = this.templateFeedEntry(this.state.template, entry);
+        } else if (entry.title && entry.link) {
+            message = this.templateFeedEntry(DEFAULT_TEMPLATE_WITH_CONTENT, entry);
         } else {
-            entryDetails = entry.title || entry.link;
+            message = this.templateFeedEntry(DEFAULT_TEMPLATE, entry);
         }
 
-        let message = `New post in ${this.state.label || entry.feed.title || entry.feed.url}`;
+        // This might be massive and cause us to fail to send the message
-        if (entryDetails) {
+        // so confine to a maximum size.
-            message += `: ${entryDetails}`;
+        if (entry.summary?.length ?? 0 > MAX_SUMMARY_LENGTH) {
+            entry.summary = entry.summary?.substring(0, MAX_SUMMARY_LENGTH) + "…" ?? null;
         }
 
         await this.intent.sendEvent(this.roomId, {
@@ -155,10 +199,12 @@ export class FeedConnection extends BaseConnection implements IConnection {
             format: "org.matrix.custom.html",
             formatted_body: md.renderInline(message),
             body: message,
+            external_url: entry.link ?? undefined,
+            "uk.half-shot.matrix-hookshot.feeds.item": entry,
         });
     }
 
-    handleFeedSuccess() {
+    public handleFeedSuccess() {
         this.hasError = false;
         this.lastResults.unshift({
             ok: true,
@@ -189,6 +235,17 @@ export class FeedConnection extends BaseConnection implements IConnection {
         }
     }
 
+    public async provisionerUpdateConfig(userId: string, config: Record<string, unknown>) {
+        // Apply previous state to the current config, as provisioners might not return "unknown" keys.
+        config = { ...this.state, ...config };
+        const validatedConfig = FeedConnection.validateState(config);
+        if (validatedConfig.url !== this.feedUrl) {
+            throw new ApiError('Cannot alter url of existing feed. Please create a new one.', ErrCode.BadValue);
+        }
+        await this.intent.underlyingClient.sendStateEvent(this.roomId, FeedConnection.CanonicalEventType, this.stateKey, validatedConfig);
+        this.state = validatedConfig;
+    }
+    
     // needed to ensure that the connection is removable
     public async onRemove(): Promise<void> {
         log.info(`Removing connection ${this.connectionId}`);

src/feeds/FeedReader.ts

@@ -51,6 +51,9 @@ export interface FeedEntry {
     },
     title: string|null,
     link:  string|null,
+    pubdate: string|null,
+    summary: string|null,
+    author: string|null,
     /**
      * Unique key to identify the specific fetch across entries.
      */
@@ -334,6 +337,9 @@ export class FeedReader {
                         url: url,
                     },
                     title: item.title ? stripHtml(item.title) : null,
+                    pubdate: item.pubDate ?? null,
+                    summary: item.summary ?? null,
+                    author: item.creator ?? null,
                     link: FeedReader.parseLinkFromItem(item),
                     fetchKey
                 };

tests/connections/FeedTest.spec.ts

@@ -0,0 +1,70 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { AppserviceMock } from "../utils/AppserviceMock";
+import { FeedConnection, FeedConnectionState } from "../../src/Connections";
+import { FeedEntry } from "../../src/feeds/FeedReader";
+import { IntentMock } from "../utils/IntentMock";
+import { randomUUID } from "crypto";
+import { expect } from "chai";
+
+const ROOM_ID = "!foo:bar";
+const FEED_URL = "https://example.com/feed.xml";
+const FEED_ENTRY_DEFAULTS: FeedEntry = {
+    feed: {
+        title: "Test feed",
+        url: FEED_URL,
+    },
+    title: "Foo",
+    link: "foo/bar",
+    pubdate: "today!",
+    summary: "fibble fobble",
+    author: "Me!",
+    fetchKey: randomUUID(),
+}
+
+function createFeed(
+    state: FeedConnectionState = { url: FEED_URL  }
+): [FeedConnection, IntentMock] {
+    const as = AppserviceMock.create();
+    const intent = as.getIntentForUserId('@webhooks:example.test');
+    const connection =  new FeedConnection(ROOM_ID, "foobar", state, intent);
+    return [connection, intent];
+}
+describe("FeedConnection", () => {
+    it("will handle simple feed message", async () => {
+        const [connection, intent] = createFeed();
+        await connection.handleFeedEntry({
+            ...FEED_ENTRY_DEFAULTS,
+        });
+        const matrixEvt = intent.sentEvents[0];
+        expect(matrixEvt).to.not.be.undefined;
+        expect(matrixEvt.roomId).to.equal(ROOM_ID);
+        expect(matrixEvt.content.external_url).to.equal(FEED_ENTRY_DEFAULTS.link);
+        expect(matrixEvt.content.body).to.equal("New post in Test feed: [Foo](foo/bar)");
+    });
+    it("will handle simple feed message without a title and link ", async () => {
+        const [connection, intent] = createFeed();
+        await connection.handleFeedEntry({
+            ...FEED_ENTRY_DEFAULTS,
+            title: null,
+            link: null,
+        });
+        const matrixEvt =intent.sentEvents[0];
+        expect(matrixEvt).to.not.be.undefined;
+        expect(matrixEvt.roomId).to.equal(ROOM_ID);
+        expect(matrixEvt.content.external_url).to.be.undefined;
+        expect(matrixEvt.content.body).to.equal("New post in Test feed");
+    });
+    it("will handle simple feed message with all the template options possible ", async () => {
+        const [connection, intent] = createFeed({
+            url: FEED_URL,
+            template: `$FEEDNAME $FEEDURL $FEEDTITLE $TITLE $LINK $AUTHOR $DATE $SUMMARY`
+        });
+        await connection.handleFeedEntry({
+            ...FEED_ENTRY_DEFAULTS,
+        });
+        const matrixEvt =intent.sentEvents[0];
+        expect(matrixEvt).to.not.be.undefined;
+        expect(matrixEvt.roomId).to.equal(ROOM_ID);
+        expect(matrixEvt.content.body).to.equal("Test feed https://example.com/feed.xml Test feed Foo [Foo](foo/bar) Me! today! fibble fobble");
+    });
+})

web/components/roomConfig/FeedsConfig.tsx

@@ -8,6 +8,8 @@ import styles from "./FeedConnection.module.scss";
 
 import FeedsIcon from "../../icons/feeds.png";
 
+const DEFAULT_TEMPLATE = "New post in $FEEDNAME: $LINK"
+
 const FeedRecentResults: FunctionComponent<{item: FeedResponseItem}> = ({ item }) => {
     if (!item.secrets) {
         return null;
@@ -24,15 +26,17 @@ const FeedRecentResults: FunctionComponent<{item: FeedResponseItem}> = ({ item }
         </ul>
     </>;
 }
+const DOCUMENTATION_LINK = "https://matrix-org.github.io/matrix-hookshot/latest/setup/feeds.html#feed-templates";
 
-const ConnectionConfiguration: FunctionComponent<ConnectionConfigurationProps<ServiceConfig, FeedResponseItem, FeedConnectionState>> = ({existingConnection, onSave, onRemove}) => {
+const ConnectionConfiguration: FunctionComponent<ConnectionConfigurationProps<ServiceConfig, FeedResponseItem, FeedConnectionState>> = ({existingConnection, onSave, onRemove, isMigrationCandidate}) => {
     const urlRef = createRef<HTMLInputElement>();
     const labelRef = createRef<HTMLInputElement>();
-
+    const templateRef = createRef<HTMLInputElement>();
-    const canEdit = !existingConnection?.id || (existingConnection?.canEdit ?? false);
+    const canSave = !existingConnection?.id || (existingConnection?.canEdit ?? false);
+    const canEdit = canSave && !isMigrationCandidate;
     const handleSave = useCallback((evt: Event) => {
         evt.preventDefault();
-        if (!canEdit) {
+        if (!canSave) {
             return;
         }
         const url = urlRef?.current?.value || existingConnection?.config.url;
@@ -40,22 +44,27 @@ const ConnectionConfiguration: FunctionComponent<ConnectionConfigurationProps<Se
             onSave({
                 url,
                 label: labelRef?.current?.value || existingConnection?.config.label,
+                template: templateRef.current?.value || existingConnection?.config.template,
             });
         }
-    }, [canEdit, onSave, urlRef, labelRef, existingConnection]);
+    }, [canSave, onSave, urlRef, labelRef, templateRef, existingConnection]);
     
     return <form onSubmit={handleSave}>
         { existingConnection && <FeedRecentResults item={existingConnection} />}
 
-        <InputField visible={!existingConnection?.id} label="URL" noPadding={true}>
+        <InputField visible={true} label="URL" noPadding={true}>
-            <input ref={urlRef} disabled={!canEdit || (existingConnection && !existingConnection.id)} type="text" value={existingConnection?.config.url} />
+            <input ref={urlRef} disabled={!canEdit || !!existingConnection} type="text" value={existingConnection?.config.url} />
         </InputField>
-        <InputField visible={!existingConnection?.id} label="Label" noPadding={true}>
+        <InputField visible={true} label="Label" noPadding={true}>
-            <input ref={labelRef} disabled={!canEdit} type="text" value={existingConnection?.config.label} />
+            <input ref={labelRef} disabled={!canSave} type="text" value={existingConnection?.config.label} />
+        </InputField>
+        <InputField visible={true} label="Template" noPadding={true}>
+            <input ref={templateRef} disabled={!canSave} type="text" value={existingConnection?.config.template} placeholder={DEFAULT_TEMPLATE} />
+            <p> See the <a target="_blank" rel="noopener noreferrer" href={DOCUMENTATION_LINK}>documentation</a> for help writing templates. </p>
         </InputField>
 
         <ButtonSet>
-            { canEdit && <Button type="submit">{ existingConnection?.id ? "Save" : "Subscribe" }</Button>}
+            { canSave && <Button type="submit">{ existingConnection?.id ? "Save" : "Subscribe" }</Button>}
             { canEdit && existingConnection?.id && <Button intent="remove" onClick={onRemove}>Unsubscribe</Button>}
         </ButtonSet>
 
@@ -89,7 +98,7 @@ export const FeedsConfig: BridgeConfig = ({ api, roomId, showHeader }) => {
         });
     }, [api, roomId]);
 
-    const compareConnections = useCallback((goNebConnection, nativeConnection) => goNebConnection.config.url === nativeConnection.config.url, []);
+    const compareConnections = useCallback((goNebConnection: FeedResponseItem, nativeConnection: FeedResponseItem) => goNebConnection.config.url === nativeConnection.config.url, []);
 
     return <RoomConfig<ServiceConfig, FeedResponseItem, FeedConnectionState>
         headerImg={FeedsIcon}