Add a go-neb migrator (#647)

* Add a go-neb migrator, capable of migrating RSS/Atom feeds to hookshot (so far)

* Add forgotten file

* Fix useCallback() usage

* Gracefully handle go-neb not being configured

* Make Feed URLs not editable when they're being migrated over from go-neb

https://github.com/matrix-org/matrix-hookshot/pull/647#discussion_r1131615944

* Changelog

* Linting

* Add the ability to migrate Github repo connections from go-neb (#651)

* Add the ability to migrate Github repo connections from go-neb

* Gracefully handle the lack of go-neb migrator

---------

Co-authored-by: Tadeusz Sośnierz <tadeusz@sosnierz.com>

* Handle Scalar-style go-neb service IDs (#656)

* Add the ability to migrate Github repo connections from go-neb

* Handle Scalar-style go-neb service IDs

* Make service type in service ID match the Scalar-generated one

https://github.com/matrix-org/matrix-hookshot/pull/656#discussion_r1131647595

* Safeguard against undefined hardcoded service IDs

https://github.com/matrix-org/matrix-hookshot/pull/656#discussion_r1131646578

* Rename a variable

https://github.com/matrix-org/matrix-hookshot/pull/656#discussion_r1131710225

---------

Co-authored-by: Tadeusz Sośnierz <tadeusz@sosnierz.com>

* Linting

---------

Co-authored-by: Tadeusz Sośnierz <tadeusz@sosnierz.com>

changelog.d/647.feature

@@ -0,0 +1 @@
+Add support from migrating go-neb services to Hookshot

src/Config/Config.ts

@@ -431,6 +431,11 @@ export interface BridgeConfigMetrics {
     port?: number;
 }
 
+export interface BridgeConfigGoNebMigrator {
+    apiUrl: string;
+    serviceIds?: string[];
+}
+
 export interface BridgeConfigRoot {
     bot?: BridgeConfigBot;
     serviceBots?: BridgeConfigServiceBot[];
@@ -451,6 +456,7 @@ export interface BridgeConfigRoot {
     widgets?: BridgeWidgetConfigYAML;
     metrics?: BridgeConfigMetrics;
     listeners?: BridgeConfigListener[];
+    goNebMigrator?: BridgeConfigGoNebMigrator;
 }
 
 export class BridgeConfig {
@@ -503,6 +509,9 @@ export class BridgeConfig {
  'resources' may be any of ${ResourceTypeArray.join(', ')}`, true)
     public readonly listeners: BridgeConfigListener[];
 
+    @configKey("go-neb migrator configuration", true)
+    public readonly goNebMigrator?: BridgeConfigGoNebMigrator;
+
     @hideKey()
     private readonly bridgePermissions: BridgePermissions;
 
@@ -574,6 +583,8 @@ For more details, see https://github.com/matrix-org/matrix-hookshot/issues/594.
             this.queue.port = env.CFG_QUEUE_POST ? parseInt(env.CFG_QUEUE_POST, 10) : undefined;
         }
 
+        this.goNebMigrator = configData.goNebMigrator;
+
         // Listeners is a bit special
         this.listeners = configData.listeners || [];
 

src/Connections/GithubRepo.ts

@@ -94,7 +94,7 @@ export type GitHubRepoConnectionTarget = GitHubRepoConnectionOrgTarget|GitHubRep
 export type GitHubRepoResponseItem = GetConnectionsResponseItem<GitHubRepoConnectionState>;
 
 
-type AllowedEventsNames =
+export type AllowedEventsNames =
     "issue.changed" |
     "issue.created" |
     "issue.edited" |
@@ -119,7 +119,7 @@ type AllowedEventsNames =
     "workflow.run.action_required" |
     "workflow.run.stale";
 
-const AllowedEvents: AllowedEventsNames[] = [
+export const AllowedEvents: AllowedEventsNames[] = [
     "issue.changed" ,
     "issue.created" ,
     "issue.edited" ,

src/Widgets/BridgeWidgetApi.ts

@@ -10,6 +10,8 @@ import { ConnectionManager } from "../ConnectionManager";
 import BotUsersManager, {BotUser} from "../Managers/BotUsersManager";
 import { assertUserPermissionsInRoom, GetConnectionsResponseItem } from "../provisioning/api";
 import { Appservice, PowerLevelsEvent } from "matrix-bot-sdk";
+import { GoNebMigrator } from "./GoNebMigrator";
+import { StatusCodes } from "http-status-codes";
 import { GithubInstance } from '../Github/GithubInstance';
 import { AllowedTokenTypes, TokenType, UserTokenStore } from '../UserTokenStore';
 
@@ -17,6 +19,8 @@ const log = new Logger("BridgeWidgetApi");
 
 export class BridgeWidgetApi {
     private readonly api: ProvisioningApi;
+    private readonly goNebMigrator?: GoNebMigrator;
+
     constructor(
         private adminRooms: Map<string, AdminRoom>,
         private readonly config: BridgeConfig,
@@ -61,9 +65,37 @@ export class BridgeWidgetApi {
         this.api.addRoute('get', '/v1/service/:service/auth', wrapHandler(this.getAuth));
         this.api.addRoute('get', '/v1/service/:service/auth/:state', wrapHandler(this.getAuthPoll));
         this.api.addRoute('post', '/v1/service/:service/auth/logout', wrapHandler(this.postAuthLogout));
+
+        if (this.config.goNebMigrator) {
+            this.goNebMigrator = new GoNebMigrator(
+                this.config.goNebMigrator.apiUrl,
+                this.config.goNebMigrator.serviceIds,
+            );
+        }
+
+        this.api.addRoute("get", "/v1/:roomId/goNebConnections", wrapHandler(this.getGoNebConnections));
     }
 
-    private getBotUserInRoom(roomId: string, serviceType: string): BotUser {
+    private async getGoNebConnections(req: ProvisioningRequest, res: Response) {
+        if (!this.goNebMigrator) {
+            res.status(StatusCodes.NO_CONTENT).send();
+            return;
+        }
+
+        const roomId = req.params.roomId;
+
+        if (!req.userId) {
+            throw Error('Cannot get connections without a valid userId');
+        }
+
+        const botUser = this.getBotUserInRoom(roomId);
+        await assertUserPermissionsInRoom(req.userId, roomId, "read", botUser.intent);
+        const connections = await this.goNebMigrator.getConnectionsForRoom(roomId, req.userId);
+
+        res.send(connections);
+    }
+
+    private getBotUserInRoom(roomId: string, serviceType?: string): BotUser {
         const botUser = this.botUsersManager.getBotUserInRoom(roomId, serviceType);
         if (!botUser) {
             throw new ApiError("Bot is not joined to the room.", ErrCode.NotInRoom);

src/Widgets/GoNebMigrator.ts

@@ -0,0 +1,149 @@
+import { Logger } from "matrix-appservice-bridge";
+import axios from "axios";
+
+import { FeedConnection, FeedConnectionState, GitHubRepoConnection, GitHubRepoConnectionState } from "../Connections";
+import { AllowedEvents as GitHubAllowedEvents, AllowedEventsNames as GitHubAllowedEventsNames } from "../Connections/GithubRepo";
+
+const log = new Logger("GoNebMigrator");
+
+interface MigratedGoNebConnection {
+    goNebId: string;
+}
+
+type MigratedFeed = FeedConnectionState & MigratedGoNebConnection;
+type MigratedGithub = GitHubRepoConnectionState & MigratedGoNebConnection;
+
+interface MigratedConnections {
+    [FeedConnection.ServiceCategory]: MigratedFeed[]|undefined,
+    [GitHubRepoConnection.ServiceCategory]: MigratedGithub[]|undefined;
+}
+
+interface GoNebFeedsConfig {
+    [url: string]: {
+        rooms: string[],
+    }
+}
+
+interface GoNebGithubRepos {
+    [githubPath: string]: {
+        Events: string[], // push, issues, pull_request, more?
+    }
+}
+
+interface GoNebService {
+    Type: string;
+    Config: any;
+}
+
+interface GoNebGithubWebhookService extends GoNebService {
+    Type: 'github-webhook';
+    Config: {
+        ClientUserID: string;
+        Rooms: {
+            [roomId: string]: { Repos: GoNebGithubRepos; }
+        }
+    };
+}
+
+export class GoNebMigrator {
+    constructor(
+        private apiUrl: string,
+        private serviceIds?: string[],
+    ) {}
+
+    static convertFeeds(goNebFeeds: GoNebFeedsConfig): Map<string, FeedConnectionState[]> {
+        const feedsPerRoom = new Map<string, FeedConnectionState[]>();
+
+        for (const [url, config] of Object.entries(goNebFeeds)) {
+            for (const roomId of config.rooms) {
+                const existing = feedsPerRoom.get(roomId) ?? [];
+                existing.push({ url });
+                feedsPerRoom.set(roomId, existing);
+            }
+        }
+
+        return feedsPerRoom;
+    }
+
+    static convertGithub(roomRepos: GoNebGithubRepos): GitHubRepoConnectionState[] {
+        const eventMapping: { [goNebEvent: string]: GitHubAllowedEventsNames } = {
+            'pull_request': 'pull_request',
+            'issues': 'issue',
+            // 'push': ???
+        };
+        return Object.entries(roomRepos).map(([githubPath, { Events }]) => {
+            const [org, repo] = githubPath.split('/');
+            const enableHooks = Events.map(goNebEvent => eventMapping[goNebEvent]).filter(e => !!e);
+
+            return {
+                org,
+                repo,
+                enableHooks,
+            };
+        });
+    }
+
+    public async getConnectionsForRoom(roomId: string, userId: string): Promise<MigratedConnections> {
+        const feeds: MigratedFeed[] = [];
+        const github: MigratedGithub[] = [];
+
+        const serviceIds = [
+            ...(this.serviceIds ?? []),
+            ...['rssbot', 'github'].map(type => `${type}/${strictEncodeURIComponent(userId)}/${strictEncodeURIComponent(roomId)}`),
+        ];
+
+        for (const id of serviceIds) {
+            const endpoint = this.apiUrl + (this.apiUrl.endsWith('/') ? '' : '/') + 'admin/getService';
+            let obj: GoNebService;
+            try {
+                const res = await axios.post(endpoint, { 'Id': id });
+                obj = res.data as GoNebService;
+            } catch (err: unknown) {
+                if (axios.isAxiosError(err)) {
+                    if (err.response?.status === 404) {
+                        continue;
+                    }
+                }
+
+                throw err;
+            }
+            switch (obj.Type) {
+                case 'rssbot': {
+                    const roomFeeds = GoNebMigrator.convertFeeds(obj.Config.feeds).get(roomId) ?? [];
+                    const migratedFeeds = roomFeeds.map(f => ({ ...f, goNebId: id }));
+                    feeds.push(...migratedFeeds);
+                    break;
+                }
+                case 'github-webhook': {
+                    const service = obj as GoNebGithubWebhookService;
+                    if (service.Config.ClientUserID === userId) {
+                        const roomRepos = service.Config.Rooms[roomId]?.Repos;
+                        if (roomRepos) {
+                            const githubConnections = GoNebMigrator.convertGithub(roomRepos);
+                            const migratedGithubs = githubConnections.map(f => ({ ...f, goNebId: id }));
+                            github.push(...migratedGithubs);
+                        }
+                    }
+                    break;
+                }
+                default: {
+                    log.warn(`Unrecognized go-neb service type (${obj.Type}), skipping`);
+                }
+            }
+        }
+
+        return {
+            feeds,
+            github,
+        };
+    }
+}
+
+// from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent#encoding_for_rfc3986
+function strictEncodeURIComponent(str: string) {
+  return encodeURIComponent(str)
+    .replace(
+      /[!'()*]/g,
+      (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`
+    );
+}

web/BridgeAPI.ts

@@ -118,6 +118,10 @@ export class BridgeAPI {
         return this.request('GET', `/widgetapi/v1/${encodeURIComponent(roomId)}/connections`);
     }
 
+    async getGoNebConnectionsForRoom(roomId: string): Promise<any|undefined> {
+        return this.request('GET', `/widgetapi/v1/${encodeURIComponent(roomId)}/goNebConnections`);
+    }
+
     async getConnectionsForService<T extends GetConnectionsResponseItem >(roomId: string, service: string): Promise<GetConnectionsForServiceResponse<T>> {
         return this.request('GET', `/widgetapi/v1/${encodeURIComponent(roomId)}/connections/${encodeURIComponent(service)}`);
     }

web/components/roomConfig/FeedsConfig.tsx

@@ -1,5 +1,5 @@
 import { FunctionComponent, createRef } from "preact";
-import { useCallback } from "preact/hooks"
+import { useCallback, useEffect, useState } from "preact/hooks"
 import { BridgeConfig } from "../../BridgeAPI";
 import { FeedConnectionState, FeedResponseItem } from "../../../src/Connections/FeedConnection";
 import { ConnectionConfigurationProps, IRoomConfigText, RoomConfig } from "./RoomConfig";
@@ -29,7 +29,7 @@ const ConnectionConfiguration: FunctionComponent<ConnectionConfigurationProps<Se
     const urlRef = createRef<HTMLInputElement>();
     const labelRef = createRef<HTMLInputElement>();
 
-    const canEdit = !existingConnection || (existingConnection?.canEdit ?? false);
+    const canEdit = !existingConnection?.id || (existingConnection?.canEdit ?? false);
     const handleSave = useCallback((evt: Event) => {
         evt.preventDefault();
         if (!canEdit) {
@@ -47,16 +47,16 @@ const ConnectionConfiguration: FunctionComponent<ConnectionConfigurationProps<Se
     return <form onSubmit={handleSave}>
         { existingConnection && <FeedRecentResults item={existingConnection} />}
 
-        <InputField visible={!existingConnection} label="URL" noPadding={true}>
-            <input ref={urlRef} disabled={!canEdit} type="text" value={existingConnection?.config.url} />
+        <InputField visible={!existingConnection?.id} label="URL" noPadding={true}>
+            <input ref={urlRef} disabled={!canEdit || (existingConnection && !existingConnection.id)} type="text" value={existingConnection?.config.url} />
         </InputField>
-        <InputField visible={!existingConnection} label="Label" noPadding={true}>
+        <InputField visible={!existingConnection?.id} label="Label" noPadding={true}>
             <input ref={labelRef} disabled={!canEdit} type="text" value={existingConnection?.config.label} />
         </InputField>
 
         <ButtonSet>
-            { canEdit && <Button type="submit">{ existingConnection ? "Save" : "Subscribe" }</Button>}
-            { canEdit && existingConnection && <Button intent="remove" onClick={onRemove}>Unsubscribe</Button>}
+            { canEdit && <Button type="submit">{ existingConnection?.id ? "Save" : "Subscribe" }</Button>}
+            { canEdit && existingConnection?.id && <Button intent="remove" onClick={onRemove}>Unsubscribe</Button>}
         </ButtonSet>
 
     </form>;
@@ -76,6 +76,21 @@ const roomConfigText: IRoomConfigText = {
 const RoomConfigListItemFunc = (c: FeedResponseItem) => c.config.label || c.config.url;
 
 export const FeedsConfig: BridgeConfig = ({ api, roomId, showHeader }) => {
+    const [ goNebConnections, setGoNebConnections ] = useState(undefined);
+
+    useEffect(() => {
+        api.getGoNebConnectionsForRoom(roomId).then((res: any) => {
+            if (!res) return;
+            setGoNebConnections(res.feeds.map((config: any) => ({
+                config,
+            })));
+        }).catch(ex => {
+            console.warn("Failed to fetch go neb connections", ex);
+        });
+    }, [api, roomId]);
+
+    const compareConnections = useCallback((goNebConnection, nativeConnection) => goNebConnection.config.url === nativeConnection.config.url, []);
+
     return <RoomConfig<ServiceConfig, FeedResponseItem, FeedConnectionState>
         headerImg={FeedsIcon}
         showHeader={showHeader}
@@ -86,5 +101,7 @@ export const FeedsConfig: BridgeConfig = ({ api, roomId, showHeader }) => {
         text={roomConfigText}
         listItemName={RoomConfigListItemFunc}
         connectionConfigComponent={ConnectionConfiguration}
+        migrationCandidates={goNebConnections}
+        migrationComparator={compareConnections}
     />;
 };

web/components/roomConfig/GithubRepoConfig.tsx

@@ -59,7 +59,7 @@ const ConnectionConfiguration: FunctionComponent<ConnectionConfigurationProps<ne
 
     const [connectionState, setConnectionState] = useState<GitHubRepoConnectionState|null>(null);
 
-    const canEdit = !existingConnection || (existingConnection?.canEdit ?? false);
+    const canEdit = !existingConnection?.id || (existingConnection?.canEdit ?? false);
     const commandPrefixRef = createRef<HTMLInputElement>();
     const handleSave = useCallback((evt: Event) => {
         evt.preventDefault();
@@ -152,8 +152,8 @@ const ConnectionConfiguration: FunctionComponent<ConnectionConfigurationProps<ne
             </ul>
         </InputField>
         <ButtonSet>
-            { canEdit && authedResponse?.authenticated && <Button type="submit" disabled={!existingConnection && !connectionState}>{ existingConnection ? "Save" : "Add repository" }</Button>}
-            { canEdit && existingConnection && <Button intent="remove" onClick={onRemove}>Remove repository</Button>}
+            { canEdit && authedResponse?.authenticated && <Button type="submit" disabled={!existingConnection && !connectionState}>{ existingConnection?.id ? "Save" : "Add repository" }</Button>}
+            { canEdit && existingConnection?.id && <Button intent="remove" onClick={onRemove}>Remove repository</Button>}
         </ButtonSet>
     </form>;
 };
@@ -169,6 +169,25 @@ const roomConfigText: IRoomConfigText = {
 const RoomConfigListItemFunc = (c: GitHubRepoResponseItem) => getRepoFullName(c.config);
 
 export const GithubRepoConfig: BridgeConfig = ({ api, roomId, showHeader }) => {
+    const [ goNebConnections, setGoNebConnections ] = useState(undefined);
+
+    useEffect(() => {
+        api.getGoNebConnectionsForRoom(roomId).then((res: any) => {
+            if (!res) return;
+            setGoNebConnections(res.github.map((config: any) => ({
+                config,
+            })));
+        }).catch(ex => {
+            console.warn("Failed to fetch go neb connections", ex);
+        });
+    }, [api, roomId]);
+
+    const compareConnections = useCallback(
+        (goNebConnection, nativeConnection) => goNebConnection.config.org === nativeConnection.config.org
+                                            && goNebConnection.config.repo === nativeConnection.config.repo,
+        []
+    );
+
     return <RoomConfig<never, GitHubRepoResponseItem, GitHubRepoConnectionState>
         headerImg={GitHubIcon}
         showHeader={showHeader}
@@ -180,5 +199,7 @@ export const GithubRepoConfig: BridgeConfig = ({ api, roomId, showHeader }) => {
         listItemName={RoomConfigListItemFunc}
         connectionEventType={EventType}
         connectionConfigComponent={ConnectionConfiguration}
+        migrationCandidates={goNebConnections}
+        migrationComparator={compareConnections}
     />;
 };

web/components/roomConfig/RoomConfig.tsx

@@ -37,6 +37,8 @@ interface IRoomConfigProps<SConfig, ConnectionType extends GetConnectionsRespons
     connectionEventType: string;
     listItemName: (c: ConnectionType) => string,
     connectionConfigComponent: FunctionComponent<ConnectionConfigurationProps<SConfig, ConnectionType, ConnectionState>>;
+    migrationCandidates?: ConnectionType[];
+    migrationComparator?: (migrated: ConnectionType, native: ConnectionType) => boolean;
 }
 export const RoomConfig = function<SConfig, ConnectionType extends GetConnectionsResponseItem, ConnectionState extends IConnectionState>(props: IRoomConfigProps<SConfig, ConnectionType, ConnectionState>) {
     const {
@@ -48,7 +50,9 @@ export const RoomConfig = function<SConfig, ConnectionType extends GetConnection
         showHeader,
         text,
         listItemName,
-        connectionEventType
+        connectionEventType,
+        migrationCandidates,
+        migrationComparator,
     } = props;
     const ConnectionConfigComponent = props.connectionConfigComponent;
     const [ error, setError ] = useState<null|{header?: string, message: string, isWarning?: boolean, forPrevious?: boolean}>(null);
@@ -76,6 +80,26 @@ export const RoomConfig = function<SConfig, ConnectionType extends GetConnection
         });
     }, [api, roomId, type, newConnectionKey]);
 
+    const [ toMigrate, setToMigrate ] = useState<ConnectionType[]>([]);
+
+    useEffect(() => {
+        // produce `toMigrate` composed of `migrationCandidates` with anything already in `connections` filtered out
+        // use `migrationComparator` to determine duplicates
+        if (!migrationCandidates) {
+            setToMigrate([]);
+            return;
+        }
+
+        if (!connections || !migrationComparator) {
+            setToMigrate(migrationCandidates);
+            return;
+        }
+
+        setToMigrate(
+            migrationCandidates.filter(cand => !connections.find(c => migrationComparator(cand, c)))
+        );
+    }, [ connections, migrationCandidates, migrationComparator ]);
+
     useEffect(() => {
         api.getServiceConfig<SConfig>(type)
             .then(setServiceConfig)
@@ -170,8 +194,19 @@ export const RoomConfig = function<SConfig, ConnectionType extends GetConnection
                         }}
                     />
                 </ListItem>)
-                }
-            </section>}
+            }
+        </section>}
+        { toMigrate.length > 0 && <section>
+            <h2> Migrate connections </h2>
+            { serviceConfig && toMigrate.map(c => <ListItem key={JSON.stringify(c)} text={listItemName(c)}>
+                  <ConnectionConfigComponent
+                      api={api}
+                      serviceConfig={serviceConfig}
+                      existingConnection={c}
+                      onSave={handleSaveOnCreation}
+                  />
+            </ListItem>) }
+        </section>}
         </main>
     </Card>;
 };