Remove GoNEB migrator (#867)

* Remove GoNEB migrator.

* changelog

* Remove frontend pieces

* Remove unused statuscodes

* Linting

* Remove warning about ts-comment

changelog.d/867.removal

@@ -0,0 +1 @@
+The GoNEB migrator is being removed in this release. Users wishing to migrate from GoNEB deployments should use <=5.0.0 and then upgrade.

src/Connections/GitlabRepo.ts

@@ -1,5 +1,3 @@
-// We need to instantiate some functions which are not directly called, which confuses typescript.
-/* eslint-disable @typescript-eslint/ban-ts-comment */
 import { UserTokenStore } from "../UserTokenStore";
 import { Appservice, Intent, StateEvent } from "matrix-bot-sdk";
 import { BotCommands, botCommand, compileBotCommands } from "../BotCommands";
@@ -752,7 +750,7 @@ ${data.description}`;
         let relation;
         const discussionWithThread = result.discussions.find(discussionId => this.discussionThreads.has(discussionId));
         if (discussionWithThread) {
-            const threadEventId = await this.discussionThreads.get(discussionWithThread)!.catch(_ => { /* already logged */ });
+            const threadEventId = await this.discussionThreads.get(discussionWithThread)?.catch(() => { /* already logged */ });
             if (threadEventId) {
                 relation = {
                     "m.relates_to": {
@@ -948,7 +946,7 @@ ${data.description}`;
     private async persistDiscussionThreads(): Promise<void> {
         const serialized: SerializedGitlabDiscussionThreads = [];
         for (const [discussionId, eventIdPromise] of this.discussionThreads.entriesAscending()) {
-            const eventId = await eventIdPromise.catch(_ => { /* logged elsewhere */ });
+            const eventId = await eventIdPromise.catch(() => { /* logged elsewhere */ });
             if (eventId) {
                 serialized.push({ discussionId, eventId });
             }

src/Widgets/BridgeWidgetApi.ts

@@ -10,16 +10,12 @@ import { ConnectionManager } from "../ConnectionManager";
 import BotUsersManager, {BotUser} from "../Managers/BotUsersManager";
 import { assertUserPermissionsInRoom, GetConnectionsResponseItem } from "../provisioning/api";
 import { Appservice, PowerLevelsEvent } from "matrix-bot-sdk";
-import { GoNebMigrator } from "./GoNebMigrator";
-import { StatusCodes } from "http-status-codes";
 import { GithubInstance } from '../github/GithubInstance';
 import { AllowedTokenTypes, TokenType, UserTokenStore } from '../UserTokenStore';
 
 const log = new Logger("BridgeWidgetApi");
 
 export class BridgeWidgetApi extends ProvisioningApi {
-    private readonly goNebMigrator?: GoNebMigrator;
-
     constructor(
         private adminRooms: Map<string, AdminRoom>,
         private readonly config: BridgeConfig,
@@ -60,45 +56,6 @@ export class BridgeWidgetApi extends ProvisioningApi {
         this.addRoute('get', '/v1/service/:service/auth', wrapHandler(this.getAuth));
         this.addRoute('get', '/v1/service/:service/auth/:state', wrapHandler(this.getAuthPoll));
         this.addRoute('post', '/v1/service/:service/auth/logout', wrapHandler(this.postAuthLogout));
-        this.baseRoute.use((err: unknown, _req: Express.Request, _res: Express.Response, _next: NextFunction) => {
-            // Needed until https://github.com/matrix-org/matrix-appservice-bridge/pull/465 lands.
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            (this as any).onError(err, _req, _res, _next);
-        });
-
-        if (this.config.goNebMigrator) {
-            this.goNebMigrator = new GoNebMigrator(
-                this.config.goNebMigrator.apiUrl,
-                this.config.goNebMigrator.serviceIds,
-                this.config.goNebMigrator.goNebBotPrefix,
-            );
-        }
-
-        this.addRoute("get", "/v1/:roomId/goNebConnections", wrapHandler(this.getGoNebConnections));
-    }
-
-    private async getGoNebConnections(req: ProvisioningRequest, res: Response) {
-        if (!this.goNebMigrator) {
-            res.status(StatusCodes.NO_CONTENT).send();
-            return;
-        }
-
-        const roomId = req.params.roomId;
-
-        if (!req.userId) {
-            throw Error('Cannot get connections without a valid userId');
-        }
-
-        const botUser = await this.getBotUserInRoom(roomId);
-        await assertUserPermissionsInRoom(req.userId, roomId, "read", botUser.intent);
-
-        const userIds = this.goNebMigrator.getGoNebUsersFromRoomMembers(
-            await botUser.intent.underlyingClient.getJoinedRoomMembers(roomId)
-        );
-
-        const connections = await this.goNebMigrator.getConnectionsForRoom(roomId, new Set(userIds));
-
-        res.send(connections);
     }
 
     private async getBotUserInRoom(roomId: string, serviceType?: string): Promise<BotUser> {

src/Widgets/GoNebMigrator.ts

@@ -1,190 +0,0 @@
-import { Logger } from "matrix-appservice-bridge";
-import axios from "axios";
-
-import { FeedConnection, FeedConnectionState, GitHubRepoConnection, GitHubRepoConnectionState } from "../Connections";
-import { AllowedEventsNames as GitHubAllowedEventsNames } from "../Connections/GithubRepo";
-
-const log = new Logger("GoNebMigrator");
-
-interface MigratedGoNebConnection {
-    goNebId: string;
-}
-
-type MigratedFeed = FeedConnectionState & MigratedGoNebConnection;
-type MigratedGithub = GitHubRepoConnectionState & MigratedGoNebConnection;
-
-interface MigratedConnections {
-    [FeedConnection.ServiceCategory]: MigratedFeed[]|undefined,
-    [GitHubRepoConnection.ServiceCategory]: MigratedGithub[]|undefined;
-}
-
-interface GoNebFeedsConfig {
-    [url: string]: {
-        rooms: string[],
-    }
-}
-
-interface GoNebGithubRepos {
-    [githubPath: string]: {
-        Events: string[], // push, issues, pull_request, more?
-    }
-}
-
-interface GoNebService {
-    Type: string;
-    Config: any;
-}
-
-interface GoNebGithubWebhookService extends GoNebService {
-    Type: 'github-webhook';
-    Config: {
-        ClientUserID: string;
-        Rooms: {
-            [roomId: string]: { Repos: GoNebGithubRepos; }
-        }
-    };
-}
-
-export class GoNebMigrator {
-    private goNebBotPrefix: string;
-    constructor(
-        private apiUrl: string,
-        private serviceIds?: string[],
-        goNebBotPrefix?: string,
-    ) {
-        this.goNebBotPrefix = goNebBotPrefix ?? '@_neb_';
-    }
-
-    static convertFeeds(goNebFeeds: GoNebFeedsConfig): Map<string, FeedConnectionState[]> {
-        const feedsPerRoom = new Map<string, FeedConnectionState[]>();
-
-        for (const [url, config] of Object.entries(goNebFeeds)) {
-            for (const roomId of config.rooms) {
-                const existing = feedsPerRoom.get(roomId) ?? [];
-                existing.push({ url });
-                feedsPerRoom.set(roomId, existing);
-            }
-        }
-
-        return feedsPerRoom;
-    }
-
-    static convertGithub(roomRepos: GoNebGithubRepos): GitHubRepoConnectionState[] {
-        const eventMapping: { [goNebEvent: string]: GitHubAllowedEventsNames } = {
-            'pull_request': 'pull_request',
-            'issues': 'issue',
-            // 'push': ???
-        };
-        return Object.entries(roomRepos).map(([githubPath, { Events }]) => {
-            const [org, repo] = githubPath.split('/');
-            const enableHooks = Events.map(goNebEvent => eventMapping[goNebEvent]).filter(e => !!e);
-
-            return {
-                org,
-                repo,
-                enableHooks,
-            };
-        });
-    }
-
-    public async getConnectionsForRoom(roomId: string, userIds: Set<string>): Promise<MigratedConnections> {
-        const feeds: MigratedFeed[] = [];
-        const github: MigratedGithub[] = [];
-
-        const serviceIds = new Set([
-            ...(this.serviceIds ?? []),
-            ...['rssbot', 'github'].flatMap(type => Array.from(userIds).map(userId => `${type}/${strictEncodeURIComponent(userId)}/${strictEncodeURIComponent(roomId)}`)),
-        ]);
-
-        for (const id of serviceIds) {
-            const endpoint = this.apiUrl + (this.apiUrl.endsWith('/') ? '' : '/') + 'admin/getService';
-            let obj: GoNebService;
-            try {
-                const res = await axios.post(endpoint, { 'Id': id });
-                obj = res.data as GoNebService;
-            } catch (err: unknown) {
-                if (axios.isAxiosError(err)) {
-                    if (err.response?.status === 404) {
-                        continue;
-                    }
-                }
-
-                throw err;
-            }
-            switch (obj.Type) {
-                case 'rssbot': {
-                    const roomFeeds = GoNebMigrator.convertFeeds(obj.Config.feeds).get(roomId) ?? [];
-                    const migratedFeeds = roomFeeds.map(f => ({ ...f, goNebId: id }));
-                    feeds.push(...migratedFeeds);
-                    break;
-                }
-                case 'github-webhook': {
-                    const service = obj as GoNebGithubWebhookService;
-                    if (userIds.has(service.Config.ClientUserID)) {
-                        const roomRepos = service.Config.Rooms[roomId]?.Repos;
-                        if (roomRepos) {
-                            const githubConnections = GoNebMigrator.convertGithub(roomRepos);
-                            const migratedGithubs = githubConnections.map(f => ({ ...f, goNebId: id }));
-                            github.push(...migratedGithubs);
-                        }
-                    }
-                    break;
-                }
-                default: {
-                    log.warn(`Unrecognized go-neb service type (${obj.Type}), skipping`);
-                }
-            }
-        }
-
-        return {
-            feeds,
-            github,
-        };
-    }
-
-    public getGoNebUsersFromRoomMembers(members: string[]): string[] {
-        const goNebUsers = [];
-
-        for (const member of members) {
-            if (member.startsWith(this.goNebBotPrefix)) {
-                try {
-                    const mxid = this.getUserMxid(member);
-                    goNebUsers.push(mxid);
-                } catch (err: unknown) {
-                    log.error(`${member} looks like a go-neb mxid, but we failed to extract the owner mxid from it (${err})`);
-                }
-            }
-        }
-
-        return goNebUsers;
-    }
-
-    private getUserMxid(botMxid: string): string {
-        let userPart = botMxid.substring(this.goNebBotPrefix.length);
-        // strip the service type (before first '_') and server name (after ':')
-        try {
-            [, userPart] = userPart.match(/[^_]+_([^:]+):.*/)!;
-        } catch (err: unknown) {
-            throw new Error(`${botMxid} does not look like a Scalar-produced go-neb mxid`);
-        }
-
-        // decode according to https://spec.matrix.org/v1.2/appendices/#mapping-from-other-character-sets,
-        return userPart.replace(/=\w\w/g, (match) => {
-            // first the lowercased string...
-            const code = parseInt(match.substring(1), 16);
-            return String.fromCharCode(code);
-        }).replace(/_\w/g, (match) => {
-            // and then reapply the uppercase where applicable
-            return match.substring(1).toUpperCase();
-        });
-    }
-}
-
-// from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent#encoding_for_rfc3986
-function strictEncodeURIComponent(str: string) {
-  return encodeURIComponent(str)
-    .replace(
-      /[!'()*]/g,
-      (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`
-    );
-}

src/config/Config.ts

@@ -446,12 +446,6 @@ export interface BridgeConfigMetrics {
     port?: number;
 }
 
-export interface BridgeConfigGoNebMigrator {
-    apiUrl: string;
-    serviceIds?: string[];
-    goNebBotPrefix?: string;
-}
-
 export interface BridgeConfigSentry {
     dsn: string;
     environment?: string;
@@ -477,7 +471,6 @@ export interface BridgeConfigRoot {
     widgets?: BridgeWidgetConfigYAML;
     metrics?: BridgeConfigMetrics;
     listeners?: BridgeConfigListener[];
-    goNebMigrator?: BridgeConfigGoNebMigrator;
     sentry?: BridgeConfigSentry;
 }
 
@@ -531,9 +524,6 @@ export class BridgeConfig {
  'resources' may be any of ${ResourceTypeArray.join(', ')}`)
     public readonly listeners: BridgeConfigListener[];
 
-    @configKey("go-neb migrator configuration", true)
-    public readonly goNebMigrator?: BridgeConfigGoNebMigrator;
-
     @configKey("Configure Sentry error reporting", true)
     public readonly sentry?: BridgeConfigSentry;
 
@@ -603,7 +593,9 @@ export class BridgeConfig {
             this.queue.port = env?.CFG_QUEUE_POST ? parseInt(env?.CFG_QUEUE_POST, 10) : undefined;
         }
 
-        this.goNebMigrator = configData.goNebMigrator;
+        if ('goNebMigrator' in configData) {
+            log.warn(`The GoNEB migrator has been removed from this release. You should remove the 'goNebMigrator' from your config.`);
+        }
 
         // Listeners is a bit special
         this.listeners = configData.listeners || [];

web/BridgeAPI.ts

@@ -130,10 +130,6 @@ export class BridgeAPI {
         return this.request('GET', `/widgetapi/v1/${encodeURIComponent(roomId)}/connections`);
     }
 
-    async getGoNebConnectionsForRoom(roomId: string): Promise<any|undefined> {
-        return this.request('GET', `/widgetapi/v1/${encodeURIComponent(roomId)}/goNebConnections`);
-    }
-
     async getConnectionsForService<T extends GetConnectionsResponseItem >(roomId: string, service: string): Promise<GetConnectionsForServiceResponse<T>> {
         return this.request('GET', `/widgetapi/v1/${encodeURIComponent(roomId)}/connections/${encodeURIComponent(service)}`);
     }

web/components/roomConfig/FeedsConfig.tsx

@@ -1,5 +1,5 @@
 import { FunctionComponent, createRef } from "preact";
-import { useCallback, useEffect, useState } from "preact/hooks"
+import { useCallback, useState } from "preact/hooks"
 import { BridgeConfig } from "../../BridgeAPI";
 import { FeedConnectionState, FeedResponseItem } from "../../../src/Connections/FeedConnection";
 import { ConnectionConfigurationProps, IRoomConfigText, RoomConfig } from "./RoomConfig";
@@ -93,20 +93,6 @@ const roomConfigText: IRoomConfigText = {
 const RoomConfigListItemFunc = (c: FeedResponseItem) => c.config.label || c.config.url;
 
 export const FeedsConfig: BridgeConfig = ({ api, roomId, showHeader }) => {
-    const [ goNebConnections, setGoNebConnections ] = useState(undefined);
-
-    useEffect(() => {
-        api.getGoNebConnectionsForRoom(roomId).then((res: any) => {
-            if (!res) return;
-            setGoNebConnections(res.feeds.map((config: any) => ({
-                config,
-            })));
-        }).catch(ex => {
-            console.warn("Failed to fetch go neb connections", ex);
-        });
-    }, [api, roomId]);
-
-    const compareConnections = useCallback((goNebConnection: FeedResponseItem, nativeConnection: FeedResponseItem) => goNebConnection.config.url === nativeConnection.config.url, []);
 
     return <RoomConfig<ServiceConfig, FeedResponseItem, FeedConnectionState>
         headerImg={FeedsIcon}
@@ -118,7 +104,5 @@ export const FeedsConfig: BridgeConfig = ({ api, roomId, showHeader }) => {
         text={roomConfigText}
         listItemName={RoomConfigListItemFunc}
         connectionConfigComponent={ConnectionConfiguration}
-        migrationCandidates={goNebConnections}
-        migrationComparator={compareConnections}
     />;
 };

web/components/roomConfig/GithubRepoConfig.tsx

@@ -176,25 +176,6 @@ const roomConfigText: IRoomConfigText = {
 const RoomConfigListItemFunc = (c: GitHubRepoResponseItem) => getRepoFullName(c.config);
 
 export const GithubRepoConfig: BridgeConfig = ({ api, roomId, showHeader }) => {
-    const [ goNebConnections, setGoNebConnections ] = useState(undefined);
-
-    useEffect(() => {
-        api.getGoNebConnectionsForRoom(roomId).then((res: any) => {
-            if (!res) return;
-            setGoNebConnections(res.github.map((config: any) => ({
-                config,
-            })));
-        }).catch(ex => {
-            console.warn("Failed to fetch go neb connections", ex);
-        });
-    }, [api, roomId]);
-
-    const compareConnections = useCallback(
-        (goNebConnection, nativeConnection) => goNebConnection.config.org === nativeConnection.config.org
-                                            && goNebConnection.config.repo === nativeConnection.config.repo,
-        []
-    );
-
     return <RoomConfig<never, GitHubRepoResponseItem, GitHubRepoConnectionState>
         headerImg={GitHubIcon}
         darkHeaderImg={true}
@@ -207,7 +188,5 @@ export const GithubRepoConfig: BridgeConfig = ({ api, roomId, showHeader }) => {
         listItemName={RoomConfigListItemFunc}
         connectionEventType={EventType}
         connectionConfigComponent={ConnectionConfiguration}
-        migrationCandidates={goNebConnections}
-        migrationComparator={compareConnections}
     />;
 };

web/components/roomConfig/RoomConfig.tsx

@@ -9,6 +9,7 @@ import { LoadingSpinner } from '../elements/LoadingSpinner';
 import { ErrCode } from "../../../src/api";
 import { retry } from "../../../src/PromiseUtil";
 import { Alert } from "@vector-im/compound-web";
+
 export interface ConnectionConfigurationProps<SConfig, ConnectionType extends GetConnectionsResponseItem, ConnectionState extends IConnectionState> {
     serviceConfig: SConfig;
     loginLabel?: string;
@@ -41,8 +42,6 @@ interface IRoomConfigProps<SConfig, ConnectionType extends GetConnectionsRespons
     connectionEventType: string;
     listItemName: (c: ConnectionType) => string,
     connectionConfigComponent: FunctionComponent<ConnectionConfigurationProps<SConfig, ConnectionType, ConnectionState>>;
-    migrationCandidates?: ConnectionType[];
-    migrationComparator?: (migrated: ConnectionType, native: ConnectionType) => boolean;
 }
 
 const MAX_CONNECTION_FETCH_ATTEMPTS = 10;
@@ -59,8 +58,6 @@ export const RoomConfig = function<SConfig, ConnectionType extends GetConnection
         text,
         listItemName,
         connectionEventType,
-        migrationCandidates,
-        migrationComparator,
     } = props;
     const ConnectionConfigComponent = props.connectionConfigComponent;
     const [ error, setError ] = useState<null|{header?: string, message: string, isWarning?: boolean, forPrevious?: boolean}>(null);
@@ -97,28 +94,8 @@ export const RoomConfig = function<SConfig, ConnectionType extends GetConnection
         })
     }, [api, roomId, type, newConnectionKey]);
 
-    const [ toMigrate, setToMigrate ] = useState<ConnectionType[]>([]);
-
     const canSendMessages = connections?.every(c => c.canSendMessages) ?? true;
 
-    useEffect(() => {
-        // produce `toMigrate` composed of `migrationCandidates` with anything already in `connections` filtered out
-        // use `migrationComparator` to determine duplicates
-        if (!migrationCandidates) {
-            setToMigrate([]);
-            return;
-        }
-
-        if (!connections || !migrationComparator) {
-            setToMigrate(migrationCandidates);
-            return;
-        }
-
-        setToMigrate(
-            migrationCandidates.filter(cand => !connections.find(c => migrationComparator(cand, c)))
-        );
-    }, [ connections, migrationCandidates, migrationComparator ]);
-
     useEffect(() => {
         api.getServiceConfig<SConfig>(type)
             .then(setServiceConfig)
@@ -226,19 +203,6 @@ export const RoomConfig = function<SConfig, ConnectionType extends GetConnection
                 </ListItem>)
             }
         </section>}
-        { toMigrate.length > 0 && <section>
-            <h2> Migrate connections </h2>
-            { serviceConfig && toMigrate.map(c => <ListItem key={JSON.stringify(c)} text={listItemName(c)}>
-                  <ConnectionConfigComponent
-                      api={api}
-                      serviceConfig={serviceConfig}
-                      existingConnection={c}
-                      isUpdating={updatingConnection}
-                      isMigrationCandidate={true}
-                      onSave={handleSaveOnCreation}
-                  />
-            </ListItem>) }
-        </section>}
         </main>
     </Card>;
 };