nodejs/hookshot
nodejs/hookshot
1
0
Fork 0
mirror of https://github.com/matrix-org/matrix-hookshot.git synced 2025-03-10 21:19:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'hs/gitlab'

Browse Source
This commit is contained in:
Will Hunt 2020-12-12 12:36:08 +00:00
commit c5f40cc591
41 changed files with 2756 additions and 1429 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
.eslintrc.js Normal file
View File

@ -0,0 +1,21 @@
// eslint-disable-next-line no-undef
module.exports = {
root: true,
parser: '@typescript-eslint/parser',
plugins: [
'@typescript-eslint',
'mocha'
],
extends: [
'eslint:recommended',
'plugin:@typescript-eslint/recommended',
],
rules: {
"@typescript-eslint/explicit-module-boundary-types": "off",
"camelcase": ["error", { "properties": "never", "ignoreDestructuring": true }],
},
"env": {
"node": true,
"es6": true,
},
};

60
package.json
View File

@ -9,46 +9,50 @@
"private": false,
"scripts": {
"build": "tsc --project tsconfig.json",
"prepare": "yarn build",
"start": "node lib/App/BridgeApp.js",
"start:app": "node lib/App/BridgeApp.js",
"start:webhooks": "node lib/App/GithubWebhookApp.js",
"start:matrixsender": "node lib/App/MatrixSenderApp.js",
"test": "mocha -r ts-node/register tests/**/*.ts",
"lint": "tslint -p tsconfig.json"
"test": "mocha -r ts-node/register tests/*.ts",
"lint": "eslint -c .eslintrc.js src/**/*.ts"
},
"dependencies": {
"@octokit/auth-app": "^2.4.2",
"@octokit/auth-token": "^2.4.0",
"@octokit/rest": "^16.43.1",
"argv-split": "^2.0.1",
"axios": "^0.19.2",
"@octokit/auth-app": "^2.10.2",
"@octokit/auth-token": "^2.4.3",
"@octokit/rest": "^18.0.9",
"axios": "^0.21.0",
"express": "^4.17.1",
"ioredis": "^4.14.0",
"markdown-it": "^9.0.1",
"matrix-bot-sdk": "^0.5.4",
"ioredis": "^4.19.2",
"markdown-it": "^12.0.2",
"matrix-bot-sdk": "^0.5.8",
"micromatch": "^4.0.2",
"mime": "^2.4.4",
"mocha": "^6.2.0",
"mime": "^2.4.6",
"mocha": "^8.2.1",
"node-emoji": "^1.10.0",
"reflect-metadata": "^0.1.13",
"uuid": "^3.3.2",
"winston": "^3.2.1",
"yaml": "^1.6.0"
"string-argv": "v0.3.1",
"uuid": "^8.3.1",
"winston": "^3.3.3",
"yaml": "^1.10.0"
},
"devDependencies": {
"@types/chai": "^4.1.7",
"@types/express": "^4.17.0",
"@types/ioredis": "^4.0.13",
"@types/markdown-it": "^0.0.8",
"@types/micromatch": "^3.1.0",
"@types/mime": "^2.0.1",
"@types/mocha": "^5.2.7",
"@types/node": "^12.6.9",
"@types/chai": "^4.2.14",
"@types/express": "^4.17.9",
"@types/ioredis": "^4.17.8",
"@types/markdown-it": "^10.0.3",
"@types/micromatch": "^4.0.1",
"@types/mime": "^2.0.3",
"@types/mocha": "^8.0.4",
"@types/node": "^12",
"@types/node-emoji": "^1.8.1",
"@types/uuid": "^3.4.5",
"@types/yaml": "^1.0.2",
"@types/uuid": "^8.3.0",
"@typescript-eslint/eslint-plugin": "^4.8.1",
"@typescript-eslint/parser": "^4.8.1",
"chai": "^4.2.0",
"ts-node": "^8.3.0",
"tslint": "^5.18.0",
"typescript": "^3.9.7"
"eslint": "^7.14.0",
"eslint-plugin-mocha": "^8.0.0",
"ts-node": "^9.0.0",
"typescript": "^4.1.2"
}
}

307
src/AdminRoom.ts
View File

@ -1,9 +1,8 @@
/* eslint-disable @typescript-eslint/ban-ts-comment */
import { Intent } from "matrix-bot-sdk";
import { Octokit } from "@octokit/rest";
import { createTokenAuth } from "@octokit/auth-token";
import { UserTokenStore } from "./UserTokenStore";
import { BridgeConfig } from "./Config";
import uuid from "uuid/v4";
import {v4 as uuid} from "uuid";
import qs from "querystring";
import { EventEmitter } from "events";
import LogWrapper from "./LogWrapper";
@ -12,22 +11,39 @@ import markdown from "markdown-it";
import { FormatUtil } from "./FormatUtil";
import { botCommand, compileBotCommands, handleCommand, BotCommands } from "./BotCommands";
import { GitLabClient } from "./Gitlab/Client";
import { GetUserResponse } from "./Gitlab/Types";
import { GithubInstance } from "./Github/GithubInstance";
import { MatrixMessageContent } from "./MatrixEvent";
import { ProjectsListForUserResponseData, ProjectsListForRepoResponseData } from "@octokit/types";
const md = new markdown();
const log = new LogWrapper('AdminRoom');
export const BRIDGE_ROOM_TYPE = "uk.half-shot.matrix-github.room";
export const BRIDGE_NOTIF_TYPE = "uk.half-shot.matrix-github.notif_state";
export const BRIDGE_GITLAB_NOTIF_TYPE = "uk.half-shot.matrix-github.gitlab.notif_state";
export interface AdminAccountData {
// eslint-disable-next-line camelcase
admin_user: string;
notifications?: {
enabled: boolean;
participating?: boolean;
github?: {
notifications?: {
enabled: boolean;
participating?: boolean;
};
};
gitlab?: {
[instanceUrl: string]: {
notifications: {
enabled: boolean;
}
}
}
}
export class AdminRoom extends EventEmitter {
static helpMessage: any;
public static helpMessage: MatrixMessageContent;
static botCommands: BotCommands;
private pendingOAuthState: string|null = null;
@ -48,19 +64,40 @@ export class AdminRoom extends EventEmitter {
return this.pendingOAuthState;
}
public get notificationsEnabled() {
return !!this.data.notifications?.enabled;
public notificationsEnabled(type: "github"|"gitlab", instanceName?: string) {
if (type === "github") {
return this.data.github?.notifications?.enabled;
}
return (type === "gitlab" &&
!!instanceName &&
this.data.gitlab &&
this.data.gitlab[instanceName].notifications.enabled
);
}
public get notificationsParticipating() {
return !!this.data.notifications?.participating;
public notificationsParticipating(type: string) {
if (type !== "github") {
return false;
}
return this.data.github?.notifications?.participating || false;
}
public clearOauthState() {
this.pendingOAuthState = null;
}
public async getNotifSince() {
public async getNotifSince(type: "github"|"gitlab", instanceName?: string) {
if (type === "gitlab") {
try {
const { since } = await this.botIntent.underlyingClient.getRoomAccountData(
`${BRIDGE_GITLAB_NOTIF_TYPE}:${instanceName}`, this.roomId
);
return since;
} catch {
// TODO: We should look at this error.
return 0;
}
}
try {
const { since } = await this.botIntent.underlyingClient.getRoomAccountData(BRIDGE_NOTIF_TYPE, this.roomId);
return since;
@ -70,7 +107,14 @@ export class AdminRoom extends EventEmitter {
}
}
public async setNotifSince(since: number) {
public async setNotifSince(type: "github"|"gitlab", since: number, instanceName?: string) {
if (type === "gitlab") {
return this.botIntent.underlyingClient.setRoomAccountData(
`${BRIDGE_GITLAB_NOTIF_TYPE}:${instanceName}`,
this.roomId, {
since,
});
}
return this.botIntent.underlyingClient.setRoomAccountData(BRIDGE_NOTIF_TYPE, this.roomId, {
since,
});
@ -82,19 +126,18 @@ export class AdminRoom extends EventEmitter {
@botCommand("help", "This help text")
public async helpCommand() {
return this.botIntent.underlyingClient.sendMessage(this.roomId, AdminRoom.helpMessage);
return this.botIntent.sendEvent(this.roomId, AdminRoom.helpMessage);
}
@botCommand("setpersonaltoken", "Set your personal access token for GitHub", ['accessToken'])
@botCommand("github setpersonaltoken", "Set your personal access token for GitHub", ['accessToken'])
// @ts-ignore - property is used
private async setGHPersonalAccessToken(accessToken: string) {
if (!this.config.github) {
return this.sendNotice("The bridge is not configured with GitHub support");
}
let me;
try {
const octokit = new Octokit({
authStrategy: createTokenAuth,
auth: accessToken,
userAgent: "matrix-github v0.0.1",
});
const octokit = GithubInstance.createUserOctokit(accessToken);
me = await octokit.users.getAuthenticated();
} catch (ex) {
await this.sendNotice("Could not authenticate with GitHub. Is your token correct?");
@ -104,25 +147,12 @@ export class AdminRoom extends EventEmitter {
await this.tokenStore.storeUserToken("github", this.userId, accessToken);
}
@botCommand("gitlab personaltoken", "Set your personal access token for GitLab", ['instanceUrl', 'accessToken'])
// @ts-ignore - property is used
private async setGitLabPersonalAccessToken(instanceUrl: string, accessToken: string) {
let me: GetUserResponse;
try {
const client = new GitLabClient(instanceUrl, accessToken);
me = await client.user();
} catch (ex) {
log.error("Gitlab auth error:", ex);
await this.sendNotice("Could not authenticate with GitLab. Is your token correct?");
return;
}
await this.sendNotice(`Connected as ${me.username}. Token stored`);
await this.tokenStore.storeUserToken("gitlab", this.userId, accessToken, instanceUrl);
}
@botCommand("hastoken", "Check if you have a token stored for GitHub")
@botCommand("github hastoken", "Check if you have a token stored for GitHub")
// @ts-ignore - property is used
private async hasPersonalToken() {
if (!this.config.github) {
return this.sendNotice("The bridge is not configured with GitHub support");
}
const result = await this.tokenStore.getUserToken("github", this.userId);
if (result === null) {
await this.sendNotice("You do not currently have a token stored");
@ -131,9 +161,12 @@ export class AdminRoom extends EventEmitter {
await this.sendNotice("A token is stored for your GitHub account.");
}
@botCommand("startoauth", "Start the OAuth process with GitHub")
@botCommand("github startoauth", "Start the OAuth process with GitHub")
// @ts-ignore - property is used
private async beginOAuth() {
if (!this.config.github) {
return this.sendNotice("The bridge is not configured with GitHub support");
}
// If this is already set, calling this command will invalidate the previous session.
this.pendingOAuthState = uuid();
const q = qs.stringify({
@ -145,41 +178,49 @@ export class AdminRoom extends EventEmitter {
await this.sendNotice(`You should follow ${url} to link your account to the bridge`);
}
@botCommand("notifications toggle", "Toggle enabling/disabling GitHub notifications in this room")
@botCommand("github notifications toggle", "Toggle enabling/disabling GitHub notifications in this room")
// @ts-ignore - property is used
private async setNotificationsStateToggle() {
const data: AdminAccountData = await this.botIntent.underlyingClient.getRoomAccountData(
BRIDGE_ROOM_TYPE, this.roomId,
);
const oldState = data.notifications || {
enabled: false,
participating: true,
};
data.notifications = { enabled: !oldState?.enabled, participating: oldState?.participating };
await this.botIntent.underlyingClient.setRoomAccountData(BRIDGE_ROOM_TYPE, this.roomId, data);
this.emit("settings.changed", this, data);
await this.sendNotice(`${data.notifications.enabled ? "En" : "Dis"}abled GitHub notifcations`);
private async setGitHubNotificationsStateToggle() {
const data = await this.saveAccountData((data) => {
return {
...data,
github: {
notifications: {
enabled: !(data.github?.notifications?.enabled ?? false),
participating: data.github?.notifications?.participating,
},
},
};
});
await this.sendNotice(`${data.github?.notifications?.enabled ? "En" : "Dis"}abled GitHub notifcations`);
}
@botCommand("notifications filter participating", "Toggle enabling/disabling GitHub notifications in this room")
@botCommand("github notifications filter participating", "Toggle enabling/disabling GitHub notifications in this room")
// @ts-ignore - property is used
private async setNotificationsStateParticipating() {
const data: AdminAccountData = await this.botIntent.underlyingClient.getRoomAccountData(
BRIDGE_ROOM_TYPE, this.roomId,
);
const oldState = data.notifications || {
enabled: false,
participating: true,
};
data.notifications = { enabled: oldState?.enabled, participating: !oldState?.participating };
await this.botIntent.underlyingClient.setRoomAccountData(BRIDGE_ROOM_TYPE, this.roomId, data);
this.emit("settings.changed", this, data);
await this.sendNotice(`${data.notifications.participating ? "En" : "Dis"}abled filtering for participating notifications`);
private async setGitHubNotificationsStateParticipating() {
const data = await this.saveAccountData((data) => {
if (!data.github?.notifications?.enabled) {
throw Error('Notifications are not enabled')
}
return {
...data,
github: {
notifications: {
participating: !(data.github?.notifications?.participating ?? false),
enabled: true,
},
},
};
});
await this.sendNotice(`${data.github?.notifications?.enabled ? "" : "Not"} filtering for events you are participating in`);
}
@botCommand("project list-for-user", "List GitHub projects for a user", [], ['user', 'repo'])
@botCommand("github project list-for-user", "List GitHub projects for a user", [], ['user', 'repo'])
// @ts-ignore - property is used
private async listProjects(username?: string, repo?: string) {
if (!this.config.github) {
return this.sendNotice("The bridge is not configured with GitHub support");
}
const octokit = await this.tokenStore.getOctokitForUser(this.userId);
if (!octokit) {
return this.sendNotice("You can not list projects without an account.");
@ -190,7 +231,7 @@ export class AdminRoom extends EventEmitter {
username = me.data.name;
}
let res: Octokit.ProjectsListForUserResponse|Octokit.ProjectsListForRepoResponse;
let res: ProjectsListForUserResponseData|ProjectsListForRepoResponseData;
try {
if (repo) {
res = (await octokit.projects.listForRepo({
@ -206,7 +247,7 @@ export class AdminRoom extends EventEmitter {
return this.sendNotice(`Failed to fetch projects due to an error. See logs for details`);
}
const content = `Projects for ${username}:\n` + res.map(r => ` - ${FormatUtil.projectListing(r)}\n`).join("\n");
const content = `Projects for ${username}:\n` + res.map(r => ` - ${FormatUtil.projectListing([r])}\n`).join("\n");
return this.botIntent.sendEvent(this.roomId,{
msgtype: "m.notice",
body: content,
@ -215,15 +256,18 @@ export class AdminRoom extends EventEmitter {
});
}
@botCommand("project list-for-org", "List GitHub projects for an org", ['org'], ['repo'])
@botCommand("github project list-for-org", "List GitHub projects for an org", ['org'], ['repo'])
// @ts-ignore - property is used
private async listProjects(org: string, repo?: string) {
if (!this.config.github) {
return this.sendNotice("The bridge is not configured with GitHub support");
}
const octokit = await this.tokenStore.getOctokitForUser(this.userId);
if (!octokit) {
return this.sendNotice("You can not list projects without an account.");
}
let res: Octokit.ProjectsListForUserResponse|Octokit.ProjectsListForRepoResponse;
let res: ProjectsListForUserResponseData|ProjectsListForRepoResponseData;
try {
if (repo) {
res = (await octokit.projects.listForRepo({
@ -239,7 +283,7 @@ export class AdminRoom extends EventEmitter {
return this.sendNotice(`Failed to fetch projects due to an error. See logs for details`);
}
const content = `Projects for ${org}:\n` + res.map(r => ` - ${FormatUtil.projectListing(r)}\n`).join("\n");
const content = `Projects for ${org}:\n` + res.map(r => ` - ${FormatUtil.projectListing([r])}\n`).join("\n");
return this.botIntent.sendEvent(this.roomId,{
msgtype: "m.notice",
body: content,
@ -248,9 +292,12 @@ export class AdminRoom extends EventEmitter {
});
}
@botCommand("project open", "Open a GitHub project as a room", [], ['projectId'])
@botCommand("github project open", "Open a GitHub project as a room", ['projectId'])
// @ts-ignore - property is used
private async openProject(projectId: string) {
if (!this.config.github) {
return this.sendNotice("The bridge is not configured with GitHub support");
}
const octokit = await this.tokenStore.getOctokitForUser(this.userId);
if (!octokit) {
return this.sendNotice("You can not list projects without an account.");
@ -267,7 +314,120 @@ export class AdminRoom extends EventEmitter {
}
}
public async handleCommand(event_id: string, command: string) {
/* GitLab commands */
@botCommand("gitlab open issue", "Open or join a issue room for GitLab", ['url'])
// @ts-ignore - property is used
private async gitLabOpenIssue(url: string) {
if (!this.config.gitlab) {
return this.sendNotice("The bridge is not configured with GitLab support");
}
const urlResult = GitLabClient.splitUrlIntoParts(this.config.gitlab.instances, url);
if (!urlResult) {
return this.sendNotice("The URL was not understood. The URL must be an issue and the bridge must know of the GitLab instance.");
}
const [instanceName, parts] = urlResult;
const instance = this.config.gitlab.instances[instanceName];
const client = await this.tokenStore.getGitLabForUser(this.userId, instance.url);
if (!client) {
return this.sendNotice("You have not added a personal access token for GitLab");
}
const getIssueOpts = {
issue: parseInt(parts[parts.length-1]),
projects: parts.slice(0, parts.length-3), // Remove - and /issues
};
log.info(`Looking up issue ${instanceName} ${getIssueOpts.projects.join("/")}#${getIssueOpts.issue}`);
const issue = await client.issues.get(getIssueOpts);
this.emit('open.gitlab-issue', getIssueOpts, issue, instanceName, instance);
}
@botCommand("gitlab personaltoken", "Set your personal access token for GitLab", ['instanceName', 'accessToken'])
// @ts-ignore - property is used
private async setGitLabPersonalAccessToken(instanceName: string, accessToken: string) {
let me: GetUserResponse;
if (!this.config.gitlab) {
return this.sendNotice("The bridge is not configured with GitLab support");
}
const instance = this.config.gitlab.instances[instanceName];
if (!instance) {
return this.sendNotice("The bridge is not configured for this GitLab instance");
}
try {
const client = new GitLabClient(instance.url, accessToken);
me = await client.user();
client.issues
} catch (ex) {
log.error("Gitlab auth error:", ex);
await this.sendNotice("Could not authenticate with GitLab. Is your token correct?");
return;
}
await this.sendNotice(`Connected as ${me.username}. Token stored`);
await this.tokenStore.storeUserToken("gitlab", this.userId, accessToken, instance.url);
}
@botCommand("gitlab hastoken", "Check if you have a token stored for GitLab", ["instanceName"])
// @ts-ignore - property is used
private async gitlabHasPersonalToken(instanceName: string) {
if (!this.config.gitlab) {
return this.sendNotice("The bridge is not configured with GitLab support");
}
const instance = this.config.gitlab.instances[instanceName];
if (!instance) {
return this.sendNotice("The bridge is not configured for this GitLab instance");
}
const result = await this.tokenStore.getUserToken("gitlab", this.userId, instance.url);
if (result === null) {
await this.sendNotice("You do not currently have a token stored");
return;
}
await this.sendNotice("A token is stored for your GitLab account.");
}
@botCommand("gitlab notifications toggle", "Toggle enabling/disabling GitHub notifications in this room", ["instanceName"])
// @ts-ignore - property is used
private async setGitLabNotificationsStateToggle(instanceName: string) {
if (!this.config.gitlab) {
return this.sendNotice("The bridge is not configured with GitLab support");
}
const instance = this.config.gitlab.instances[instanceName];
if (!instance) {
return this.sendNotice("The bridge is not configured for this GitLab instance");
}
const hasClient = await this.tokenStore.getGitLabForUser(this.userId, instance.url);
if (!hasClient) {
return this.sendNotice("You do not have a GitLab token configured for this instance");
}
let newValue = false;
await this.saveAccountData((data) => {
const currentNotifs = (data.gitlab || {})[instanceName].notifications;
console.log("current:", currentNotifs.enabled);
newValue = !currentNotifs.enabled;
return {
...data,
gitlab: {
[instanceName]: {
notifications: {
enabled: newValue,
},
}
},
};
});
await this.sendNotice(`${newValue ? "En" : "Dis"}abled GitLab notifications for ${instanceName}`);
}
private async saveAccountData(updateFn: (record: AdminAccountData) => AdminAccountData) {
const oldData: AdminAccountData = await this.botIntent.underlyingClient.getRoomAccountData(
BRIDGE_ROOM_TYPE, this.roomId,
);
const newData = updateFn(oldData);
await this.botIntent.underlyingClient.setRoomAccountData(BRIDGE_ROOM_TYPE, this.roomId, newData);
this.emit("settings.changed", this, oldData, newData);
return newData;
}
public async handleCommand(eventId: string, command: string) {
const { error, handled } = await handleCommand(this.userId, command, AdminRoom.botCommands, this);
if (!handled) {
return this.sendNotice("Command not understood");
@ -286,6 +446,7 @@ export class AdminRoom extends EventEmitter {
}
}
const res = compileBotCommands(AdminRoom.prototype);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const res = compileBotCommands(AdminRoom.prototype as any);
AdminRoom.helpMessage = res.helpMessage;
AdminRoom.botCommands = res.botCommands;

22
src/BotCommands.ts
View File

@ -1,11 +1,11 @@
import markdown from "markdown-it";
// @ts-ignore
import argvSplit from "argv-split";
import stringArgv from "string-argv";
import { MatrixMessageContent } from "./MatrixEvent";
const md = new markdown();
export const botCommandSymbol = Symbol("botCommandMetadata");
export function botCommand(prefix: string, help: string, requiredArgs: string[] = [], optionalArgs: string[] = [], includeUserId: boolean = false) {
export function botCommand(prefix: string, help: string, requiredArgs: string[] = [], optionalArgs: string[] = [], includeUserId = false) {
return Reflect.metadata(botCommandSymbol, {
prefix,
help,
@ -15,21 +15,23 @@ export function botCommand(prefix: string, help: string, requiredArgs: string[]
});
}
type BotCommandFunction = (...args: string[]) => Promise<{status: boolean}>;
export type BotCommands = {[prefix: string]: {
fn: (...args: string[]) => Promise<{status: boolean}>,
fn: BotCommandFunction,
requiredArgs: string[],
optionalArgs: string[],
includeUserId: boolean,
}};
export function compileBotCommands(prototype: any): {helpMessage: any, botCommands: BotCommands} {
export function compileBotCommands(prototype: Record<string, BotCommandFunction>): {helpMessage: MatrixMessageContent, botCommands: BotCommands} {
let content = "Commands:\n";
let botCommands: BotCommands = {};
const botCommands: BotCommands = {};
Object.getOwnPropertyNames(prototype).forEach(propetyKey => {
const b = Reflect.getMetadata(botCommandSymbol, prototype, propetyKey);
if (b) {
const requiredArgs = b.requiredArgs.join((arg: string) => `__${arg}__`);
const optionalArgs = b.optionalArgs.join((arg: string) => `\[${arg}\]`);
const requiredArgs = b.requiredArgs.join(" ");
const optionalArgs = b.optionalArgs.map((arg: string) => `[${arg}]`).join(" ");
content += ` - \`${b.prefix}\` ${requiredArgs} ${optionalArgs} - ${b.help}\n`;
// We know that this is safe.
botCommands[b.prefix as string] = {
@ -51,8 +53,8 @@ export function compileBotCommands(prototype: any): {helpMessage: any, botComman
}
}
export async function handleCommand(userId: string, command: string, botCommands: BotCommands, obj: any): Promise<{error?: string, handled?: boolean}> {
const parts = argvSplit(command);
export async function handleCommand(userId: string, command: string, botCommands: BotCommands, obj: unknown): Promise<{error?: string, handled?: boolean}> {
const parts = stringArgv(command);
for (let i = parts.length; i > 0; i--) {
const prefix = parts.slice(0, i).join(" ").toLowerCase();
// We have a match!

31
src/CommentProcessor.ts
View File

@ -1,4 +1,3 @@
import { Octokit } from "@octokit/rest";
import { Appservice } from "matrix-bot-sdk";
import markdown from "markdown-it";
import mime from "mime";
@ -7,6 +6,8 @@ import { MatrixMessageContent, MatrixEvent } from "./MatrixEvent";
import LogWrapper from "./LogWrapper";
import axios from "axios";
import { FormatUtil } from "./FormatUtil";
import { IssuesGetCommentResponseData, ReposGetResponseData, IssuesGetResponseData } from "@octokit/types";
import { IGitLabWebhookNoteEvent } from "./Gitlab/WebhookTypes";
const REGEX_MENTION = /(^|\s)(@[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,38})(\s|$)/ig;
const REGEX_MATRIX_MENTION = /<a href="https:\/\/matrix\.to\/#\/(.+)">(.*)<\/a>/gmi;
@ -14,11 +15,8 @@ const REGEX_IMAGES = /!\[.*]\((.*\.(\w+))\)/gm;
const md = new markdown();
const log = new LogWrapper("CommentProcessor");
interface IMatrixCommentEvent {
msgtype: string;
body: string;
formatted_body: string;
format: string;
interface IMatrixCommentEvent extends MatrixMessageContent {
// eslint-disable-next-line camelcase
external_url: string;
"uk.half-shot.matrix-github.comment": {
id: number;
@ -59,9 +57,9 @@ export class CommentProcessor {
return body;
}
public async getEventBodyForComment(comment: Octokit.IssuesGetCommentResponse,
repo?: Octokit.ReposGetResponse,
issue?: Octokit.IssuesGetResponse): Promise<IMatrixCommentEvent> {
public async getEventBodyForGitHubComment(comment: IssuesGetCommentResponseData,
repo?: ReposGetResponseData,
issue?: IssuesGetResponseData): Promise<IMatrixCommentEvent> {
let body = comment.body;
body = this.replaceMentions(body);
body = await this.replaceImages(body, true);
@ -76,6 +74,21 @@ export class CommentProcessor {
};
}
public async getEventBodyForGitLabNote(comment: IGitLabWebhookNoteEvent): Promise<MatrixMessageContent> {
let body = comment.object_attributes.description;
body = this.replaceMentions(body);
body = await this.replaceImages(body, true);
body = emoji.emojify(body);
const htmlBody = md.render(body);
return {
body,
formatted_body: htmlBody,
msgtype: "m.text",
format: "org.matrix.custom.html",
// ...FormatUtil.getPartialBodyForComment(comment, repo, issue)
};
}
private replaceMentions(body: string): string {
return body.replace(REGEX_MENTION, (match: string, part1: string, githubId: string) => {
const userId = this.as.getUserIdForSuffix(githubId.substr(1));

38
src/Config.ts
View File

@ -2,50 +2,52 @@ import YAML from "yaml";
import { promises as fs } from "fs";
import { IAppserviceRegistration } from "matrix-bot-sdk";
interface BridgeConfigGitHub {
export interface BridgeConfigGitHub {
auth: {
id: number|string;
privateKeyFile: string;
};
webhook: {
port: number;
bindAddress: string;
secret: string;
},
userTokens: {
[userId: string]: string;
}
oauth: {
// eslint-disable-next-line camelcase
client_id: string;
// eslint-disable-next-line camelcase
client_secret: string;
// eslint-disable-next-line camelcase
redirect_uri: string;
};
installationId: number|string;
}
export interface GitLabInstance {
url: string;
// oauth: {
// client_id: string;
// client_secret: string;
// redirect_uri: string;
// };
}
interface BridgeConfigGitLab {
auth: {
id: number|string;
privateKeyFile: string;
};
webhook: {
port: number;
bindAddress: string;
secret: string;
},
userTokens: {
[userId: string]: string;
}
oauth: {
client_id: string;
client_secret: string;
redirect_uri: string;
};
instances: {[name: string]: GitLabInstance};
}
export interface BridgeConfig {
github: BridgeConfigGitHub;
gitlab: BridgeConfigGitLab;
github?: BridgeConfigGitHub;
gitlab?: BridgeConfigGitLab;
webhook: {
port: number;
bindAddress: string;
};
bridge: {
domain: string;
url: string;

52
src/Connections/GithubIssue.ts
View File

@ -11,12 +11,15 @@ import { getIntentForUser } from "../IntentUtils";
import { FormatUtil } from "../FormatUtil";
import { IGitHubWebhookEvent } from "../GithubWebhooks";
import axios from "axios";
import { GithubInstance } from "../Github/GithubInstance";
import { IssuesGetResponseData } from "@octokit/types";
export interface GitHubIssueConnectionState {
org: string;
repo: string;
state: string;
issues: string[];
// eslint-disable-next-line camelcase
comments_processed: number;
}
@ -43,15 +46,19 @@ export class GitHubIssueConnection implements IConnection {
static readonly QueryRoomRegex = /#github_(.+)_(.+)_(\d+):.*/;
static async onQueryRoom(result: RegExpExecArray, opts: IQueryRoomOpts): Promise<any> {
const parts = result!.slice(1);
static async onQueryRoom(result: RegExpExecArray, opts: IQueryRoomOpts): Promise<unknown> {
const parts = result?.slice(1);
if (!parts) {
log.error("Invalid alias pattern");
throw Error("Could not find issue");
}
const owner = parts[0];
const repo = parts[1];
const issueNumber = parseInt(parts[2], 10);
log.info(`Fetching ${owner}/${repo}/${issueNumber}`);
let issue: Octokit.IssuesGetResponse;
let issue: IssuesGetResponseData;
try {
issue = (await opts.octokit.issues.get({
owner,
@ -126,7 +133,7 @@ export class GitHubIssueConnection implements IConnection {
private tokenStore: UserTokenStore,
private commentProcessor: CommentProcessor,
private messageClient: MessageSenderClient,
private octokit: Octokit) { }
private github: GithubInstance) { }
public isInterestedInStateEvent(eventType: string, stateKey: string) {
return GitHubIssueConnection.EventTypes.includes(eventType) && this.stateKey === stateKey;
@ -145,7 +152,10 @@ export class GitHubIssueConnection implements IConnection {
}
public async onCommentCreated(event: IGitHubWebhookEvent, updateState = true) {
const comment = event.comment!;
const comment = event.comment;
if (!comment || !comment.user) {
throw Error('Comment undefined');
}
if (event.repository) {
// Delay to stop comments racing sends
await new Promise((resolve) => setTimeout(resolve, 500));
@ -153,8 +163,11 @@ export class GitHubIssueConnection implements IConnection {
return;
}
}
const commentIntent = await getIntentForUser(comment.user, this.as, this.octokit);
const matrixEvent = await this.commentProcessor.getEventBodyForComment(comment, event.repository, event.issue);
const commentIntent = await getIntentForUser({
login: comment.user.login,
avatarUrl: comment.user.avatar_url,
}, this.as);
const matrixEvent = await this.commentProcessor.getEventBodyForGitHubComment(comment, event.repository, event.issue);
await this.messageClient.sendMatrixMessage(this.roomId, matrixEvent, "m.room.message", commentIntent.userId);
if (!updateState) {
@ -171,7 +184,7 @@ export class GitHubIssueConnection implements IConnection {
private async syncIssueState() {
log.debug("Syncing issue state for", this.roomId);
const issue = await this.octokit.issues.get({
const issue = await this.github.octokit.issues.get({
owner: this.state.org,
repo: this.state.repo,
issue_number: this.issueNumber,
@ -179,7 +192,10 @@ export class GitHubIssueConnection implements IConnection {
if (this.state.comments_processed === -1) {
// This has a side effect of creating a profile for the user.
const creator = await getIntentForUser(issue.data.user, this.as, this.octokit);
const creator = await getIntentForUser({
login: issue.data.user.login,
avatarUrl: issue.data.user.avatar_url
}, this.as);
// We've not sent any messages into the room yet, let's do it!
if (issue.data.body) {
await this.messageClient.sendMatrixMessage(this.roomId, {
@ -198,7 +214,7 @@ export class GitHubIssueConnection implements IConnection {
}
if (this.state.comments_processed !== issue.data.comments) {
const comments = (await this.octokit.issues.listComments({
const comments = (await this.github.octokit.issues.listComments({
owner: this.state.org,
repo: this.state.repo,
issue_number: this.issueNumber,
@ -240,7 +256,7 @@ export class GitHubIssueConnection implements IConnection {
}
public async onMatrixIssueComment(event: MatrixEvent<MatrixMessageContent>, allowEcho: boolean = false) {
public async onMatrixIssueComment(event: MatrixEvent<MatrixMessageContent>, allowEcho = false) {
const clientKit = await this.tokenStore.getOctokitForUser(event.sender);
if (clientKit === null) {
await this.as.botClient.sendEvent(this.roomId, "m.reaction", {
@ -272,25 +288,17 @@ export class GitHubIssueConnection implements IConnection {
return; // No changes made.
}
if (event.changes.title) {
if (event.issue && event.changes.title) {
await this.as.botIntent.underlyingClient.sendStateEvent(this.roomId, "m.room.name", "", {
name: FormatUtil.formatIssueRoomName(event.issue!),
name: FormatUtil.formatIssueRoomName(event.issue),
});
}
}
public onIssueStateChange(event: IGitHubWebhookEvent) {
public onIssueStateChange() {
return this.syncIssueState();
}
public async onEvent() {
}
public async onStateUpdate() {
}
public async onMessageEvent(ev: MatrixEvent<MatrixMessageContent>) {
if (ev.content.body === '!sync') {
// Sync data.

14
src/Connections/GithubProject.ts
View File

@ -1,13 +1,13 @@
import { IConnection } from "./IConnection";
import { Appservice } from "matrix-bot-sdk";
import LogWrapper from "../LogWrapper";
import { Octokit } from "@octokit/rest";
import { ProjectsGetResponseData } from "@octokit/types";
export interface GitHubProjectConnectionState {
// eslint-disable-next-line camelcase
project_id: number;
state: "open"|"closed";
}
const log = new LogWrapper("GitHubProjectConnection");
/**
@ -20,7 +20,7 @@ export class GitHubProjectConnection implements IConnection {
GitHubProjectConnection.CanonicalEventType, // Legacy event, with an awful name.
];
static async onOpenProject(project: Octokit.ProjectsGetResponse, as: Appservice, inviteUser: string): Promise<GitHubProjectConnection> {
static async onOpenProject(project: ProjectsGetResponseData, as: Appservice, inviteUser: string): Promise<GitHubProjectConnection> {
log.info(`Fetching ${project.name} ${project.id}`);
// URL hack so we don't need to fetch the repo itself.
@ -57,14 +57,6 @@ export class GitHubProjectConnection implements IConnection {
return GitHubProjectConnection.EventTypes.includes(eventType) && this.stateKey === stateKey;
}
public async onEvent() {
}
public async onStateUpdate() {
}
public toString() {
return `GitHubProjectConnection ${this.state.project_id}}`;
}

60
src/Connections/GithubRepo.ts
View File

@ -1,3 +1,4 @@
/* eslint-disable @typescript-eslint/ban-ts-comment */
import { IConnection } from "./IConnection";
import { Appservice } from "matrix-bot-sdk";
import { MatrixMessageContent, MatrixEvent, MatrixReactionContent } from "../MatrixEvent";
@ -11,6 +12,7 @@ import { FormatUtil } from "../FormatUtil";
import axios from "axios";
import { BotCommands, handleCommand, botCommand, compileBotCommands } from "../BotCommands";
import { IGitHubWebhookEvent } from "../GithubWebhooks";
import { ReposGetResponseData } from "@octokit/types";
const log = new LogWrapper("GitHubRepoConnection");
const md = new markdown();
@ -46,7 +48,7 @@ const ALLOWED_REACTIONS = {
"👐": "open",
}
function compareEmojiStrings(e0: string, e1: string, e0Index: number = 0) {
function compareEmojiStrings(e0: string, e1: string, e0Index = 0) {
return e0.codePointAt(e0Index) === e1.codePointAt(0);
}
@ -62,15 +64,19 @@ export class GitHubRepoConnection implements IConnection {
static readonly QueryRoomRegex = /#github_(.+)_(.+):.*/;
static async onQueryRoom(result: RegExpExecArray, opts: IQueryRoomOpts): Promise<any> {
const parts = result!.slice(1);
static async onQueryRoom(result: RegExpExecArray, opts: IQueryRoomOpts): Promise<unknown> {
const parts = result?.slice(1);
if (!parts) {
log.error("Invalid alias pattern");
throw Error("Could not find repo");
}
const owner = parts[0];
const repo = parts[1];
const issueNumber = parseInt(parts[2], 10);
log.info(`Fetching ${owner}/${repo}/${issueNumber}`);
let repoRes: Octokit.ReposGetResponse;
let repoRes: ReposGetResponseData;
try {
repoRes = (await opts.octokit.repos.get({
owner,
@ -135,14 +141,13 @@ export class GitHubRepoConnection implements IConnection {
};
}
static helpMessage: any;
static helpMessage: MatrixMessageContent;
static botCommands: BotCommands;
constructor(public readonly roomId: string,
private readonly as: Appservice,
private readonly state: GitHubRepoConnectionState,
private readonly tokenStore: UserTokenStore,
octokit: Octokit) {
private readonly tokenStore: UserTokenStore) {
}
@ -154,7 +159,7 @@ export class GitHubRepoConnection implements IConnection {
return this.state.repo;
}
public isInterestedInStateEvent(eventType: string) {
public isInterestedInStateEvent() {
return false;
}
@ -252,13 +257,18 @@ export class GitHubRepoConnection implements IConnection {
public async onIssueCreated(event: IGitHubWebhookEvent) {
log.info(`onIssueCreated ${this.roomId} ${this.org}/${this.repo} #${event.issue?.number}`);
const orgRepoName = event.issue!.repository_url.substr("https://api.github.com/repos/".length);
const content = `New issue created [${orgRepoName}#${event.issue!.number}](${event.issue!.html_url}): "${event.issue!.title}"`;
console.log(event.issue?.labels);
const labelsHtml = event.issue?.labels.map((label) =>
if (!event.issue) {
throw Error('No issue content!');
}
if (!event.repository) {
throw Error('No repository content!');
}
const orgRepoName = event.issue.repository_url.substr("https://api.github.com/repos/".length);
const content = `New issue created [${orgRepoName}#${event.issue.number}](${event.issue.html_url}): "${event.issue.title}"`;
const labelsHtml = event.issue.labels.map((label: {color: string, name: string, description: string}) =>
`<span title="${label.description}" data-mx-color="#CCCCCC" data-mx-bg-color="#${label.color}">${label.name}</span>`
).join(" ") || "";
const labels = event.issue?.labels.map((label) =>
const labels = event.issue?.labels.map((label: {name: string}) =>
label.name
).join(", ") || "";
await this.as.botIntent.sendEvent(this.roomId, {
@ -266,21 +276,27 @@ export class GitHubRepoConnection implements IConnection {
body: content + (labels.length > 0 ? ` with labels ${labels}`: ""),
formatted_body: md.renderInline(content) + (labelsHtml.length > 0 ? ` with labels ${labelsHtml}`: ""),
format: "org.matrix.custom.html",
...FormatUtil.getPartialBodyForIssue(event.repository!, event.issue!),
...FormatUtil.getPartialBodyForIssue(event.repository, event.issue),
});
}
public async onIssueStateChange(event: IGitHubWebhookEvent) {
log.info(`onIssueStateChange ${this.roomId} ${this.org}/${this.repo} #${event.issue?.number}`);
if (event.issue?.state === "closed") {
const orgRepoName = event.issue!.repository_url.substr("https://api.github.com/repos/".length);
const content = `**@${event.sender!.login}** closed issue [${orgRepoName}#${event.issue!.number}](${event.issue!.html_url}): "${event.issue!.title}"`;
if (!event.issue) {
throw Error('No issue content!');
}
if (!event.repository) {
throw Error('No repository content!');
}
if (event.issue.state === "closed" && event.sender) {
const orgRepoName = event.issue.repository_url.substr("https://api.github.com/repos/".length);
const content = `**@${event.sender.login}** closed issue [${orgRepoName}#${event.issue.number}](${event.issue.html_url}): "${event.issue.title}"`;
await this.as.botIntent.sendEvent(this.roomId, {
msgtype: "m.notice",
body: content,
formatted_body: md.renderInline(content),
format: "org.matrix.custom.html",
...FormatUtil.getPartialBodyForIssue(event.repository!, event.issue!),
...FormatUtil.getPartialBodyForIssue(event.repository, event.issue),
});
}
}
@ -291,6 +307,7 @@ export class GitHubRepoConnection implements IConnection {
return;
}
if (evt.type === 'm.reaction') {
// eslint-disable-next-line camelcase
const {event_id, key} = (evt.content as MatrixReactionContent)["m.relates_to"];
const ev = await this.as.botClient.getEvent(this.roomId, event_id);
const issueContent = ev.content["uk.half-shot.matrix-github.issue"];
@ -298,7 +315,7 @@ export class GitHubRepoConnection implements IConnection {
return; // Not our event.
}
const [,reactionName] = Object.entries(GITHUB_REACTION_CONTENT).find(([emoji, content]) => compareEmojiStrings(emoji, key)) || [];;
const [,reactionName] = Object.entries(GITHUB_REACTION_CONTENT).find(([emoji]) => compareEmojiStrings(emoji, key)) || [];
const [,action] = Object.entries(ALLOWED_REACTIONS).find(([emoji]) => compareEmojiStrings(emoji, key)) || [];
if (reactionName) {
log.info(`Sending reaction of ${reactionName} for ${this.org}${this.repo}#${issueContent.number}`)
@ -333,13 +350,12 @@ export class GitHubRepoConnection implements IConnection {
}
}
public async onStateUpdate() { }
public toString() {
return `GitHubRepo`;
}
}
const res = compileBotCommands(GitHubRepoConnection.prototype);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const res = compileBotCommands(GitHubRepoConnection.prototype as any);
GitHubRepoConnection.helpMessage = res.helpMessage;
GitHubRepoConnection.botCommands = res.botCommands;

207
src/Connections/GitlabIssue.ts Normal file
View File

@ -0,0 +1,207 @@
import { IConnection } from "./IConnection";
import { Appservice } from "matrix-bot-sdk";
import { MatrixMessageContent, MatrixEvent } from "../MatrixEvent";
import markdown from "markdown-it";
import { UserTokenStore } from "../UserTokenStore";
import LogWrapper from "../LogWrapper";
import { CommentProcessor } from "../CommentProcessor";
import { MessageSenderClient } from "../MatrixSender";
import { FormatUtil } from "../FormatUtil";
import { IGitHubWebhookEvent } from "../GithubWebhooks";
import { GitLabInstance } from "../Config";
import { GetIssueResponse } from "../Gitlab/Types";
import { IGitLabWebhookNoteEvent } from "../Gitlab/WebhookTypes";
import { getIntentForUser } from "../IntentUtils";
export interface GitLabIssueConnectionState {
instance: string;
projects: string[];
state: string;
iid: number;
id: number;
authorName: string;
}
const log = new LogWrapper("GitLabIssueConnection");
const md = new markdown();
md.render("foo");
// interface IQueryRoomOpts {
// as: Appservice;
// tokenStore: UserTokenStore;
// commentProcessor: CommentProcessor;
// messageClient: MessageSenderClient;
// octokit: Octokit;
// }
/**
* Handles rooms connected to a github repo.
*/
export class GitLabIssueConnection implements IConnection {
static readonly CanonicalEventType = "uk.half-shot.matrix-github.gitlab.issue";
static readonly EventTypes = [
GitLabIssueConnection.CanonicalEventType,
];
static readonly QueryRoomRegex = /#gitlab_(.+)_(.+)_(\d+):.*/;
static getTopicString(authorName: string, state: string) {
`Author: ${authorName} | State: ${state === "closed" ? "closed" : "open"}`
}
public static async createRoomForIssue(instanceName: string, instance: GitLabInstance,
issue: GetIssueResponse, projects: string[], as: Appservice,
tokenStore: UserTokenStore, commentProcessor: CommentProcessor,
messageSender: MessageSenderClient) {
const state: GitLabIssueConnectionState = {
projects,
state: issue.state,
iid: issue.iid,
id: issue.id,
instance: instanceName,
authorName: issue.author.name,
};
const roomId = await as.botClient.createRoom({
visibility: "private",
name: `${issue.references.full}`,
topic: GitLabIssueConnection.getTopicString(issue.author.name, issue.state),
preset: "private_chat",
invite: [],
initial_state: [
{
type: this.CanonicalEventType,
content: state,
state_key: issue.web_url,
},
],
});
return new GitLabIssueConnection(roomId, as, state, issue.web_url, tokenStore, commentProcessor, messageSender, instance);
}
public get projectPath() {
return this.state.projects.join("/");
}
public get instanceUrl() {
return this.instance.url;
}
constructor(public readonly roomId: string,
private readonly as: Appservice,
private state: GitLabIssueConnectionState,
private readonly stateKey: string,
private tokenStore: UserTokenStore,
private commentProcessor: CommentProcessor,
private messageClient: MessageSenderClient,
private instance: GitLabInstance,) {
}
public isInterestedInStateEvent(eventType: string, stateKey: string) {
return GitLabIssueConnection.EventTypes.includes(eventType) && this.stateKey === stateKey;
}
public get issueNumber() {
return this.state.iid;
}
public async onCommentCreated(event: IGitLabWebhookNoteEvent) {
log.info(`${this.toString()} onCommentCreated ${event.object_attributes.noteable_id}`);
console.log(event);
if (event.repository) {
// Delay to stop comments racing sends
await new Promise((resolve) => setTimeout(resolve, 500));
if (this.commentProcessor.hasCommentBeenProcessed(
this.state.instance,
this.state.projects.join("/"),
this.state.iid.toString(),
event.object_attributes.id)) {
return;
}
}
const commentIntent = await getIntentForUser({
login: event.user.name,
avatarUrl: event.user.avatar_url,
}, this.as);
const matrixEvent = await this.commentProcessor.getEventBodyForGitLabNote(event);
await this.messageClient.sendMatrixMessage(this.roomId, matrixEvent, "m.room.message", commentIntent.userId);
}
public async onMatrixIssueComment(event: MatrixEvent<MatrixMessageContent>, allowEcho = false) {
const clientKit = await this.tokenStore.getGitLabForUser(event.sender, this.instanceUrl);
if (clientKit === null) {
await this.as.botClient.sendEvent(this.roomId, "m.reaction", {
"m.relates_to": {
rel_type: "m.annotation",
event_id: event.event_id,
key: "⚠️ Not bridged",
}
})
log.info("Ignoring comment, user is not authenticated");
return;
}
const result = await clientKit.notes.createForIssue(
this.state.projects,
this.state.iid, {
body: await this.commentProcessor.getCommentBodyForEvent(event, false),
}
);
log.info(`${this.toString()} created note ${result.noteable_id} for ${event.event_id} ${event.sender}`);
if (!allowEcho) {
this.commentProcessor.markCommentAsProcessed(
this.state.instance,
this.state.projects.join("/"),
this.state.iid.toString(),
result.id,
);
}
}
public async onIssueReopened() {
// TODO: We don't store the author data.
this.state.state = "reopened";
await this.as.botClient.sendStateEvent(this.roomId, GitLabIssueConnection.CanonicalEventType, this.stateKey, this.state);
return this.as.botClient.sendStateEvent(this.roomId, "m.room.topic", "", {
topic: GitLabIssueConnection.getTopicString(this.state.authorName, this.state.state),
});
}
public async onIssueClosed() {
// TODO: We don't store the author data.
this.state.state = "closed";
await this.as.botClient.sendStateEvent(this.roomId, GitLabIssueConnection.CanonicalEventType, this.stateKey , this.state);
return this.as.botClient.sendStateEvent(this.roomId, "m.room.topic", "", {
topic: GitLabIssueConnection.getTopicString(this.state.authorName, this.state.state),
});
}
public async onIssueEdited(event: IGitHubWebhookEvent) {
if (!event.changes) {
log.debug("No changes given");
return; // No changes made.
}
if (event.issue && event.changes.title) {
await this.as.botIntent.underlyingClient.sendStateEvent(this.roomId, "m.room.name", "", {
name: FormatUtil.formatIssueRoomName(event.issue),
});
}
}
public async onMessageEvent(ev: MatrixEvent<MatrixMessageContent>) {
if (ev.content.body === '!sync') {
// Sync data.
// return this.syncIssueState();
}
await this.onMatrixIssueComment(ev);
}
public toString() {
return `GitLabIssue ${this.instanceUrl}/${this.projectPath}#${this.issueNumber}`;
}
}

30
src/Connections/GitlabRepo.ts
View File

@ -1,3 +1,5 @@
// We need to instantiate some functions which are not directly called, which confuses typescript.
/* eslint-disable @typescript-eslint/ban-ts-comment */
import { IConnection } from "./IConnection";
import { UserTokenStore } from "../UserTokenStore";
import { Appservice } from "matrix-bot-sdk";
@ -5,15 +7,16 @@ import { BotCommands, handleCommand, botCommand, compileBotCommands } from "../B
import { MatrixEvent, MatrixMessageContent } from "../MatrixEvent";
import markdown from "markdown-it";
import LogWrapper from "../LogWrapper";
import { GitLabInstance } from "../Config";
export interface GitLabRepoConnectionState {
instance_url: string;
instance: string;
org: string;
repo: string;
state: string;
}
const log = new LogWrapper("GitHubRepoConnection");
const log = new LogWrapper("GitLabRepoConnection");
const md = new markdown();
/**
@ -26,13 +29,14 @@ export class GitLabRepoConnection implements IConnection {
GitLabRepoConnection.CanonicalEventType, // Legacy event, with an awful name.
];
static helpMessage: any;
static helpMessage: MatrixMessageContent;
static botCommands: BotCommands;
constructor(public readonly roomId: string,
private readonly as: Appservice,
private readonly state: GitLabRepoConnectionState,
private readonly tokenStore: UserTokenStore) {
private readonly tokenStore: UserTokenStore,
private readonly instance: GitLabInstance) {
}
@ -44,7 +48,7 @@ export class GitLabRepoConnection implements IConnection {
return this.state.repo;
}
public isInterestedInStateEvent(eventType: string) {
public isInterestedInStateEvent() {
return false;
}
@ -74,7 +78,7 @@ export class GitLabRepoConnection implements IConnection {
@botCommand("gl create", "Create an issue for this repo", ["title"], ["description", "labels"], true)
// @ts-ignore
private async onCreateIssue(userId: string, title: string, description?: string, labels?: string) {
const client = await this.tokenStore.getGitLabForUser(userId, this.state.instance_url);
const client = await this.tokenStore.getGitLabForUser(userId, this.instance.url);
if (!client) {
await this.as.botIntent.sendText(this.roomId, "You must login to create an issue", "m.notice");
throw Error('Not logged in');
@ -97,8 +101,8 @@ export class GitLabRepoConnection implements IConnection {
@botCommand("gl close", "Close an issue", ["number"], ["comment"], true)
// @ts-ignore
private async onClose(userId: string, number: string, comment?: string) {
const client = await this.tokenStore.getGitLabForUser(userId, this.state.instance_url);
private async onClose(userId: string, number: string) {
const client = await this.tokenStore.getGitLabForUser(userId, this.instance.url);
if (!client) {
await this.as.botIntent.sendText(this.roomId, "You must login to create an issue", "m.notice");
throw Error('Not logged in');
@ -119,17 +123,13 @@ export class GitLabRepoConnection implements IConnection {
// }
public async onEvent(evt: MatrixEvent<unknown>) {
}
public async onStateUpdate() { }
public toString() {
return `GitHubRepo`;
}
}
const res = compileBotCommands(GitLabRepoConnection.prototype);
// Typescript doesn't understand Prototypes very well yet.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const res = compileBotCommands(GitLabRepoConnection.prototype as any);
GitLabRepoConnection.helpMessage = res.helpMessage;
GitLabRepoConnection.botCommands = res.botCommands;

9
src/Connections/IConnection.ts
View File

@ -6,22 +6,17 @@ export interface IConnection {
/**
* When a room gets an update to it's state.
*/
onStateUpdate: (ev: any) => Promise<void>;
onStateUpdate?: (ev: MatrixEvent<unknown>) => Promise<void>;
/**
* When a room gets any event
*/
onEvent: (ev: MatrixEvent<unknown>) => Promise<void>;
onEvent?: (ev: MatrixEvent<unknown>) => Promise<void>;
/**
* When a room gets a message event
*/
onMessageEvent?: (ev: MatrixEvent<MatrixMessageContent>) => Promise<void>;
/**
* When a comment is created on a repo
*/
onCommentCreated?: (ev: IGitHubWebhookEvent) => Promise<void>;
onIssueCreated?: (ev: IGitHubWebhookEvent) => Promise<void>;
onIssueStateChange?: (ev: IGitHubWebhookEvent) => Promise<void>;

17
src/FormatUtil.ts
View File

@ -1,4 +1,4 @@
import { Octokit } from "@octokit/rest";
import { IssuesGetCommentResponseData, IssuesGetResponseData, ProjectsListForOrgResponseData, ProjectsListForUserResponseData, ProjectsListForRepoResponseData } from "@octokit/types";
interface IMinimalRepository {
id: number;
@ -12,9 +12,8 @@ export class FormatUtil {
return `${orgRepoName}#${issue.number}: ${issue.title}`;
}
public static formatRepoRoomName(repo: {full_name: string, url: string}) {
const orgRepoName = repo.url.substr("https://api.github.com/repos/".length);
return `${orgRepoName}: ${repo.full_name}`;
public static formatRepoRoomName(repo: {full_name: string, description: string}) {
return `${repo.full_name}: ${repo.description}`;
}
public static formatRoomTopic(repo: {state: string, html_url: string}) {
@ -36,7 +35,7 @@ export class FormatUtil {
};
}
public static getPartialBodyForIssue(repo: IMinimalRepository, issue: Octokit.IssuesGetResponse) {
public static getPartialBodyForIssue(repo: IMinimalRepository, issue: IssuesGetResponseData) {
return {
...FormatUtil.getPartialBodyForRepo(repo),
"external_url": issue.html_url,
@ -50,9 +49,9 @@ export class FormatUtil {
};
}
public static getPartialBodyForComment(comment: Octokit.IssuesGetCommentResponse,
public static getPartialBodyForComment(comment: IssuesGetCommentResponseData,
repo?: IMinimalRepository,
issue?: Octokit.IssuesGetResponse) {
issue?: IssuesGetResponseData) {
return {
...(issue && repo ? FormatUtil.getPartialBodyForIssue(repo, issue) : undefined),
"external_url": comment.html_url,
@ -62,7 +61,7 @@ export class FormatUtil {
};
}
public static projectListing(projectItem: Octokit.ProjectsListForOrgResponseItem|Octokit.ProjectsListForUserResponseItem|Octokit.ProjectsListForRepoResponseItem) {
return `${projectItem.name} (#${projectItem.number}) - Project ID: ${projectItem.id}`
public static projectListing(projectItem: ProjectsListForOrgResponseData|ProjectsListForUserResponseData|ProjectsListForRepoResponseData) {
return `${projectItem[0].name} (#${projectItem[0].number}) - Project ID: ${projectItem[0].id}`
}
}

53
src/Github/GithubInstance.ts Normal file
View File

@ -0,0 +1,53 @@
import { createAppAuth } from "@octokit/auth-app";
import { createTokenAuth } from "@octokit/auth-token";
import { Octokit } from "@octokit/rest";
import { promises as fs } from "fs";
import { BridgeConfigGitHub } from "../Config";
import LogWrapper from "../LogWrapper";
const log = new LogWrapper("GithubInstance");
const USER_AGENT = "matrix-github v0.0.1";
export class GithubInstance {
private internalOctokit!: Octokit;
public get octokit() {
return this.internalOctokit;
}
constructor (private config: BridgeConfigGitHub) {
}
public static createUserOctokit(token: string) {
return new Octokit({
authStrategy: createTokenAuth,
auth: token,
userAgent: USER_AGENT,
});
}
public async start() {
// TODO: Make this generic.
const auth = {
appId: parseInt(this.config.auth.id as string, 10),
privateKey: await fs.readFile(this.config.auth.privateKeyFile, "utf-8"),
installationId: parseInt(this.config.installationId as string, 10),
};
this.internalOctokit = new Octokit({
authStrategy: createAppAuth,
auth,
userAgent: USER_AGENT,
});
try {
await this.octokit.rateLimit.get();
log.info("Auth check success");
} catch (ex) {
log.info("Auth check failed:", ex);
throw Error("Attempting to verify GitHub authentication configration failed");
}
}
}

25
src/Github/Types.ts Normal file
View File

@ -0,0 +1,25 @@
import { IssuesGetResponseData, IssuesGetCommentResponseData, PullsListReviewsResponseData, ReposGetResponseData, PullsListRequestedReviewersResponseData } from "@octokit/types";
/* eslint-disable camelcase */
export interface GitHubUserNotification {
id: string;
reason: "assign"|"author"|"comment"|"invitation"|"manual"|"mention"|"review_requested"|
"security_alert"|"state_change"|"subscribed"|"team_mention";
unread: boolean;
updated_at: number;
last_read_at: number;
url: string;
subject: {
title: string;
url: string;
latest_comment_url: string|null;
type: "PullRequest"|"Issue"|"RepositoryVulnerabilityAlert";
// Probably.
url_data?: IssuesGetResponseData;
latest_comment_url_data?: IssuesGetCommentResponseData;
requested_reviewers?: PullsListRequestedReviewersResponseData;
reviews?: PullsListReviewsResponseData;
};
// Not quite the right type but good nuff.
repository: ReposGetResponseData;
}

350
src/GithubBridge.ts
View File

@ -1,7 +1,6 @@
import { Appservice, IAppserviceRegistration, RichRepliesPreprocessor, IRichReplyMetadata } from "matrix-bot-sdk";
import { Octokit } from "@octokit/rest";
import { createAppAuth } from "@octokit/auth-app";
import { BridgeConfig } from "./Config";
import { ProjectsGetResponseData } from "@octokit/types";
import { BridgeConfig, GitLabInstance } from "./Config";
import { IGitHubWebhookEvent, IOAuthRequest, IOAuthTokens, NotificationsEnableEvent,
NotificationsDisableEvent } from "./GithubWebhooks";
import { CommentProcessor } from "./CommentProcessor";
@ -11,8 +10,7 @@ import { UserTokenStore } from "./UserTokenStore";
import { MatrixEvent, MatrixMemberContent, MatrixMessageContent } from "./MatrixEvent";
import LogWrapper from "./LogWrapper";
import { MessageSenderClient } from "./MatrixSender";
import { promises as fs } from "fs";
import { UserNotificationsEvent } from "./UserNotificationWatcher";
import { UserNotificationsEvent } from "./Notifications/UserNotificationWatcher";
import { RedisStorageProvider } from "./Stores/RedisStorageProvider";
import { MemoryStorageProvider } from "./Stores/MemoryStorageProvider";
import { NotificationProcessor } from "./NotificationsProcessor";
@ -23,12 +21,17 @@ import { GitHubRepoConnection } from "./Connections/GithubRepo";
import { GitHubIssueConnection } from "./Connections/GithubIssue";
import { GitHubProjectConnection } from "./Connections/GithubProject";
import { GitLabRepoConnection } from "./Connections/GitlabRepo";
import { GithubInstance } from "./Github/GithubInstance";
import { IGitLabWebhookIssueStateEvent, IGitLabWebhookMREvent, IGitLabWebhookNoteEvent } from "./Gitlab/WebhookTypes";
import { GitLabIssueConnection } from "./Connections/GitlabIssue";
import { GetIssueResponse, GetIssueOpts } from "./Gitlab/Types"
import { GitLabClient } from "./Gitlab/Client";
// import { IGitLabWebhookMREvent } from "./Gitlab/WebhookTypes";
const log = new LogWrapper("GithubBridge");
export class GithubBridge {
private octokit!: Octokit;
private github?: GithubInstance;
private as!: Appservice;
private adminRooms: Map<string, AdminRoom> = new Map();
private commentProcessor!: CommentProcessor;
@ -42,20 +45,50 @@ export class GithubBridge {
constructor(private config: BridgeConfig, private registration: IAppserviceRegistration) { }
private createConnectionForState(roomId: string, state: MatrixEvent<any>) {
log.debug(`Looking to create connection for ${roomId}`);
if (state.content.disabled === false) {
log.debug(`${roomId} has disabled state for ${state.type}`);
return;
}
if (GitHubRepoConnection.EventTypes.includes(state.type)) {
return new GitHubRepoConnection(roomId, this.as, state.content, this.tokenStore, this.octokit);
if (!this.github) {
throw Error('GitHub is not configured');
}
return new GitHubRepoConnection(roomId, this.as, state.content, this.tokenStore);
}
if (GitHubIssueConnection.EventTypes.includes(state.type)) {
return new GitHubIssueConnection(roomId, this.as, state.content, state.state_key || "", this.tokenStore, this.commentProcessor, this.messageClient, this.octokit);
if (!this.github) {
throw Error('GitHub is not configured');
}
return new GitHubIssueConnection(roomId, this.as, state.content, state.state_key || "", this.tokenStore, this.commentProcessor, this.messageClient, this.github);
}
if (GitLabRepoConnection.EventTypes.includes(state.type)) {
return new GitLabRepoConnection(roomId, this.as, state.content, this.tokenStore);
if (!this.config.gitlab) {
throw Error('GitLab is not configured');
}
const instance = this.config.gitlab.instances[state.content.instance];
if (!instance) {
throw Error('Instance name not recongnised');
}
return new GitLabRepoConnection(roomId, this.as, state.content, this.tokenStore, instance);
}
if (GitLabIssueConnection.EventTypes.includes(state.type)) {
if (!this.config.gitlab) {
throw Error('GitLab is not configured');
}
const instance = this.config.gitlab.instances[state.content.instance];
return new GitLabIssueConnection(
roomId,
this.as,
state.content,
state.state_key as string,
this.tokenStore,
this.commentProcessor,
this.messageClient,
instance);
}
return;
}
@ -65,18 +98,39 @@ export class GithubBridge {
return state.map((event) => this.createConnectionForState(roomId, event)).filter((connection) => !!connection) as unknown as IConnection[];
}
private getConnectionsForGithubIssue(org: string, repo: string, issueNumber: number) {
private getConnectionsForGithubIssue(org: string, repo: string, issueNumber: number): (GitHubIssueConnection|GitLabRepoConnection)[] {
return this.connections.filter((c) => (c instanceof GitHubIssueConnection && c.org === org && c.repo === repo && c.issueNumber === issueNumber) ||
(c instanceof GitHubRepoConnection && c.org === org && c.repo === repo));
(c instanceof GitHubRepoConnection && c.org === org && c.repo === repo)) as (GitHubIssueConnection|GitLabRepoConnection)[];
}
// private getConnectionsForGitLabIssue(org: string, repo: string, issueNumber: number) {
// return this.connections.filter((c) => (c instanceof GitLabRepoConnection && c.org === org && c.repo === repo));
// }
private getConnectionsForGithubRepo(org: string, repo: string): GitHubRepoConnection[] {
return this.connections.filter((c) => (c instanceof GitHubRepoConnection && c.org === org && c.repo === repo)) as GitHubRepoConnection[];
}
private getConnectionsForGitLabIssueWebhook(repoHome: string, issueId: number) {
if (!this.config.gitlab) {
throw Error('GitLab configuration missing, cannot handle note');
}
const res = GitLabClient.splitUrlIntoParts(this.config.gitlab.instances, repoHome);
if (!res) {
throw Error('No instance found for note');
}
const instance = this.config.gitlab.instances[res[0]];
return this.getConnectionsForGitLabIssue(instance, res[1], issueId);
}
private getConnectionsForGitLabIssue(instance: GitLabInstance, projects: string[], issueNumber: number): GitLabIssueConnection[] {
return this.connections.filter((c) => (
c instanceof GitLabIssueConnection &&
c.issueNumber == issueNumber &&
c.instanceUrl == instance.url &&
c.projectPath == projects.join("/")
)) as GitLabIssueConnection[];
}
public stop() {
this.as.stop();
this.queue.stop();
if(this.queue.stop) this.queue.stop();
}
public async start() {
@ -84,25 +138,14 @@ export class GithubBridge {
this.queue = createMessageQueue(this.config);
this.messageClient = new MessageSenderClient(this.queue);
// TODO: Make this generic.
const auth = {
id: parseInt(this.config.github.auth.id as string, 10),
privateKey: await fs.readFile(this.config.github.auth.privateKeyFile, "utf-8"),
installationId: parseInt(this.config.github.installationId as string, 10),
};
if (!this.config.github && !this.config.gitlab) {
log.error("You haven't configured support for GitHub or GitLab!");
throw Error('Bridge cannot start -- no connectors are configured');
}
this.octokit = new Octokit({
authStrategy: createAppAuth,
auth,
userAgent: "matrix-github v0.0.1",
});
try {
await this.octokit.rateLimit.get();
log.info("Auth check success");
} catch (ex) {
log.info("Auth check failed:", ex);
throw Error("Attempting to verify GitHub authentication configration failed");
if (this.config.github) {
this.github = new GithubInstance(this.config.github);
await this.github.start();
}
let storage: IStorageProvider;
@ -160,79 +203,95 @@ export class GithubBridge {
this.queue.subscribe("issue.*");
this.queue.subscribe("response.matrix.message");
this.queue.subscribe("notifications.user.events");
this.queue.subscribe("merge_request.*")
this.queue.subscribe("merge_request.*");
this.queue.subscribe("gitlab.*");
this.queue.on<IGitHubWebhookEvent>("comment.created", async (msg) => {
const connections = this.getConnectionsForGithubIssue(msg.data.repository!.owner.login, msg.data.repository!.name, msg.data.issue!.number);
const validateRepoIssue = (data: IGitHubWebhookEvent) => {
if (!data.repository || !data.issue) {
throw Error("Malformed webhook event, missing repository or issue");
}
return {
repository: data.repository,
issue: data.issue,
};
}
this.queue.on<IGitHubWebhookEvent>("comment.created", async ({ data }) => {
const { repository, issue } = validateRepoIssue(data);
const connections = this.getConnectionsForGithubIssue(repository.owner.login, repository.name, issue.number);
connections.map(async (c) => {
try {
if (c.onCommentCreated)
await c.onCommentCreated(msg.data);
if (c instanceof GitHubIssueConnection)
await c.onCommentCreated(data);
} catch (ex) {
log.warn(`Connection ${c.toString()} failed to handle comment.created:`, ex);
}
})
});
this.queue.on<IGitHubWebhookEvent>("issue.opened", async (msg) => {
const connections = this.getConnectionsForGithubIssue(msg.data.repository!.owner.login, msg.data.repository!.name, msg.data.issue!.number);
this.queue.on<IGitHubWebhookEvent>("issue.opened", async ({ data }) => {
const { repository } = validateRepoIssue(data);
const connections = this.getConnectionsForGithubRepo(repository.owner.login, repository.name);
connections.map(async (c) => {
try {
if (c.onIssueCreated)
await c.onIssueCreated(msg.data);
await c.onIssueCreated(data);
} catch (ex) {
log.warn(`Connection ${c.toString()} failed to handle comment.created:`, ex);
}
})
});
this.queue.on<IGitHubWebhookEvent>("issue.edited", async (msg) => {
const connections = this.getConnectionsForGithubIssue(msg.data.repository!.owner.login, msg.data.repository!.name, msg.data.issue!.number);
this.queue.on<IGitHubWebhookEvent>("issue.edited", async ({ data }) => {
const { repository, issue } = validateRepoIssue(data);
const connections = this.getConnectionsForGithubIssue(repository.owner.login, repository.name, issue.number);
connections.map(async (c) => {
try {
if (c.onIssueEdited)
await c.onIssueEdited(msg.data);
if (c instanceof GitHubIssueConnection)
await c.onIssueEdited(data);
} catch (ex) {
log.warn(`Connection ${c.toString()} failed to handle comment.created:`, ex);
}
})
});
this.queue.on<IGitHubWebhookEvent>("issue.closed", async (msg) => {
const connections = this.getConnectionsForGithubIssue(msg.data.repository!.owner.login, msg.data.repository!.name, msg.data.issue!.number);
this.queue.on<IGitHubWebhookEvent>("issue.closed", async ({ data }) => {
const { repository, issue } = validateRepoIssue(data);
const connections = this.getConnectionsForGithubIssue(repository.owner.login, repository.name, issue.number);
connections.map(async (c) => {
try {
if (c.onIssueStateChange)
await c.onIssueStateChange(msg.data);
if (c instanceof GitHubIssueConnection)
await c.onIssueStateChange();
} catch (ex) {
log.warn(`Connection ${c.toString()} failed to handle comment.created:`, ex);
}
})
});
this.queue.on<IGitHubWebhookEvent>("issue.reopened", async (msg) => {
const connections = this.getConnectionsForGithubIssue(msg.data.repository!.owner.login, msg.data.repository!.name, msg.data.issue!.number);
this.queue.on<IGitHubWebhookEvent>("issue.reopened", async ({ data }) => {
const { repository, issue } = validateRepoIssue(data);
const connections = this.getConnectionsForGithubIssue(repository.owner.login, repository.name, issue.number);
connections.map(async (c) => {
try {
if (c.onIssueStateChange)
await c.onIssueStateChange(msg.data);
if (c instanceof GitHubIssueConnection)
await c.onIssueStateChange();
} catch (ex) {
log.warn(`Connection ${c.toString()} failed to handle comment.created:`, ex);
}
})
});
// this.queue.on<IGitLabWebhookMREvent>("merge_request.open", async (msg) => {
// const connections = this.getConnectionsForGitLabIssue(msg.data.project.namespace, msg.data.repository!.name, msg.data.issue!.number);
// connections.map(async (c) => {
// try {
// if (c.onIssueCreated)
// await c.onIssueStateChange(msg.data);
// } catch (ex) {
// log.warn(`Connection ${c.toString()} failed to handle comment.created:`, ex);
// }
// })
// });
this.queue.on<IGitLabWebhookMREvent>("merge_request.open", async (msg) => {
console.log(msg);
// const connections = this.(msg.data.project.namespace, msg.data.repository!.name, msg.data.issue!.number);
// connections.map(async (c) => {
// try {
// if (c.onIssueCreated)
// await c.onIssueStateChange(msg.data);
// } catch (ex) {
// log.warn(`Connection ${c.toString()} failed to handle comment.created:`, ex);
// }
// })
});
this.queue.on<UserNotificationsEvent>("notifications.user.events", async (msg) => {
const adminRoom = this.adminRooms.get(msg.data.roomId);
@ -262,15 +321,48 @@ export class GithubBridge {
adminRoom.clearOauthState();
await this.tokenStore.storeUserToken("github", adminRoom.userId, msg.data.access_token);
});
this.queue.on<IGitLabWebhookNoteEvent>("gitlab.note.created", async ({data}) => {
const connections = this.getConnectionsForGitLabIssueWebhook(data.repository.homepage, data.issue.iid);
connections.map(async (c) => {
try {
if (c.onCommentCreated)
await c.onCommentCreated(data);
} catch (ex) {
log.warn(`Connection ${c.toString()} failed to handle comment.created:`, ex);
}
})
});
this.queue.on<IGitLabWebhookIssueStateEvent>("gitlab.issue.reopen", async ({data}) => {
const connections = this.getConnectionsForGitLabIssueWebhook(data.repository.homepage, data.object_attributes.iid);
connections.map(async (c) => {
try {
await c.onIssueReopened();
} catch (ex) {
log.warn(`Connection ${c.toString()} failed to handle comment.created:`, ex);
}
})
});
this.queue.on<IGitLabWebhookIssueStateEvent>("gitlab.issue.close", async ({data}) => {
const connections = this.getConnectionsForGitLabIssueWebhook(data.repository.homepage, data.object_attributes.iid);
connections.map(async (c) => {
try {
await c.onIssueClosed();
} catch (ex) {
log.warn(`Connection ${c.toString()} failed to handle comment.created:`, ex);
}
})
});
// Fetch all room state
let joinedRooms: string[];
while(true) {
let joinedRooms: string[]|undefined;
while(joinedRooms === undefined) {
try {
log.info("Connecting to homeserver and fetching joined rooms..");
joinedRooms = await this.as.botIntent.underlyingClient.getJoinedRooms();
log.info(`Found ${joinedRooms.length} rooms`);
break;
} catch (ex) {
// This is our first interaction with the homeserver, so wait if it's not ready yet.
log.warn("Failed to connect to homeserver:", ex, "retrying in 5s");
@ -282,7 +374,12 @@ export class GithubBridge {
if (this.config.bot) {
// Ensure we are registered before we set a profile
await this.as.botIntent.ensureRegistered();
const profile = await this.as.botClient.getUserProfile(this.as.botUserId);
let profile;
try {
profile = await this.as.botClient.getUserProfile(this.as.botUserId);
} catch {
profile = {}
}
if (this.config.bot.avatar && profile.avatar_url !== this.config.bot.avatar) {
log.info(`Setting avatar to ${this.config.bot.avatar}`);
await this.as.botClient.setAvatarUrl(this.config.bot.avatar);
@ -294,7 +391,7 @@ export class GithubBridge {
}
for (const roomId of joinedRooms) {
log.info("Fetching state for " + roomId);
log.debug("Fetching state for " + roomId);
const connections = await this.createConnectionsForRoomId(roomId);
this.connections.push(...connections);
if (connections.length === 0) {
@ -305,9 +402,9 @@ export class GithubBridge {
);
const adminRoom = this.setupAdminRoom(roomId, accountData);
// Call this on startup to set the state
await this.onAdminRoomSettingsChanged(adminRoom, accountData);
await this.onAdminRoomSettingsChanged(adminRoom, accountData, { admin_user: accountData.admin_user });
} catch (ex) {
log.warn(`Room ${roomId} has no connections and is not an admin room`);
log.debug(`Room ${roomId} has no connections and is not an admin room`);
}
} else {
log.info(`Room ${roomId} is connected to: ${connections.join(',')}`);
@ -330,7 +427,7 @@ export class GithubBridge {
}
await retry(() => this.as.botIntent.joinRoom(roomId), 5);
if (event.content.is_direct) {
const room = this.setupAdminRoom(roomId, {admin_user: event.sender, notifications: { enabled: false, participating: false}});
const room = this.setupAdminRoom(roomId, {admin_user: event.sender});
await this.as.botIntent.underlyingClient.setRoomAccountData(
BRIDGE_ROOM_TYPE, roomId, room.data,
);
@ -382,14 +479,17 @@ export class GithubBridge {
}
const command = event.content.body;
if (command) {
await this.adminRooms.get(roomId)!.handleCommand(event.event_id, command);
const adminRoom = this.adminRooms.get(roomId);
if (command && adminRoom) {
await adminRoom.handleCommand(event.event_id, command);
}
}
for (const connection of this.connections.filter((c) => c.roomId === roomId && c.onMessageEvent)) {
for (const connection of this.connections.filter((c) => c.roomId === roomId)) {
try {
await connection.onMessageEvent!(event);
if (connection.onMessageEvent) {
await connection.onMessageEvent(event);
}
} catch (ex) {
log.warn(`Connection ${connection.toString()} failed to handle message:`, ex);
}
@ -412,10 +512,14 @@ export class GithubBridge {
}
private async onRoomEvent(roomId: string, event: MatrixEvent<unknown>) {
if (event.sender === this.as.botUserId) {
// It's us
return;
}
if (event.state_key) {
// A state update, hurrah!
const existingConnection = this.connections.find((c) => c.roomId === roomId && c.isInterestedInStateEvent(event.type, event.state_key || ""));
if (existingConnection) {
if (existingConnection?.onStateUpdate) {
existingConnection.onStateUpdate(event);
} else {
// Is anyone interested in this state?
@ -429,7 +533,7 @@ export class GithubBridge {
}
// Alas, it's just an event.
return this.connections.filter((c) => c.roomId === roomId).map((c) => c.onEvent(event))
return this.connections.filter((c) => c.roomId === roomId).map((c) => c.onEvent ? c.onEvent(event) : undefined);
}
private async onQueryRoom(roomAlias: string) {
@ -438,13 +542,16 @@ export class GithubBridge {
let res: RegExpExecArray | null;
res = GitHubIssueConnection.QueryRoomRegex.exec(roomAlias);
if (res) {
if (!this.github) {
throw Error("GitHub is not configured on this bridge");
}
try {
return await GitHubIssueConnection.onQueryRoom(res, {
as: this.as,
tokenStore: this.tokenStore,
messageClient: this.messageClient,
commentProcessor: this.commentProcessor,
octokit: this.octokit,
octokit: this.github.octokit,
});
} catch (ex) {
log.error(`Could not handle alias with GitHubIssueConnection`, ex);
@ -454,13 +561,16 @@ export class GithubBridge {
res = GitHubRepoConnection.QueryRoomRegex.exec(roomAlias);
if (res) {
if (!this.github) {
throw Error("GitHub is not configured on this bridge");
}
try {
return await GitHubRepoConnection.onQueryRoom(res, {
as: this.as,
tokenStore: this.tokenStore,
messageClient: this.messageClient,
commentProcessor: this.commentProcessor,
octokit: this.octokit,
octokit: this.github.octokit,
});
} catch (ex) {
log.error(`Could not handle alias with GitHubRepoConnection`, ex);
@ -472,9 +582,10 @@ export class GithubBridge {
throw Error('No regex matching query pattern');
}
private async onAdminRoomSettingsChanged(adminRoom: AdminRoom, settings: AdminAccountData) {
log.info(`Settings changed for ${adminRoom.userId} ${settings}`);
if (adminRoom.notificationsEnabled) {
private async onAdminRoomSettingsChanged(adminRoom: AdminRoom, settings: AdminAccountData, oldSettings: AdminAccountData) {
log.debug(`Settings changed for ${adminRoom.userId}`, settings);
// Make this more efficent.
if (!oldSettings.github?.notifications?.enabled && settings.github?.notifications?.enabled) {
log.info(`Notifications enabled for ${adminRoom.userId}`);
const token = await this.tokenStore.getUserToken("github", adminRoom.userId);
if (token) {
@ -483,25 +594,62 @@ export class GithubBridge {
eventName: "notifications.user.enable",
sender: "GithubBridge",
data: {
user_id: adminRoom.userId,
room_id: adminRoom.roomId,
userId: adminRoom.userId,
roomId: adminRoom.roomId,
token,
since: await adminRoom.getNotifSince(),
filter_participating: adminRoom.notificationsParticipating,
since: await adminRoom.getNotifSince("github"),
filterParticipating: adminRoom.notificationsParticipating("github"),
type: "github",
instanceUrl: undefined,
},
});
} else {
log.warn(`Notifications enabled for ${adminRoom.userId} but no token stored!`);
}
} else {
} else if (oldSettings.github?.notifications?.enabled && !settings.github?.notifications?.enabled) {
await this.queue.push<NotificationsDisableEvent>({
eventName: "notifications.user.disable",
sender: "GithubBridge",
data: {
user_id: adminRoom.userId,
userId: adminRoom.userId,
type: "github",
instanceUrl: undefined,
},
});
}
for (const [instanceName, instanceSettings] of Object.entries(settings.gitlab || {})) {
const instanceUrl = this.config.gitlab?.instances[instanceName].url;
const token = await this.tokenStore.getUserToken("gitlab", adminRoom.userId, instanceUrl);
if (token && instanceSettings.notifications.enabled) {
log.info(`GitLab ${instanceName} notifications enabled for ${adminRoom.userId}`);
await this.queue.push<NotificationsEnableEvent>({
eventName: "notifications.user.enable",
sender: "GithubBridge",
data: {
userId: adminRoom.userId,
roomId: adminRoom.roomId,
token,
since: await adminRoom.getNotifSince("gitlab", instanceName),
filterParticipating: adminRoom.notificationsParticipating("gitlab"),
type: "gitlab",
instanceUrl,
},
});
} else if (!instanceSettings.notifications.enabled) {
log.info(`GitLab ${instanceName} notifications disabled for ${adminRoom.userId}`);
await this.queue.push<NotificationsDisableEvent>({
eventName: "notifications.user.disable",
sender: "GithubBridge",
data: {
userId: adminRoom.userId,
type: "gitlab",
instanceUrl,
},
});
}
}
}
private setupAdminRoom(roomId: string, accountData: AdminAccountData) {
@ -509,10 +657,28 @@ export class GithubBridge {
roomId, accountData, this.as.botIntent, this.tokenStore, this.config,
);
adminRoom.on("settings.changed", this.onAdminRoomSettingsChanged.bind(this));
adminRoom.on("open.project", async (project: Octokit.ProjectsGetResponse) => {
adminRoom.on("open.project", async (project: ProjectsGetResponseData) => {
const connection = await GitHubProjectConnection.onOpenProject(project, this.as, adminRoom.userId);
this.connections.push(connection);
});
adminRoom.on("open.gitlab-issue", async (issueInfo: GetIssueOpts, res: GetIssueResponse, instanceName: string, instance: GitLabInstance) => {
const [ connection ] = this.getConnectionsForGitLabIssue(instance, issueInfo.projects, issueInfo.issue);
if (connection) {
return this.as.botClient.inviteUser(adminRoom.userId, connection.roomId);
}
const newConnection = await GitLabIssueConnection.createRoomForIssue(
instanceName,
instance,
res,
issueInfo.projects,
this.as,
this.tokenStore,
this.commentProcessor,
this.messageClient
);
this.connections.push(newConnection);
return this.as.botClient.inviteUser(adminRoom.userId, newConnection.roomId);
});
this.adminRooms.set(roomId, adminRoom);
log.info(`Setup ${roomId} as an admin room for ${adminRoom.userId}`);
return adminRoom;

56
src/GithubWebhooks.ts
View File

@ -1,24 +1,26 @@
import { BridgeConfig } from "./Config";
import { Application, default as express, Request, Response } from "express";
import { createHmac } from "crypto";
import { Octokit } from "@octokit/rest";
import { IssuesGetResponseData, IssuesGetCommentResponseData, ReposGetResponseData } from "@octokit/types";
import { EventEmitter } from "events";
import { MessageQueue, createMessageQueue, MessageQueueMessage } from "./MessageQueue/MessageQueue";
import LogWrapper from "./LogWrapper";
import qs from "querystring";
import { Server } from "http";
import axios from "axios";
import { UserNotificationWatcher } from "./UserNotificationWatcher";
import { UserNotificationWatcher } from "./Notifications/UserNotificationWatcher";
import { IGitLabWebhookEvent } from "./Gitlab/WebhookTypes";
const log = new LogWrapper("GithubWebhooks");
export interface IGitHubWebhookEvent {
action: string;
issue?: Octokit.IssuesGetResponse;
comment?: Octokit.IssuesGetCommentResponse;
repository?: Octokit.ReposGetResponse;
sender?: Octokit.IssuesGetResponseUser;
issue?: IssuesGetResponseData;
comment?: IssuesGetCommentResponseData;
repository?: ReposGetResponseData;
sender?: {
login: string;
}
changes?: {
title?: {
from: string;
@ -38,15 +40,19 @@ export interface IOAuthTokens {
}
export interface NotificationsEnableEvent {
user_id: string;
room_id: string;
userId: string;
roomId: string;
since: number;
token: string;
filter_participating: boolean;
filterParticipating: boolean;
type: "github"|"gitlab";
instanceUrl?: string;
}
export interface NotificationsDisableEvent {
user_id: string;
userId: string;
type: "github"|"gitlab";
instanceUrl?: string;
}
export class GithubWebhooks extends EventEmitter {
@ -69,7 +75,7 @@ export class GithubWebhooks extends EventEmitter {
this.userNotificationWatcher.addUser(msg.data);
});
this.queue.on("notifications.user.disable", (msg: MessageQueueMessage<NotificationsDisableEvent>) => {
this.userNotificationWatcher.removeUser(msg.data.user_id);
this.userNotificationWatcher.removeUser(msg.data.userId, msg.data.type, msg.data.instanceUrl);
});
// This also listens for notifications for users, which is long polly.
@ -77,15 +83,17 @@ export class GithubWebhooks extends EventEmitter {
public listen() {
this.server = this.expressApp.listen(
this.config.github.webhook.port,
this.config.github.webhook.bindAddress,
this.config.webhook.port,
this.config.webhook.bindAddress,
);
log.info(`Listening on http://${this.config.github.webhook.bindAddress}:${this.config.github.webhook.port}`)
log.info(`Listening on http://${this.config.webhook.bindAddress}:${this.config.webhook.port}`)
this.userNotificationWatcher.start();
}
public stop() {
this.queue.stop();
if (this.queue.stop) {
this.queue.stop();
}
if (this.server) {
this.server.close();
}
@ -109,17 +117,23 @@ export class GithubWebhooks extends EventEmitter {
}
private onGitLabPayload(body: IGitLabWebhookEvent) {
log.info(`onGitLabPayload ${body.event_type}:`, body);
if (body.event_type === "merge_request") {
return `merge_request.${body.object_attributes.action}`;
return `gitlab.merge_request.${body.object_attributes.action}`;
} else if (body.event_type === "issue") {
return `gitlab.issue.${body.object_attributes.action}`;
} else if (body.event_type === "note") {
return `gitlab.note.created`;
} else {
return null;
}
return null;
}
private onPayload(req: Request, res: Response) {
log.debug(`New webhook: ${req.url}`);
try {
let eventName: string|null = null;
let body = req.body;
const body = req.body;
res.sendStatus(200);
if (req.headers['x-hub-signature']) {
eventName = this.onGitHubPayload(body);
@ -145,6 +159,9 @@ export class GithubWebhooks extends EventEmitter {
public async onGetOauth(req: Request, res: Response) {
log.info("Got new oauth request");
try {
if (!this.config.github) {
throw Error("Got GitHub oauth request but github was not configured!");
}
const exists = await this.queue.pushWait<IOAuthRequest, boolean>({
eventName: "oauth.response",
sender: "GithubWebhooks",
@ -179,6 +196,9 @@ export class GithubWebhooks extends EventEmitter {
// Calculate the X-Hub-Signature header value.
private getSignature(buf: Buffer) {
if (!this.config.github) {
throw Error("Got GitHub oauth request but github was not configured!");
}
const hmac = createHmac("sha1", this.config.github.webhook.secret);
hmac.update(buf);
return "sha1=" + hmac.digest("hex");

79
src/Gitlab/Client.ts
View File

@ -1,40 +1,107 @@
import axios from "axios";
import { GitLabInstance } from "../Config";
import { GetIssueResponse, GetUserResponse, CreateIssueOpts, CreateIssueResponse, GetIssueOpts, EditIssueOpts, GetTodosResponse, EventsOpts, CreateIssueNoteOpts, CreateIssueNoteResponse } from "./Types";
import LogWrapper from "../LogWrapper";
import { URLSearchParams } from "url";
const log = new LogWrapper("GitLabClient");
export class GitLabClient {
constructor(private instanceUrl: string, private token: string) {
}
public static splitUrlIntoParts(instances: {[name: string]: GitLabInstance}, url: string): [string, string[]]|null {
for (const [instanceKey, instanceConfig] of Object.entries(instances)) {
if (url.startsWith(instanceConfig.url)) {
return [instanceKey, url.substr(instanceConfig.url.length).split("/").filter(part => part.length > 0)];
}
}
return null;
}
get defaultConfig() {
return {
headers: {
Authorization: `Bearer ${this.token}`,
UserAgent: "matrix-github v0.0.1",
"Authorization": `Bearer ${this.token}`,
"User-Agent": "matrix-github v0.0.1",
},
baseURL: this.instanceUrl
};
}
async version() {
return (await axios.get(`${this.instanceUrl}/api/v4/versions`, this.defaultConfig)).data;
return (await axios.get("api/v4/versions", this.defaultConfig)).data;
}
async user(): Promise<GetUserResponse> {
return (await axios.get(`${this.instanceUrl}/api/v4/user`, this.defaultConfig)).data;
return (await axios.get("api/v4/user", this.defaultConfig)).data;
}
private async createIssue(opts: CreateIssueOpts): Promise<CreateIssueResponse> {
return (await axios.post(`${this.instanceUrl}/api/v4/projects/${opts.id}/issues`, opts, this.defaultConfig)).data;
return (await axios.post(`api/v4/projects/${opts.id}/issues`, opts, this.defaultConfig)).data;
}
private async getIssue(opts: GetIssueOpts): Promise<GetIssueResponse> {
try {
return (await axios.get(`api/v4/projects/${opts.projects.join("%2F")}/issues/${opts.issue}`, this.defaultConfig)).data;
} catch (ex) {
log.warn(`Failed to get issue:`, ex);
throw ex;
}
}
private async editIssue(opts: EditIssueOpts): Promise<CreateIssueResponse> {
return (await axios.put(`${this.instanceUrl}/api/v4/projects/${opts.id}/issues/${opts.issue_iid}`, opts, this.defaultConfig)).data;
return (await axios.put(`api/v4/projects/${opts.id}/issues/${opts.issue_iid}`, opts, this.defaultConfig)).data;
}
private async getProject(projectParts: string[]): Promise<GetIssueResponse> {
try {
return (await axios.get(`api/v4/projects/${projectParts.join("%2F")}`, this.defaultConfig)).data;
} catch (ex) {
log.warn(`Failed to get issue:`, ex);
throw ex;
}
}
public async getEvents(opts: EventsOpts) {
const after = `${opts.after.getFullYear()}-` +
`${(opts.after.getMonth()+1).toString().padStart(2, "0")}`+
`-${opts.after.getDay().toString().padStart(2, "0")}`;
return (await axios.get(
`api/v4/events?after=${after}`,
this.defaultConfig)
).data as GetTodosResponse[];
}
public async createIssueNote(projectParts: string[], issueId: number, opts: CreateIssueNoteOpts): Promise<CreateIssueNoteResponse> {
try {
const qp = new URLSearchParams({
body: opts.body,
confidential: (opts.confidential || false).toString(),
}).toString();
return (await axios.post(`api/v4/projects/${projectParts.join("%2F")}/issues/${issueId}/notes?${qp}`, undefined, this.defaultConfig)).data as CreateIssueNoteResponse;
} catch (ex) {
log.warn(`Failed to create issue note:`, ex);
throw ex;
}
}
get issues() {
return {
create: this.createIssue.bind(this),
edit: this.editIssue.bind(this),
get: this.getIssue.bind(this),
}
}
get projects() {
return {
get: this.getProject.bind(this),
}
}
get notes() {
return {
createForIssue: this.createIssueNote.bind(this),
}
}
}

100
src/Gitlab/Types.ts
View File

@ -1,4 +1,14 @@
interface GetUserResponse {
/* eslint-disable camelcase */
export interface GitLabAuthor {
id: number;
name: string;
username: string;
state: 'active';
avatar_url: string;
web_url: string;
}
export interface GetUserResponse {
id: number;
username: string;
email: string;
@ -33,8 +43,14 @@ interface GetUserResponse {
private_profile: boolean;
}
// hhttps://docs.gitlab.com/ee/api/issues.html#single-project-issue
export interface GetIssueOpts {
projects: string[];
issue: number;
}
// https://docs.gitlab.com/ee/api/issues.html#new-issue
interface CreateIssueOpts {
export interface CreateIssueOpts {
id: string|number;
title: string;
description?: string;
@ -42,7 +58,7 @@ interface CreateIssueOpts {
labels?: string[];
}
interface CreateIssueResponse {
export interface CreateIssueResponse {
state: string;
id: string;
iid: string;
@ -50,7 +66,7 @@ interface CreateIssueResponse {
}
// https://docs.gitlab.com/ee/api/issues.html#new-issue
interface EditIssueOpts {
export interface EditIssueOpts {
id: string|number;
issue_iid: string|number;
title?: string;
@ -60,8 +76,80 @@ interface EditIssueOpts {
state_event?: string;
}
interface CreateIssueResponse {
export interface CreateIssueResponse {
state: string;
id: string;
web_url: string;
}
}
export interface GetIssueResponse {
id: number;
iid: number;
title: string;
description: string;
state: 'opened'|'closed';
author: GitLabAuthor;
references: {
short: string;
relative: string;
full: string;
}
web_url: string;
}
export interface GetTodosResponse {
id: number;
author: GitLabAuthor;
action_name: string;
project: {
id: number;
name: string;
name_with_namespace: string;
path: string;
path_with_namespace: string;
};
target: {
title: string;
description: string;
state: 'opened'|'closed';
assignee: {
name: string;
username: string;
id: 1;
state: "active";
avatar_url: string;
web_url: string;
}
}
target_url: string;
body: string;
created_at: string;
updated_at: string;
}
export interface EventsOpts {
after: Date;
}
export interface CreateIssueNoteOpts {
body: string;
confidential?: boolean;
}
export interface CreateIssueNoteResponse {
id: number;
type: string|null;
body: string;
attachment: null;
author: GitLabAuthor;
created_at: string;
updated_at: string;
system: boolean;
noteable_id: number;
noteable_type: 'Issue';
resolvable: boolean;
confidential: boolean;
noteable_iid: string;
commands_changes: unknown;
}

56
src/Gitlab/WebhookTypes.ts
View File

@ -1,20 +1,66 @@
/* eslint-disable camelcase */
export interface IGitLabWebhookEvent {
object_kind: string;
event_type: string;
object_attributes: {
action: string;
state: string;
}
}
export interface IGitlabUser {
name: string;
username: string;
avatar_url: string;
email: string;
}
export interface IGitlabProject {
path_with_namespace: string;
web_url: string;
}
export interface IGitlabIssue {
iid: number;
description: string;
}
export interface IGitLabWebhookMREvent {
object_kind: "merge_request";
user: {
user: IGitlabUser;
project: IGitlabProject;
}
export interface IGitLabWebhookNoteEvent {
user: IGitlabUser;
project: IGitlabProject;
issue: IGitlabIssue;
repository: {
name: string;
username: string;
avatar_url: string;
url: string;
description: string;
homepage: string;
};
project: {
namespace: string;
object_attributes: {
id: number;
noteable_id: number;
description: string;
}
}
export interface IGitLabWebhookIssueStateEvent {
user: IGitlabUser;
project: IGitlabProject;
repository: {
name: string;
url: string;
description: string;
homepage: string;
};
object_attributes: {
id: number;
iid: number;
description: string;
}
}

15
src/IntentUtils.ts
View File

@ -1,10 +1,10 @@
import LogWrapper from "./LogWrapper";
import { Octokit } from "@octokit/rest";
import { Appservice } from "matrix-bot-sdk";
import axios from "axios";
const log = new LogWrapper("IntentUtils");
export async function getIntentForUser(user: Octokit.IssuesGetResponseUser, as: Appservice, octokit: Octokit) {
export async function getIntentForUser(user: {avatarUrl?: string, login: string}, as: Appservice) {
const intent = as.getIntentForSuffix(user.login);
const displayName = `${user.login}`;
// Verify up-to-date profile
@ -22,19 +22,20 @@ export async function getIntentForUser(user: Octokit.IssuesGetResponseUser, as:
await intent.underlyingClient.setDisplayName(displayName);
}
if (!profile.avatar_url && user.avatar_url) {
if (!profile.avatar_url && user.avatarUrl) {
log.debug(`Updating ${intent.userId}'s avatar`);
const buffer = await octokit.request(user.avatar_url);
log.info(`uploading ${user.avatar_url}`);
const buffer = await axios.get(user.avatarUrl, {
responseType: "arraybuffer",
});
log.info(`Uploading ${user.avatarUrl}`);
// This does exist, but headers is silly and doesn't have content-type.
// tslint:disable-next-line: no-any
const contentType = (buffer.headers as any)["content-type"];
const contentType = buffer.headers["content-type"];
const mxc = await intent.underlyingClient.uploadContent(
Buffer.from(buffer.data as ArrayBuffer),
contentType,
);
await intent.underlyingClient.setAvatarUrl(mxc);
}
return intent;

5
src/LogWrapper.ts
View File

@ -38,6 +38,11 @@ export default class LogWrapper {
};
LogService.setLogger({
info: (module: string, ...messageOrObject: any[]) => {
// These are noisy, redirect to debug.
if (module.startsWith("MatrixLiteClient")) {
log.debug(getMessageString(messageOrObject), { module });
return;
}
log.info(getMessageString(messageOrObject), { module });
},
warn: (module: string, ...messageOrObject: any[]) => {

12
src/MatrixSender.ts
View File

@ -3,7 +3,7 @@ import { MessageQueue, createMessageQueue } from "./MessageQueue/MessageQueue";
import { MatrixEventContent, MatrixMessageContent } from "./MatrixEvent";
import { Appservice, IAppserviceRegistration } from "matrix-bot-sdk";
import LogWrapper from "./LogWrapper";
import { v4 as uuid } from "uuid";
export interface IMatrixSendMessage {
sender: string|null;
type: string;
@ -35,7 +35,7 @@ export class MatrixSender {
this.mq.subscribe("matrix.message");
this.mq.on<IMatrixSendMessage>("matrix.message", async (msg) => {
try {
await this.sendMatrixMessage(msg.messageId!, msg.data);
await this.sendMatrixMessage(msg.messageId || uuid(), msg.data);
} catch (ex) {
log.error(`Failed to send message (${msg.data.roomId}, ${msg.data.sender}, ${msg.data.type})`);
}
@ -43,7 +43,9 @@ export class MatrixSender {
}
public stop() {
this.mq.stop();
if (this.mq.stop) {
this.mq.stop();
}
}
public async sendMatrixMessage(messageId: string, msg: IMatrixSendMessage) {
@ -65,7 +67,7 @@ export class MatrixSender {
export class MessageSenderClient {
constructor(private queue: MessageQueue) { }
public async sendMatrixText(roomId: string, text: string, msgtype: string = "m.text",
public async sendMatrixText(roomId: string, text: string, msgtype = "m.text",
sender: string|null = null): Promise<string> {
return this.sendMatrixMessage(roomId, {
msgtype,
@ -74,7 +76,7 @@ export class MessageSenderClient {
}
public async sendMatrixMessage(roomId: string,
content: MatrixEventContent, eventType: string = "m.room.message",
content: MatrixEventContent, eventType = "m.room.message",
sender: string|null = null): Promise<string> {
return (await this.queue.pushWait<IMatrixSendMessage, IMatrixSendMessageResponse>({
eventName: "matrix.message",

7
src/MessageQueue/LocalMQ.ts
View File

@ -1,7 +1,7 @@
import { EventEmitter } from "events";
import { MessageQueue, MessageQueueMessage, DEFAULT_RES_TIMEOUT } from "./MessageQueue";
import micromatch from "micromatch";
import uuid from "uuid/v4";
import {v4 as uuid} from "uuid";
export class LocalMQ extends EventEmitter implements MessageQueue {
private subs: Set<string>;
@ -30,7 +30,6 @@ export class LocalMQ extends EventEmitter implements MessageQueue {
public async pushWait<T, X>(message: MessageQueueMessage<T>,
timeout: number = DEFAULT_RES_TIMEOUT): Promise<X> {
let awaitResponse: (response: MessageQueueMessage<X>) => void;
let resolve: (value: X) => void;
let timer: NodeJS.Timer;
@ -41,7 +40,7 @@ export class LocalMQ extends EventEmitter implements MessageQueue {
}, timeout);
});
awaitResponse = (response: MessageQueueMessage<X>) => {
const awaitResponse = (response: MessageQueueMessage<X>) => {
if (response.messageId === message.messageId) {
clearTimeout(timer);
this.removeListener(`response.${message.eventName}`, awaitResponse);
@ -53,6 +52,4 @@ export class LocalMQ extends EventEmitter implements MessageQueue {
this.push(message);
return p;
}
public stop() { }
}

10
src/MessageQueue/MessageQueue.ts
View File

@ -7,22 +7,26 @@ export const DEFAULT_RES_TIMEOUT = 30000;
const staticLocalMq = new LocalMQ();
let staticRedisMq: RedisMQ|null = null;
export interface MessageQueueMessage<T> {
sender: string;
eventName: string;
data: T;
ts?: number;
messageId?: string;
for?: string;
}
export interface MessageQueueMessageOut<T> extends MessageQueueMessage<T> {
ts: number;
}
export interface MessageQueue {
subscribe: (eventGlob: string) => void;
unsubscribe: (eventGlob: string) => void;
push: <T>(data: MessageQueueMessage<T>, single?: boolean) => Promise<void>;
pushWait: <T, X>(data: MessageQueueMessage<T>, timeout?: number, single?: boolean) => Promise<X>;
on: <T>(eventName: string, cb: (data: MessageQueueMessage<T>) => void) => void;
stop(): void;
on: <T>(eventName: string, cb: (data: MessageQueueMessageOut<T>) => void) => void;
stop?(): void;
}
export function createMessageQueue(config: BridgeConfig): MessageQueue {

25
src/MessageQueue/RedisQueue.ts
View File

@ -1,10 +1,10 @@
import { MessageQueue, MessageQueueMessage, DEFAULT_RES_TIMEOUT } from "./MessageQueue";
import { MessageQueue, MessageQueueMessage, DEFAULT_RES_TIMEOUT, MessageQueueMessageOut } from "./MessageQueue";
import { Redis, default as redis } from "ioredis";
import { BridgeConfig } from "../Config";
import { EventEmitter } from "events";
import LogWrapper from "../LogWrapper";
import uuid from "uuid/v4";
import {v4 as uuid} from "uuid";
const log = new LogWrapper("RedisMq");
@ -26,13 +26,13 @@ export class RedisMQ extends EventEmitter implements MessageQueue {
this.redisPub = new redis(config.queue.port, config.queue.host);
this.redis = new redis(config.queue.port, config.queue.host);
this.myUuid = uuid();
this.redisSub.on("pmessage", (pattern: string, channel: string, message: string) => {
const msg = JSON.parse(message) as MessageQueueMessage<unknown>;
this.redisSub.on("pmessage", (_: string, channel: string, message: string) => {
const msg = JSON.parse(message) as MessageQueueMessageOut<unknown>;
if (msg.for && msg.for !== this.myUuid) {
log.debug(`Got message for ${msg.for}, dropping`);
return;
}
const delay = (process.hrtime()[1]) - msg.ts!;
const delay = (process.hrtime()[1]) - msg.ts;
log.debug("Delay: ", delay / 1000000, "ms");
this.emit(channel, JSON.parse(message));
});
@ -49,7 +49,7 @@ export class RedisMQ extends EventEmitter implements MessageQueue {
this.redis.srem(`${CONSUMER_TRACK_PREFIX}${eventGlob}`, this.myUuid);
}
public async push<T>(message: MessageQueueMessage<T>, single: boolean = false) {
public async push<T>(message: MessageQueueMessage<T>, single = false) {
if (!message.messageId) {
message.messageId = uuid();
}
@ -60,9 +60,12 @@ export class RedisMQ extends EventEmitter implements MessageQueue {
}
message.for = recipient;
}
message.ts = process.hrtime()[1];
const outMsg: MessageQueueMessageOut<T> = {
...message,
ts: process.hrtime()[1],
}
try {
await this.redisPub.publish(message.eventName, JSON.stringify(message));
await this.redisPub.publish(message.eventName, JSON.stringify(outMsg));
log.debug(`Pushed ${message.eventName}`);
} catch (ex) {
log.warn("Failed to push an event:", ex);
@ -71,9 +74,7 @@ export class RedisMQ extends EventEmitter implements MessageQueue {
}
public async pushWait<T, X>(message: MessageQueueMessage<T>,
timeout: number = DEFAULT_RES_TIMEOUT,
single: boolean = false): Promise<X> {
let awaitResponse: (response: MessageQueueMessage<X>) => void;
timeout: number = DEFAULT_RES_TIMEOUT): Promise<X> {
let resolve: (value: X) => void;
let timer: NodeJS.Timer;
@ -84,7 +85,7 @@ export class RedisMQ extends EventEmitter implements MessageQueue {
}, timeout);
});
awaitResponse = (response: MessageQueueMessage<X>) => {
const awaitResponse = (response: MessageQueueMessage<X>) => {
if (response.messageId === message.messageId) {
clearTimeout(timer);
this.removeListener(`response.${message.eventName}`, awaitResponse);

140
src/Notifications/GitHubWatcher.ts Normal file
View File

@ -0,0 +1,140 @@
import { Octokit } from "@octokit/rest";
import { EventEmitter } from "events";
import { GithubInstance } from "../Github/GithubInstance";
import LogWrapper from "../LogWrapper";
import { NotificationWatcherTask } from "./NotificationWatcherTask";
import { RequestError } from "@octokit/request-error";
import { GitHubUserNotification } from "../Github/Types";
import { OctokitResponse } from "@octokit/types";
const log = new LogWrapper("GitHubWatcher");
const GH_API_THRESHOLD = 50;
const GH_API_RETRY_IN = 1000 * 60;
export class GitHubWatcher extends EventEmitter implements NotificationWatcherTask {
private static apiFailureCount: number;
private static globalRetryIn: number;
public static checkGitHubStatus() {
this.apiFailureCount = Math.min(this.apiFailureCount + 1, GH_API_THRESHOLD);
if (this.apiFailureCount < GH_API_THRESHOLD) {
log.warn(`API Failure count at ${this.apiFailureCount}`);
return;
}
// The API is actively failing.
if (this.globalRetryIn > 0) {
this.globalRetryIn = Date.now() + GH_API_RETRY_IN;
}
log.warn(`API Failure limit reached, holding off new requests for ${GH_API_RETRY_IN / 1000}s`);
}
private octoKit: Octokit;
public failureCount = 0;
private interval?: NodeJS.Timeout;
private lastReadTs = 0;
public readonly type = "github";
public readonly instanceUrl = undefined;
constructor(token: string, public userId: string, public roomId: string, public since: number, private participating = false) {
super();
this.octoKit = GithubInstance.createUserOctokit(token);
}
public start(intervalMs: number) {
this.interval = setTimeout(() => {
this.getNotifications();
}, intervalMs);
}
public stop() {
if (this.interval) {
clearInterval(this.interval);
}
}
private handleGitHubFailure(ex: RequestError) {
log.error("An error occured getting notifications:", ex);
if (ex.status === 401 || ex.status === 404) {
log.warn(`Got status ${ex.status} when handing user stream: ${ex.message}`);
this.failureCount++;
} else if (ex.status >= 500) {
setImmediate(() => GitHubWatcher.checkGitHubStatus());
}
this.emit("fetch_failure", this);
}
private async getNotifications() {
if (GitHubWatcher.globalRetryIn !== 0 && GitHubWatcher.globalRetryIn > Date.now()) {
log.info(`Not getting notifications for ${this.userId}, API is still down.`);
return;
}
log.info(`Getting notifications for ${this.userId} ${this.lastReadTs}`);
const since = this.lastReadTs !== 0 ? `&since=${new Date(this.lastReadTs).toISOString()}`: "";
let response: OctokitResponse<GitHubUserNotification[]>;
try {
response = await this.octoKit.request(
`/notifications?participating=${this.participating}${since}`,
);
// We were succesful, clear any timeouts.
GitHubWatcher.globalRetryIn = 0;
// To avoid a bouncing issue, gradually reduce the failure count.
GitHubWatcher.apiFailureCount = Math.max(0, GitHubWatcher.apiFailureCount - 2);
} catch (ex) {
await this.handleGitHubFailure(ex);
return;
}
log.info(`Got ${response.data.length} notifications`);
this.lastReadTs = Date.now();
const events: GitHubUserNotification[] = [];
for (const rawEvent of response.data) {
try {
await (async () => {
if (rawEvent.subject.url) {
const res = await this.octoKit.request(rawEvent.subject.url);
rawEvent.subject.url_data = res.data;
}
if (rawEvent.subject.latest_comment_url) {
const res = await this.octoKit.request(rawEvent.subject.latest_comment_url);
rawEvent.subject.latest_comment_url_data = res.data;
}
if (rawEvent.reason === "review_requested") {
if (!rawEvent.subject.url_data?.number) {
log.warn("review_requested was missing subject.url_data.number");
return;
}
rawEvent.subject.requested_reviewers = (await this.octoKit.pulls.listRequestedReviewers({
pull_number: rawEvent.subject.url_data.number,
owner: rawEvent.repository.owner.login,
repo: rawEvent.repository.name,
})).data;
rawEvent.subject.reviews = (await this.octoKit.pulls.listReviews({
pull_number: rawEvent.subject.url_data.number,
owner: rawEvent.repository.owner.login,
repo: rawEvent.repository.name,
})).data;
}
events.push(rawEvent);
})();
} catch (ex) {
log.warn(`Failed to pre-process ${rawEvent.id}: ${ex}`);
// If it fails, we can just push the raw thing.
events.push(rawEvent);
}
}
if (events.length > 0) {
this.emit("notification_events", {
eventName: "notifications.user.events",
data: {
roomId: this.roomId,
events,
lastReadTs: this.lastReadTs,
},
sender: "GithubWebhooks",
});
}
}
}

37
src/Notifications/GitLabWatcher.ts Normal file
View File

@ -0,0 +1,37 @@
import { EventEmitter } from "events";
import { GitLabClient } from "../Gitlab/Client";
import LogWrapper from "../LogWrapper";
import { NotificationWatcherTask } from "./NotificationWatcherTask";
const log = new LogWrapper("GitLabWatcher");
export class GitLabWatcher extends EventEmitter implements NotificationWatcherTask {
private client: GitLabClient;
private interval?: NodeJS.Timeout;
public readonly type = "gitlab";
public failureCount = 0;
constructor(token: string, url: string, public userId: string, public roomId: string, public since: number) {
super();
this.client = new GitLabClient(url, token);
}
public start(intervalMs: number) {
this.interval = setTimeout(() => {
this.getNotifications();
}, intervalMs);
}
public stop() {
if (this.interval) {
clearInterval(this.interval);
}
}
private async getNotifications() {
log.info(`Fetching events from GitLab for ${this.userId}`);
const events = await this.client.getEvents({
after: new Date(this.since)
});
console.log(events);
}
}

13
src/Notifications/NotificationWatcherTask.ts Normal file
View File

@ -0,0 +1,13 @@
import { EventEmitter } from "events";
type NotificationTypes = "github"|"gitlab";
export interface NotificationWatcherTask extends EventEmitter {
userId: string;
type: NotificationTypes;
instanceUrl?: string;
roomId: string;
failureCount: number;
start(intervalMs: number): void;
stop(): void;
}

78
src/Notifications/UserNotificationWatcher.ts Normal file
View File

@ -0,0 +1,78 @@
import { NotificationsEnableEvent } from "../GithubWebhooks";
import LogWrapper from "../LogWrapper";
import { MessageQueue } from "../MessageQueue/MessageQueue";
import { MessageSenderClient } from "../MatrixSender";
import { NotificationWatcherTask } from "./NotificationWatcherTask";
import { GitHubWatcher } from "./GitHubWatcher";
import { GitHubUserNotification } from "../Github/Types";
import { GitLabWatcher } from "./GitLabWatcher";
export interface UserNotificationsEvent {
roomId: string;
lastReadTs: number;
events: GitHubUserNotification[];
}
const MIN_INTERVAL_MS = 15000;
const FAILURE_THRESHOLD = 50;
const log = new LogWrapper("UserNotificationWatcher");
export class UserNotificationWatcher {
/* Key: userId:type:instanceUrl */
private userIntervals = new Map<string, NotificationWatcherTask>();
private matrixMessageSender: MessageSenderClient;
constructor(private queue: MessageQueue) {
this.matrixMessageSender = new MessageSenderClient(queue);
}
private static constructMapKey(userId: string, type: "github"|"gitlab", instanceUrl?: string) {
return `${userId}:${type}:${instanceUrl || ""}`;
}
public start() {
// No-op
}
public removeUser(userId: string, type: "github"|"gitlab", instanceUrl?: string) {
const key = UserNotificationWatcher.constructMapKey(userId, type, instanceUrl);
const task = this.userIntervals.get(key);
if (task) {
task.stop();
this.userIntervals.delete(key);
log.info(`Removed ${key} from the notif queue`);
}
}
private onFetchFailure(task: NotificationWatcherTask) {
if (task.failureCount > FAILURE_THRESHOLD) {
this.removeUser(task.userId, task.type, task.instanceUrl);
this.matrixMessageSender.sendMatrixText(
task.roomId,
`The bridge has been unable to process your notification stream for some time, and has disabled notifications.
Check your token is still valid, and then turn notifications back on.`, "m.notice",
);
}
}
public addUser(data: NotificationsEnableEvent) {
let task: NotificationWatcherTask;
const key = UserNotificationWatcher.constructMapKey(data.userId, data.type, data.instanceUrl);
if (data.type === "github") {
task = new GitHubWatcher(data.token, data.userId, data.roomId, data.since, data.filterParticipating);
} else if (data.type === "gitlab" && data.instanceUrl) {
task = new GitLabWatcher(data.token, data.instanceUrl, data.userId, data.roomId, data.since);
} else {
throw Error('Notification type not known');
}
this.userIntervals.get(key)?.stop();
task.start(MIN_INTERVAL_MS);
task.on("fetch_failure", this.onFetchFailure.bind(this));
task.on("new_events", (payload) => {
this.queue.push<UserNotificationsEvent>(payload);
});
this.userIntervals.set(key, task);
log.info(`Inserted ${key} into the notif queue`);
}
}

41
src/NotificationsProcessor.ts
View File

@ -1,44 +1,49 @@
import { MessageSenderClient } from "./MatrixSender";
import { IStorageProvider } from "./Stores/StorageProvider";
import { UserNotificationsEvent, UserNotification } from "./UserNotificationWatcher";
import { UserNotificationsEvent } from "./Notifications/UserNotificationWatcher";
import LogWrapper from "./LogWrapper";
import { AdminRoom } from "./AdminRoom";
import markdown from "markdown-it";
import { Octokit } from "@octokit/rest";
import { FormatUtil } from "./FormatUtil";
import { IssuesListAssigneesResponseData, PullsGetResponseData, IssuesGetResponseData, PullsListRequestedReviewersResponseData, PullsListReviewsResponseData, IssuesGetCommentResponseData } from "@octokit/types";
import { GitHubUserNotification } from "./Github/Types";
const log = new LogWrapper("GithubBridge");
const md = new markdown();
export interface IssueDiff {
state: null|string;
assignee: null|Octokit.IssuesGetResponseAssignee;
assignee: null|IssuesListAssigneesResponseData;
title: null|string;
merged: boolean;
mergedBy: null|{
login: string;
// eslint-disable-next-line camelcase
html_url: string;
};
user: {
login: string;
// eslint-disable-next-line camelcase
html_url: string;
};
}
export interface CachedReviewData {
requested_reviewers: Octokit.PullsListReviewRequestsResponse;
reviews: Octokit.PullsListReviewsResponse;
// eslint-disable-next-line camelcase
requested_reviewers: PullsListRequestedReviewersResponseData;
reviews: PullsListReviewsResponseData;
}
type PROrIssue = Octokit.IssuesGetResponse|Octokit.PullsGetResponse;
type PROrIssue = IssuesGetResponseData|PullsGetResponseData;
export class NotificationProcessor {
// eslint-disable-next-line camelcase
private static formatUser(user: {login: string, html_url: string}) {
return `**[${user.login}](${user.html_url})**`;
}
private static formatNotification(notif: UserNotification, diff: IssueDiff|null, newComment: boolean) {
private static formatNotification(notif: GitHubUserNotification, diff: IssueDiff|null, newComment: boolean) {
const user = diff ? ` by ${this.formatUser(diff?.user)}` : "";
let plain =
`${this.getEmojiForNotifType(notif)} [${notif.subject.title}](${notif.subject.url_data?.html_url})${user}`;
@ -61,11 +66,11 @@ export class NotificationProcessor {
plain += `\n\n Title changed to: ${diff.title}`;
}
if (diff.assignee) {
plain += `\n\n Assigned to: ${diff.assignee.login}`;
plain += `\n\n Assigned to: ${diff.assignee[0].login}`;
}
}
if (newComment) {
const comment = notif.subject.latest_comment_url_data as Octokit.IssuesGetCommentResponse;
const comment = notif.subject.latest_comment_url_data as IssuesGetCommentResponseData;
plain += `\n\n ${NotificationProcessor.formatUser(comment.user)}:\n\n > ${comment.body}`;
}
return {
@ -74,7 +79,7 @@ export class NotificationProcessor {
};
}
private static getEmojiForNotifType(notif: UserNotification): string {
private static getEmojiForNotifType(notif: GitHubUserNotification): string {
let reasonFlag = "";
switch (notif.reason) {
case "review_requested":
@ -135,7 +140,7 @@ export class NotificationProcessor {
}
}
try {
await adminRoom.setNotifSince(msg.lastReadTs);
await adminRoom.setNotifSince("github", msg.lastReadTs);
} catch (ex) {
log.error("Failed to update stream position for notifications:", ex);
}
@ -167,7 +172,7 @@ export class NotificationProcessor {
// }
// }
private formatSecurityAlert(notif: UserNotification) {
private formatSecurityAlert(notif: GitHubUserNotification) {
const body = `⚠️ ${notif.subject.title} - `
+ `for **[${notif.repository.full_name}](${notif.repository.html_url})**`;
return {
@ -182,26 +187,26 @@ export class NotificationProcessor {
private diffIssueChanges(curr: PROrIssue, prev: PROrIssue): IssueDiff {
let merged = false;
let mergedBy = null;
if ((curr as Octokit.PullsGetResponse).merged !== (prev as Octokit.PullsGetResponse).merged) {
if ((curr as PullsGetResponseData).merged !== (prev as PullsGetResponseData).merged) {
merged = true;
mergedBy = (curr as Octokit.PullsGetResponse).merged_by;
mergedBy = (curr as PullsGetResponseData).merged_by;
}
const diff: IssueDiff = {
state: curr.state === prev.state ? null : curr.state,
merged,
mergedBy,
assignee: curr.assignee?.id === prev.assignee?.id ? null : curr.assignee,
assignee: curr.assignee?.id === prev.assignee?.id ? null : [curr.assignee],
title: curr.title === prev.title ? null : curr.title,
user: curr.user,
};
return diff;
}
private async formatIssueOrPullRequest(roomId: string, notif: UserNotification) {
private async formatIssueOrPullRequest(roomId: string, notif: GitHubUserNotification) {
const issueNumber = notif.subject.url_data?.number.toString();
let diff = null;
if (issueNumber) {
const prevIssue: Octokit.IssuesGetResponse|null = await this.storage.getGithubIssue(
const prevIssue: IssuesGetResponseData|null = await this.storage.getGithubIssue(
notif.repository.full_name, issueNumber, roomId);
if (prevIssue && notif.subject.url_data) {
diff = this.diffIssueChanges(notif.subject.url_data, prevIssue);
@ -240,7 +245,7 @@ export class NotificationProcessor {
return this.matrixSender.sendMatrixMessage(roomId, body);
}
private async handleUserNotification(roomId: string, notif: UserNotification) {
private async handleUserNotification(roomId: string, notif: GitHubUserNotification) {
log.info("New notification event:", notif);
if (notif.reason === "security_alert") {
return this.matrixSender.sendMatrixMessage(roomId, this.formatSecurityAlert(notif));

15
src/Stores/MemoryStorageProvider.ts
View File

@ -1,36 +1,37 @@
import { MemoryStorageProvider as MSP } from "matrix-bot-sdk";
import { IStorageProvider } from "./StorageProvider";
import { IssuesGetResponseData } from "@octokit/types";
export class MemoryStorageProvider extends MSP implements IStorageProvider {
private issues: Map<string, any> = new Map();
private issues: Map<string, IssuesGetResponseData> = new Map();
private issuesLastComment: Map<string, string> = new Map();
private reviewData: Map<string, string> = new Map();
constructor() {
super();
}
public async setGithubIssue(repo: string, issueNumber: string, data: any, scope: string = "") {
public async setGithubIssue(repo: string, issueNumber: string, data: IssuesGetResponseData, scope = "") {
this.issues.set(`${scope}${repo}/${issueNumber}`, data);
}
public async getGithubIssue(repo: string, issueNumber: string, scope: string = "") {
public async getGithubIssue(repo: string, issueNumber: string, scope = "") {
return this.issues.get(`${scope}${repo}/${issueNumber}`) || null;
}
public async setLastNotifCommentUrl(repo: string, issueNumber: string, url: string, scope: string = "") {
public async setLastNotifCommentUrl(repo: string, issueNumber: string, url: string, scope = "") {
this.issuesLastComment.set(`${scope}${repo}/${issueNumber}`, url);
}
public async getLastNotifCommentUrl(repo: string, issueNumber: string, scope: string = "") {
public async getLastNotifCommentUrl(repo: string, issueNumber: string, scope = "") {
return this.issuesLastComment.get(`${scope}${repo}/${issueNumber}`) || null;
}
public async setPRReviewData(repo: string, issueNumber: string, data: any, scope: string = "") {
public async setPRReviewData(repo: string, issueNumber: string, data: any, scope = "") {
const key = `${scope}:${repo}/${issueNumber}`;
this.reviewData.set(key, data);
}
public async getPRReviewData(repo: string, issueNumber: string, scope: string = "") {
public async getPRReviewData(repo: string, issueNumber: string, scope = "") {
const key = `${scope}:${repo}/${issueNumber}`;
return this.reviewData.get(key) || null;
}

15
src/Stores/RedisStorageProvider.ts
View File

@ -1,3 +1,4 @@
import { IssuesGetResponseData } from "@octokit/types";
import { Redis, default as redis } from "ioredis";
import LogWrapper from "../LogWrapper";
@ -20,7 +21,7 @@ export class RedisStorageProvider implements IStorageProvider {
constructor(host: string, port: number) {
this.redis = new redis(port, host);
this.redis.expire(COMPLETED_TRANSACTIONS_KEY, COMPLETED_TRANSACTIONS_EXPIRE_AFTER).catch((ex) => {
log.warn("Failed to set expiry time on as.completed_transactions");
log.warn("Failed to set expiry time on as.completed_transactions", ex);
});
}
@ -40,35 +41,35 @@ export class RedisStorageProvider implements IStorageProvider {
return (await this.redis.sismember(COMPLETED_TRANSACTIONS_KEY, transactionId)) === 1;
}
public async setGithubIssue(repo: string, issueNumber: string, data: any, scope: string = "") {
public async setGithubIssue(repo: string, issueNumber: string, data: IssuesGetResponseData, scope = "") {
const key = `${scope}${GH_ISSUES_KEY}:${repo}/${issueNumber}`;
await this.redis.set(key, JSON.stringify(data));
await this.redis.expire(key, ISSUES_EXPIRE_AFTER);
}
public async getGithubIssue(repo: string, issueNumber: string, scope: string = "") {
public async getGithubIssue(repo: string, issueNumber: string, scope = "") {
const res = await this.redis.get(`${scope}:${GH_ISSUES_KEY}:${repo}/${issueNumber}`);
return res ? JSON.parse(res) : null;
}
public async setLastNotifCommentUrl(repo: string, issueNumber: string, url: string, scope: string = "") {
public async setLastNotifCommentUrl(repo: string, issueNumber: string, url: string, scope = "") {
const key = `${scope}${GH_ISSUES_LAST_COMMENT_KEY}:${repo}/${issueNumber}`;
await this.redis.set(key, url);
await this.redis.expire(key, ISSUES_LAST_COMMENT_EXPIRE_AFTER);
}
public async getLastNotifCommentUrl(repo: string, issueNumber: string, scope: string = "") {
public async getLastNotifCommentUrl(repo: string, issueNumber: string, scope = "") {
const res = await this.redis.get(`${scope}:${GH_ISSUES_LAST_COMMENT_KEY}:${repo}/${issueNumber}`);
return res ? res : null;
}
public async setPRReviewData(repo: string, issueNumber: string, url: string, scope: string = "") {
public async setPRReviewData(repo: string, issueNumber: string, url: string, scope = "") {
const key = `${scope}${GH_ISSUES_REVIEW_DATA_KEY}:${repo}/${issueNumber}`;
await this.redis.set(key, url);
await this.redis.expire(key, ISSUES_LAST_COMMENT_EXPIRE_AFTER);
}
public async getPRReviewData(repo: string, issueNumber: string, scope: string = "") {
public async getPRReviewData(repo: string, issueNumber: string, scope = "") {
const res = await this.redis.get(`${scope}:${GH_ISSUES_REVIEW_DATA_KEY}:${repo}/${issueNumber}`);
return res ? res : null;
}

7
src/Stores/StorageProvider.ts
View File

@ -1,10 +1,11 @@
import { IAppserviceStorageProvider } from "matrix-bot-sdk";
import { IssuesGetResponseData } from "@octokit/types";
export interface IStorageProvider extends IAppserviceStorageProvider {
setGithubIssue(repo: string, issueNumber: string, data: any, scope?: string): Promise<void>;
getGithubIssue(repo: string, issueNumber: string, scope?: string): Promise<any|null>;
setGithubIssue(repo: string, issueNumber: string, data: IssuesGetResponseData, scope?: string): Promise<void>;
getGithubIssue(repo: string, issueNumber: string, scope?: string): Promise<IssuesGetResponseData|null>;
setLastNotifCommentUrl(repo: string, issueNumber: string, url: string, scope?: string): Promise<void>;
getLastNotifCommentUrl(repo: string, issueNumber: string, scope?: string): Promise<string|null>;
setPRReviewData(repo: string, issueNumber: string, data: any, scope?: string): Promise<void>;
setPRReviewData(repo: string, issueNumber: string, data: unknown, scope?: string): Promise<void>;
getPRReviewData(repo: string, issueNumber: string, scope?: string): Promise<any|null>;
}

204
src/UserNotificationWatcher.ts
View File

@ -1,204 +0,0 @@
import { NotificationsEnableEvent } from "./GithubWebhooks";
import { Octokit } from "@octokit/rest";
import { createTokenAuth } from "@octokit/auth-token";
import { RequestError } from "@octokit/request-error";
import LogWrapper from "./LogWrapper";
import { MessageQueue } from "./MessageQueue/MessageQueue";
import { MessageSenderClient } from "./MatrixSender";
interface UserStream {
octoKit: Octokit;
userId: string;
roomId: string;
lastReadTs: number;
participating: boolean;
failureCount: number;
}
export interface UserNotificationsEvent {
roomId: string;
lastReadTs: number;
events: UserNotification[];
}
export interface UserNotification {
id: string;
reason: "assign"|"author"|"comment"|"invitation"|"manual"|"mention"|"review_requested"|
"security_alert"|"state_change"|"subscribed"|"team_mention";
unread: boolean;
updated_at: number;
last_read_at: number;
url: string;
subject: {
title: string;
url: string;
latest_comment_url: string|null;
type: "PullRequest"|"Issue"|"RepositoryVulnerabilityAlert";
// Probably.
url_data?: Octokit.IssuesGetResponse;
latest_comment_url_data?: Octokit.IssuesGetCommentResponse;
requested_reviewers?: Octokit.PullsListReviewRequestsResponse;
reviews?: Octokit.PullsListReviewsResponse;
};
repository: Octokit.ActivityGetThreadResponseRepository;
}
const MIN_INTERVAL_MS = 15000;
const FAILURE_THRESHOLD = 50;
const GH_API_THRESHOLD = 50;
const GH_API_RETRY_IN = 1000 * 60;
const log = new LogWrapper("UserNotificationWatcher");
export class UserNotificationWatcher {
private userIntervals: Map<string, NodeJS.Timeout> = new Map();
private matrixMessageSender: MessageSenderClient;
private apiFailureCount: number = 0;
private globalRetryIn: number = 0;
constructor(private queue: MessageQueue) {
this.matrixMessageSender = new MessageSenderClient(queue);
}
public checkGitHubStatus() {
this.apiFailureCount = Math.min(this.apiFailureCount + 1, GH_API_THRESHOLD);
if (this.apiFailureCount < GH_API_THRESHOLD) {
log.warn(`API Failure count at ${this.apiFailureCount}`);
return;
}
// The API is actively failing.
if (this.globalRetryIn > 0) {
this.globalRetryIn = Date.now() + GH_API_RETRY_IN;
}
log.warn(`API Failure limit reached, holding off new requests for ${GH_API_RETRY_IN / 1000}s`);
}
public start() {
// No-op
}
public async fetchUserNotifications(stream: UserStream) {
if (this.globalRetryIn !== 0 && this.globalRetryIn > Date.now()) {
log.info(`Not getting notifications for ${stream.userId}, API is still down.`);
return stream;
}
log.info(`Getting notifications for ${stream.userId} ${stream.lastReadTs}`);
const since = stream.lastReadTs !== 0 ? `&since=${new Date(stream.lastReadTs).toISOString()}`: "";
let response: Octokit.AnyResponse;
try {
response = await stream.octoKit.request(
`/notifications?participating=${stream.participating}${since}`,
);
// We were succesful, clear any timeouts.
this.globalRetryIn = 0;
// To avoid a bouncing issue, gradually reduce the failure count.
this.apiFailureCount = Math.max(0, this.apiFailureCount - 2);
} catch (ex) {
await this.handleGitHubFailure(stream, ex);
return stream;
}
log.info(`Got ${response.data.length} notifications`);
stream.lastReadTs = Date.now();
const events: UserNotification[] = [];
for (const rawEvent of response.data as UserNotification[]) {
try {
await (async () => {
if (rawEvent.subject.url) {
const res = await stream.octoKit.request(rawEvent.subject.url);
rawEvent.subject.url_data = res.data;
}
if (rawEvent.subject.latest_comment_url) {
const res = await stream.octoKit.request(rawEvent.subject.latest_comment_url);
rawEvent.subject.latest_comment_url_data = res.data;
}
if (rawEvent.reason === "review_requested") {
rawEvent.subject.requested_reviewers = (await stream.octoKit.pulls.listReviewRequests({
pull_number: rawEvent.subject.url_data?.number!,
owner: rawEvent.repository.owner.login,
repo: rawEvent.repository.name,
})).data;
rawEvent.subject.reviews = (await stream.octoKit.pulls.listReviews({
pull_number: rawEvent.subject.url_data?.number!,
owner: rawEvent.repository.owner.login,
repo: rawEvent.repository.name,
})).data;
}
events.push(rawEvent);
})();
} catch (ex) {
log.warn(`Failed to pre-process ${rawEvent.id}: ${ex}`);
// If it fails, we can just push the raw thing.
events.push(rawEvent);
}
}
if (events.length > 0) {
await this.queue.push<UserNotificationsEvent>({
eventName: "notifications.user.events",
data: {
roomId: stream.roomId,
events,
lastReadTs: stream.lastReadTs,
},
sender: "GithubWebhooks",
});
}
return stream;
}
public handleGitHubFailure(stream: UserStream, ex: RequestError) {
log.error("An error occured getting notifications:", ex);
if (ex.status === 401 || ex.status === 404) {
log.warn(`Got status ${ex.status} when handing user stream: ${ex.message}`);
stream.failureCount++;
} else if (ex.status >= 500) {
setImmediate(() => this.checkGitHubStatus());
}
if (stream.failureCount > FAILURE_THRESHOLD) {
this.removeUser(stream.userId);
return this.matrixMessageSender.sendMatrixText(
stream.roomId,
`The bridge has been unable to process your notification stream for some time, and has disabled notifications.
Check your GitHub token is still valid, and then turn notifications back on.`, "m.notice",
);
}
return null;
}
public removeUser(userId: string) {
const timer = this.userIntervals.get(userId);
if (timer) {
clearInterval(timer);
log.info(`Removed ${userId} to notif queue`);
}
}
public addUser(data: NotificationsEnableEvent) {
const clientKit = new Octokit({
authStrategy: createTokenAuth,
auth: data.token,
userAgent: "matrix-github v0.0.1",
});
const userId = data.user_id;
this.removeUser(userId);
let stream: UserStream = {
octoKit: clientKit,
userId,
roomId: data.room_id,
lastReadTs: data.since,
participating: data.filter_participating,
failureCount: 0,
};
log.info(`Inserted ${userId} into the notif queue`);
const interval = setInterval(async () => {
stream = await this.fetchUserNotifications(stream);
}, MIN_INTERVAL_MS);
this.userIntervals.set(userId, interval);
return;
}
}

43
src/UserTokenStore.ts
View File

@ -2,14 +2,20 @@ import { Intent } from "matrix-bot-sdk";
import { promises as fs } from "fs";
import { publicEncrypt, privateDecrypt } from "crypto";
import LogWrapper from "./LogWrapper";
import { Octokit } from "@octokit/rest";
import { createTokenAuth } from "@octokit/auth-token";
import { GitLabClient } from "./Gitlab/Client";
import { GithubInstance } from "./Github/GithubInstance";
const ACCOUNT_DATA_TYPE = "uk.half-shot.matrix-github.password-store:";
const ACCOUNT_DATA_GITLAB_TYPE = "uk.half-shot.matrix-github.gitlab.password-store:";
const log = new LogWrapper("UserTokenStore");
function tokenKey(type: "github"|"gitlab", userId: string, instanceUrl?: string) {
if (type === "github") {
return `${ACCOUNT_DATA_TYPE}${userId}`;
}
return `${ACCOUNT_DATA_GITLAB_TYPE}${instanceUrl}${userId}`;
}
export class UserTokenStore {
private key!: Buffer;
private userTokens: Map<string, string>;
@ -23,33 +29,34 @@ export class UserTokenStore {
}
public async storeUserToken(type: "github"|"gitlab", userId: string, token: string, instanceUrl?: string): Promise<void> {
let prefix = type === "github" ? ACCOUNT_DATA_TYPE : ACCOUNT_DATA_GITLAB_TYPE;
if (instanceUrl) {
prefix += instanceUrl;
}
await this.intent.underlyingClient.setAccountData(`${prefix}${userId}`, {
const key = tokenKey(type, userId, instanceUrl);
const data = {
encrypted: publicEncrypt(this.key, Buffer.from(token)).toString("base64"),
instance_url: instanceUrl,
});
this.userTokens.set(userId, token);
instance: instanceUrl,
};
await this.intent.underlyingClient.setAccountData(key, data);
this.userTokens.set(key, token);
log.info(`Stored new ${type} token for ${userId}`);
log.debug(`Stored`, data);
}
public async getUserToken(type: "github"|"gitlab", userId: string, instanceUrl?: string): Promise<string|null> {
if (this.userTokens.has(userId)) {
return this.userTokens.get(userId)!;
const key = tokenKey(type, userId, instanceUrl);
const existingToken = this.userTokens.get(key);
if (existingToken) {
return existingToken;
}
let obj;
try {
if (type === "github") {
obj = await this.intent.underlyingClient.getAccountData(`${ACCOUNT_DATA_TYPE}${userId}`);
obj = await this.intent.underlyingClient.getAccountData(key);
} else if (type === "gitlab") {
obj = await this.intent.underlyingClient.getAccountData(`${ACCOUNT_DATA_GITLAB_TYPE}${instanceUrl}${userId}`);
obj = await this.intent.underlyingClient.getAccountData(key);
}
const encryptedTextB64 = obj.encrypted;
const encryptedText = Buffer.from(encryptedTextB64, "base64");
const token = privateDecrypt(this.key, encryptedText).toString("utf-8");
this.userTokens.set(userId, token);
this.userTokens.set(key, token);
return token;
} catch (ex) {
log.error(`Failed to get token for user ${userId}`);
@ -64,11 +71,7 @@ export class UserTokenStore {
if (!senderToken) {
return null;
}
return new Octokit({
authStrategy: createTokenAuth,
auth: senderToken,
userAgent: "matrix-github v0.0.1",
});
return GithubInstance.createUserOctokit(senderToken);
}
public async getGitLabForUser(userId: string, instanceUrl: string) {

29
tests/AdminRoomTest.ts Normal file
View File

@ -0,0 +1,29 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { expect } from "chai";
import { AdminRoom } from "../src/AdminRoom";
import { UserTokenStore } from "../src/UserTokenStore";
import { IntentMock } from "./utils/IntentMock";
const ROOM_ID = "!foo:bar";
function createAdminRoom(data: any = {admin_user: "@admin:bar"}): [AdminRoom, IntentMock] {
const intent = IntentMock.create();
if (!data.admin_user) {
data.admin_user = "@admin:bar";
}
const tokenStore = new UserTokenStore("notapath", intent);
return [new AdminRoom(ROOM_ID, data, intent, tokenStore, {} as any), intent];
}
describe("AdminRoom", () => {
it("will present help text", async () => {
const [adminRoom, intent] = createAdminRoom();
await adminRoom.handleCommand("$foo:bar", "help");
expect(intent.sentEvents).to.have.lengthOf(1);
expect(intent.sentEvents[0]).to.deep.equal({
roomId: ROOM_ID,
content: AdminRoom.helpMessage,
});
});
})

18
tests/FormatUtilTest.ts
View File

@ -5,13 +5,27 @@ const SIMPLE_ISSUE = {
number: 123,
state: "open",
title: "A simple title",
full_name: "evilcorp/lab",
url: "https://github.com/evilcorp/lab/issues/123",
html_url: "https://github.com/evilcorp/lab/issues/123",
repository_url: "https://api.github.com/repos/evilcorp/lab",
};
const SIMPLE_REPO = {
description: "A simple description",
full_name: "evilcorp/lab",
html_url: "https://github.com/evilcorp/lab/issues/123",
};
describe("FormatUtilTest", () => {
it("correctly formats a room name", () => {
expect(FormatUtil.formatRoomName(SIMPLE_ISSUE)).to.equal(
it("correctly formats a repo room name", () => {
expect(FormatUtil.formatRepoRoomName(SIMPLE_REPO)).to.equal(
"evilcorp/lab: A simple description",
);
});
it("correctly formats a issue room name", () => {
expect(FormatUtil.formatIssueRoomName(SIMPLE_ISSUE)).to.equal(
"evilcorp/lab#123: A simple title",
);
});

25
tests/utils/IntentMock.ts Normal file
View File

@ -0,0 +1,25 @@
export class IntentMock {
public sentEvents: {roomId: string, content: any}[] = [];
static create(){
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return new this() as any;
}
sendText(roomId: string, noticeText: string, msgtype: string) {
this.sentEvents.push({
roomId,
content: {
msgtype,
body: noticeText,
}
});
}
sendEvent(roomId: string, content: any) {
this.sentEvents.push({
roomId,
content,
});
}
}

62
tslint.json
View File

@ -1,62 +0,0 @@
{
"extends": "tslint:recommended",
"rules": {
"ordered-imports": false,
"no-trailing-whitespace": {
"severity": "error"
},
"max-classes-per-file": {
"severity": "warning"
},
"object-literal-sort-keys": false,
"no-any": {
"severity": "warning"
},
"arrow-return-shorthand": true,
"prefer-for-of": true,
"typedef": {
"severity": "warning"
},
"await-promise": [
true,
"RequestPromise"
],
"curly": true,
"no-empty": false,
"no-invalid-this": true,
"no-string-throw": {
"severity": "warning"
},
"no-unused-expression": true,
"prefer-const": true,
"indent": [
true,
"spaces",
4
],
"no-duplicate-imports": true,
"array-type": [
true,
"array"
],
"promise-function-async": true,
"no-bitwise": true,
"no-debugger": true,
"no-floating-promises": true,
"prefer-template": [
true,
"allow-single-concat"
],
"interface-name": false
},
"overrides": [
{
"files": [
"test_*.ts"
],
"rules": {
"no-unused-expressions": "off"
}
}
]
}

1797
yarn.lock
View File

File diff suppressed because it is too large Load Diff