nodejs/hookshot
nodejs/hookshot
1
0
Fork 0
mirror of https://github.com/matrix-org/matrix-hookshot.git synced 2025-03-10 21:19:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Handle github API errors

Browse Source
This commit is contained in:
Half-Shot 2020-04-27 15:52:48 +01:00
parent 0d1127e139
commit cc7ca46b33
1 changed files with 73 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
src/UserNotificationWatcher.ts
View File

@ -1,6 +1,7 @@
import { NotificationsEnableEvent } from "./GithubWebhooks"; import { NotificationsEnableEvent } from "./GithubWebhooks";
import { Octokit } from "@octokit/rest"; import { Octokit } from "@octokit/rest";
import { createTokenAuth } from "@octokit/auth-token"; import { createTokenAuth } from "@octokit/auth-token";
import { RequestError } from "@octokit/request-error";
import { LogWrapper } from "./LogWrapper"; import { LogWrapper } from "./LogWrapper";
import { MessageQueue } from "./MessageQueue/MessageQueue"; import { MessageQueue } from "./MessageQueue/MessageQueue";
import { MessageSenderClient } from "./MatrixSender"; import { MessageSenderClient } from "./MatrixSender";
@ -22,7 +23,7 @@ export interface UserNotificationsEvent {
export interface UserNotification { export interface UserNotification {
id: string; id: string;
reason: "assign"|"author"|"comment"|"invitation"|"manual"|"mention"|"review_required"|"review_requested"| reason: "assign"|"author"|"comment"|"invitation"|"manual"|"mention"|"review_requested"|
"security_alert"|"state_change"|"subscribed"|"team_mention"; "security_alert"|"state_change"|"subscribed"|"team_mention";
unread: boolean; unread: boolean;
updated_at: number; updated_at: number;
@ -36,39 +37,71 @@ export interface UserNotification {
// Probably. // Probably.
url_data?: Octokit.IssuesGetResponse; url_data?: Octokit.IssuesGetResponse;
latest_comment_url_data?: Octokit.IssuesGetCommentResponse; latest_comment_url_data?: Octokit.IssuesGetCommentResponse;
requested_reviewers?: Octokit.PullsListReviewRequestsResponse;
reviews?: Octokit.PullsListReviewsResponse;
}; };
repository: Octokit.ActivityGetThreadResponseRepository; repository: Octokit.ActivityGetThreadResponseRepository;
} }
const MIN_INTERVAL_MS = 15000; const MIN_INTERVAL_MS = 15000;
const FAILURE_THRESHOLD = 50; const FAILURE_THRESHOLD = 50;
const GH_API_THRESHOLD = 50;
const GH_API_RETRY_IN = 1000 * 60;
const log = new LogWrapper("UserNotificationWatcher"); const log = new LogWrapper("UserNotificationWatcher");
export class UserNotificationWatcher { export class UserNotificationWatcher {
private userIntervals: Map<string, NodeJS.Timeout> = new Map(); private userIntervals: Map<string, NodeJS.Timeout> = new Map();
private matrixMessageSender: MessageSenderClient; private matrixMessageSender: MessageSenderClient;
private apiFailureCount: number = 0;
private globalRetryIn: number = 0;
constructor(private queue: MessageQueue) { constructor(private queue: MessageQueue) {
this.matrixMessageSender = new MessageSenderClient(queue); this.matrixMessageSender = new MessageSenderClient(queue);
} }
public checkGitHubStatus() {
this.apiFailureCount = Math.min(this.apiFailureCount + 1, GH_API_THRESHOLD);
if (this.apiFailureCount < GH_API_THRESHOLD) {
log.warn(`API Failure count at ${this.apiFailureCount}`);
return;
}
// The API is actively failing.
if (this.globalRetryIn > 0) {
this.globalRetryIn = Date.now() + GH_API_RETRY_IN;
}
log.warn(`API Failure limit reached, holding off new requests for ${GH_API_RETRY_IN / 1000}s`);
}
public start() { public start() {
// No-op // No-op
} }
public async fetchUserNotifications(stream: UserStream) { public async fetchUserNotifications(stream: UserStream) {
if (this.globalRetryIn !== 0 && this.globalRetryIn > Date.now()) {
log.info(`Not getting notifications for ${stream.userId}, API is still down.`);
return stream;
}
log.info(`Getting notifications for ${stream.userId} ${stream.lastReadTs}`); log.info(`Getting notifications for ${stream.userId} ${stream.lastReadTs}`);
const since = stream.lastReadTs !== 0 ? `&since=${new Date(stream.lastReadTs).toISOString()}`: "";
let response: Octokit.AnyResponse;
try { try {
const since = stream.lastReadTs !== 0 ? `&since=${new Date(stream.lastReadTs).toISOString()}`: ""; response = await stream.octoKit.request(
const response = await stream.octoKit.request(
`/notifications?participating=${stream.participating}${since}`, `/notifications?participating=${stream.participating}${since}`,
); );
log.info(`Got ${response.data.length} notifications`); // We were succesful, clear any timeouts.
stream.lastReadTs = Date.now(); this.globalRetryIn = 0;
const events: UserNotification[] = []; // To avoid a bouncing issue, gradually reduce the failure count.
this.apiFailureCount = Math.max(0, this.apiFailureCount - 2);
} catch (ex) {
await this.handleGitHubFailure(stream, ex);
return stream;
}
log.info(`Got ${response.data.length} notifications`);
stream.lastReadTs = Date.now();
const events: UserNotification[] = [];
for (const rawEvent of response.data as UserNotification[]) { for (const rawEvent of response.data as UserNotification[]) {
try { try {
await (async () => { await (async () => {
if (rawEvent.subject.url) { if (rawEvent.subject.url) {
@ -79,6 +112,18 @@ export class UserNotificationWatcher {
const res = await stream.octoKit.request(rawEvent.subject.latest_comment_url); const res = await stream.octoKit.request(rawEvent.subject.latest_comment_url);
rawEvent.subject.latest_comment_url_data = res.data; rawEvent.subject.latest_comment_url_data = res.data;
} }
if (rawEvent.reason === "review_requested") {
rawEvent.subject.requested_reviewers = (await stream.octoKit.pulls.listReviewRequests({
pull_number: rawEvent.subject.url_data?.number!,
owner: rawEvent.repository.owner.login,
repo: rawEvent.repository.name,
})).data;
rawEvent.subject.reviews = (await stream.octoKit.pulls.listReviews({
pull_number: rawEvent.subject.url_data?.number!,
owner: rawEvent.repository.owner.login,
repo: rawEvent.repository.name,
})).data;
}
events.push(rawEvent); events.push(rawEvent);
})(); })();
} catch (ex) { } catch (ex) {
@ -88,32 +133,38 @@ export class UserNotificationWatcher {
} }
} }
if (events.length > 0) { if (events.length > 0) {
await this.queue.push<UserNotificationsEvent>({ await this.queue.push<UserNotificationsEvent>({
eventName: "notifications.user.events", eventName: "notifications.user.events",
data: { data: {
roomId: stream.roomId, roomId: stream.roomId,
events, events,
lastReadTs: stream.lastReadTs, lastReadTs: stream.lastReadTs,
}, },
sender: "GithubWebhooks", sender: "GithubWebhooks",
}); });
} }
return stream;
}
} catch (ex) { public handleGitHubFailure(stream: UserStream, ex: RequestError) {
log.error("An error occured getting notifications:", ex);
if (ex.status === 401 || ex.status === 404) {
log.warn(`Got status ${ex.status} when handing user stream: ${ex.message}`);
stream.failureCount++; stream.failureCount++;
log.error("An error occured getting notifications:", ex); } else if (ex.status >= 500) {
setImmediate(() => this.checkGitHubStatus());
} }
if (stream.failureCount > FAILURE_THRESHOLD) { if (stream.failureCount > FAILURE_THRESHOLD) {
this.removeUser(stream.userId); this.removeUser(stream.userId);
await this.matrixMessageSender.sendMatrixText( return this.matrixMessageSender.sendMatrixText(
stream.roomId, stream.roomId,
`The bridge has been unable to process your notification stream for some time, and has disabled notifications. `The bridge has been unable to process your notification stream for some time, and has disabled notifications.
Check your GitHub token is still valid, and then turn notifications back on.`, "m.notice", Check your GitHub token is still valid, and then turn notifications back on.`, "m.notice",
); );
} }
return stream; return null;
} }
public removeUser(userId: string) { public removeUser(userId: string) {