Add support for edit tracking in challenge hound & fix a few bugs. (#927)

* Fix a namespace conflict.

* Add support for edits.

* Couple of bugfixes.

* changelog

* Support pkcs1 format keys.

* Add docs for official API.

* couple of cleanups

* Revert "Support pkcs1 format keys."

This reverts commit 157cc4ac1269ecdeb64529c51b79d11463cdbbfd.

changelog.d/927.bugfix

@@ -0,0 +1 @@
+Fix a few bugs introduced in challenge hound support.

docs/setup/challengehound.md

@@ -5,23 +5,14 @@ into Matrix.
 
 ### Getting the API secret.
 
-Unfortunately, there is no way to directly request a persistent Challenge Hound API token. The
+You will need to email ChallengeHound support for an API token. They seem happy to provide one
-only way to authenticate with the service at present is to login with an email address and receive
+as long as you are an admin of a challenge. See [this support article](https://support.challengehound.com/article/69-does-challenge-hound-have-an-api)
-a magic token in an email. This is not something Hookshot has the capability to do on it's own.
-
-In order to extract the token for use with the bridge, login to Challenge Hound. Once logged in,
-please locate the local storage via the devtools of your browser. Inside you will find a `ch:user`
-entry with a `token` value. That value should be used as the secret for your Hookshot config.
 
 ```yaml
 challengeHound:
   token: <the token>
 ```
 
-This token tends to expire roughly once a month, and for the moment you'll need to manually
-replace it. You can also ask Challenge Hound's support for an API key, although this has not
-been tested.
-
 ## Usage
 
 You can add a new challenge hound challenge by command:

src/Connections/HoundConnection.ts

@@ -4,7 +4,8 @@ import { BaseConnection } from "./BaseConnection";
 import { IConnection, IConnectionState } from ".";
 import { Connection, InstantiateConnectionOpts, ProvisionConnectionOpts } from "./IConnection";
 import { CommandError } from "../errors";
-
+import { IBridgeStorageProvider } from "../Stores/StorageProvider";
+import { Logger } from "matrix-appservice-bridge";
 export interface HoundConnectionState extends IConnectionState {
     challengeId: string;
 }
@@ -14,20 +15,44 @@ export interface HoundPayload {
     challengeId: string,
 }
 
+/**
+ * @url https://documenter.getpostman.com/view/22349866/UzXLzJUV#0913e0b9-9cb5-440e-9d8d-bf6430285ee9
+ */
 export interface HoundActivity {
-    id: string;
+    userId: string,
-    distance: number; // in meters
+    activityId: string,
-    duration: number;
+    participant: string,
-    elevation: number;
+    /**
-    createdAt: string;
+     * @example "07/26/2022"
-    activityType: string;
+     */
-    activityName: string;
+    date: string,
-    user: {
+    /**
-        id: string;
+     * @example "2022-07-26T13:49:22Z"
-        fullname: string;
+     */
-        fname: string;
+    datetime: string,
-        lname: string;
+    name: string,
-    }
+    type: string,
+    /**
+     * @example strava
+     */
+    app: string,
+    durationSeconds: number,
+    /**
+     * @example "1.39"
+     */
+    distanceKilometers: string,
+    /**
+     * @example "0.86"
+     */
+    distanceMiles: string,
+    /**
+     * @example "0.86"
+     */
+    elevationMeters: string,
+    /**
+     * @example "0.86"
+     */
+    elevationFeet: string,
 }
 
 export interface IChallenge {
@@ -76,6 +101,7 @@ function getEmojiForType(type: string) {
     }
 }
 
+const log = new Logger("HoundConnection");
 const md = markdownit();
 @Connection
 export class HoundConnection extends BaseConnection implements IConnection {
@@ -95,12 +121,12 @@ export class HoundConnection extends BaseConnection implements IConnection {
 
     public static validateState(data: Record<string, unknown>): HoundConnectionState {
         // Convert URL to ID.
-        if (!data.challengeId && data.url && data.url === "string") {
+        if (!data.challengeId && data.url && typeof data.url === "string") {
             data.challengeId = this.getIdFromURL(data.url);
         }
 
         // Test for v1 uuid.
-        if (!data.challengeId || typeof data.challengeId !== "string" || /^\w{8}(?:-\w{4}){3}-\w{12}$/.test(data.challengeId)) {
+        if (!data.challengeId || typeof data.challengeId !== "string" || !/^\w{8}(?:-\w{4}){3}-\w{12}$/.test(data.challengeId)) {
             throw Error('Missing or invalid id');
         }
 
@@ -109,14 +135,14 @@ export class HoundConnection extends BaseConnection implements IConnection {
         }
     }
 
-    public static createConnectionForState(roomId: string, event: StateEvent<Record<string, unknown>>, {config, intent}: InstantiateConnectionOpts) {
+    public static createConnectionForState(roomId: string, event: StateEvent<Record<string, unknown>>, {config, intent, storage}: InstantiateConnectionOpts) {
         if (!config.challengeHound) {
             throw Error('Challenge hound is not configured');
         }
-        return new HoundConnection(roomId, event.stateKey, this.validateState(event.content), intent);
+        return new HoundConnection(roomId, event.stateKey, this.validateState(event.content), intent, storage);
     }
 
-    static async provisionConnection(roomId: string, _userId: string, data: Record<string, unknown> = {}, {intent, config}: ProvisionConnectionOpts) {
+    static async provisionConnection(roomId: string, _userId: string, data: Record<string, unknown> = {}, {intent, config, storage}: ProvisionConnectionOpts) {
         if (!config.challengeHound) {
             throw Error('Challenge hound is not configured');
         }
@@ -127,7 +153,7 @@ export class HoundConnection extends BaseConnection implements IConnection {
             throw new CommandError(`Fetch failed, status ${statusDataRequest.status}`, "Challenge could not be found. Is it active?");
         }
         const { challengeName } = await statusDataRequest.json() as {challengeName: string};
-        const connection = new HoundConnection(roomId, validState.challengeId, validState, intent);
+        const connection = new HoundConnection(roomId, validState.challengeId, validState, intent, storage);
         await intent.underlyingClient.sendStateEvent(roomId, HoundConnection.CanonicalEventType, validState.challengeId, validState);
         return {
             connection,
@@ -140,7 +166,8 @@ export class HoundConnection extends BaseConnection implements IConnection {
         roomId: string,
         stateKey: string,
         private state: HoundConnectionState,
-        private readonly intent: Intent) {
+        private readonly intent: Intent,
+        private readonly storage: IBridgeStorageProvider) {
         super(roomId, stateKey, HoundConnection.CanonicalEventType)
     }
 
@@ -156,25 +183,41 @@ export class HoundConnection extends BaseConnection implements IConnection {
         return this.state.priority || super.priority;
     }
 
-    public async handleNewActivity(payload: HoundActivity) {
+    public async handleNewActivity(activity: HoundActivity) {
-        const distance = `${(payload.distance / 1000).toFixed(2)}km`;
+        log.info(`New activity recorded ${activity.activityId}`);
-        const emoji = getEmojiForType(payload.activityType);
+        const existingActivityEventId = await this.storage.getHoundActivity(this.challengeId, activity.activityId);
-        const body = `🎉 **${payload.user.fullname}** completed a ${distance} ${emoji} ${payload.activityType} (${payload.activityName})`;
+        const distance = parseFloat(activity.distanceKilometers);
-        const content: any = {
+        const distanceUnits = `${(distance).toFixed(2)}km`;
+        const emoji = getEmojiForType(activity.type);
+        const body = `🎉 **${activity.participant}** completed a ${distanceUnits} ${emoji} ${activity.type} (${activity.name})`;
+        let content: any = {
             body,
             format: "org.matrix.custom.html",
             formatted_body: md.renderInline(body),
         };
         content["msgtype"] = "m.notice";
-        content["uk.half-shot.matrix-challenger.activity.id"] = payload.id;
+        content["uk.half-shot.matrix-challenger.activity.id"] = activity.activityId;
-        content["uk.half-shot.matrix-challenger.activity.distance"] = Math.round(payload.distance);
+        content["uk.half-shot.matrix-challenger.activity.distance"] = Math.round(distance * 1000);
-        content["uk.half-shot.matrix-challenger.activity.elevation"] = Math.round(payload.elevation);
+        content["uk.half-shot.matrix-challenger.activity.elevation"] = Math.round(parseFloat(activity.elevationMeters));
-        content["uk.half-shot.matrix-challenger.activity.duration"] = Math.round(payload.duration);
+        content["uk.half-shot.matrix-challenger.activity.duration"] = Math.round(activity.durationSeconds);
         content["uk.half-shot.matrix-challenger.activity.user"] = {
-            "name": payload.user.fullname,
+            "name": activity.participant,
-            id: payload.user.id,
+            id: activity.userId,
         };
-        await this.intent.underlyingClient.sendMessage(this.roomId, content);
+        if (existingActivityEventId) {
+            log.debug(`Updating existing activity ${activity.activityId} ${existingActivityEventId}`);
+            content = {
+                body: `* ${content.body}`,
+                msgtype: "m.notice",
+                "m.new_content": content,
+                "m.relates_to": {
+                    "event_id": existingActivityEventId,
+                    "rel_type": "m.replace"
+                },
+            };
+        }
+        const eventId = await this.intent.underlyingClient.sendMessage(this.roomId, content);
+        await this.storage.storeHoundActivityEvent(this.challengeId, activity.activityId, eventId);
     }
 
     public toString() {

src/Stores/MemoryStorageProvider.ts

@@ -15,6 +15,7 @@ export class MemoryStorageProvider extends MSP implements IBridgeStorageProvider
     private gitlabDiscussionThreads = new Map<string, SerializedGitlabDiscussionThreads>();
     private feedGuids = new Map<string, Array<string>>();
     private houndActivityIds = new Map<string, Array<string>>();
+    private houndActivityIdToEvent = new Map<string, string>();
 
     constructor() {
         super();
@@ -110,19 +111,27 @@ export class MemoryStorageProvider extends MSP implements IBridgeStorageProvider
         this.gitlabDiscussionThreads.set(connectionId, value);
     }
 
-    async storeHoundActivity(url: string, ...ids: string[]): Promise<void> {
+    async storeHoundActivity(challengeId: string, ...activityIds: string[]): Promise<void> {
-        let set = this.houndActivityIds.get(url);
+        let set = this.houndActivityIds.get(challengeId);
         if (!set) {
             set = []
-            this.houndActivityIds.set(url, set);
+            this.houndActivityIds.set(challengeId, set);
         }
-        set.unshift(...ids);
+        set.unshift(...activityIds);
         while (set.length > MAX_FEED_ITEMS) {
             set.pop();
         } 
     }
-    async hasSeenHoundActivity(url: string, ...ids: string[]): Promise<string[]> {
+    async hasSeenHoundActivity(challengeId: string, ...activityIds: string[]): Promise<string[]> {
-        const existing = this.houndActivityIds.get(url);
+        const existing = this.houndActivityIds.get(challengeId);
-        return existing ? ids.filter((existingGuid) => existing.includes(existingGuid)) : [];
+        return existing ? activityIds.filter((existingGuid) => existing.includes(existingGuid)) : [];
+    }
+
+    public async storeHoundActivityEvent(challengeId: string, activityId: string, eventId: string): Promise<void> {
+        this.houndActivityIdToEvent.set(`${challengeId}.${activityId}`, eventId);
+    }
+
+    public async getHoundActivity(challengeId: string, activityId: string): Promise<string|null> {
+        return this.houndActivityIdToEvent.get(`${challengeId}.${activityId}`) ?? null;
     }
 }

src/Stores/RedisStorageProvider.ts

@@ -23,13 +23,15 @@ const STORED_FILES_EXPIRE_AFTER = 24 * 60 * 60; // 24 hours
 const COMPLETED_TRANSACTIONS_EXPIRE_AFTER = 24 * 60 * 60; // 24 hours
 const ISSUES_EXPIRE_AFTER = 7 * 24 * 60 * 60; // 7 days
 const ISSUES_LAST_COMMENT_EXPIRE_AFTER = 14 * 24 * 60 * 60; // 7 days
+const HOUND_EVENT_CACHE = 90 * 24 * 60 * 60; // 30 days
 
 
 const WIDGET_TOKENS = "widgets.tokens.";
 const WIDGET_USER_TOKENS = "widgets.user-tokens.";
 
 const FEED_GUIDS = "feeds.guids.";
-const HOUND_IDS = "feeds.guids.";
+const HOUND_GUIDS = "hound.guids.";
+const HOUND_EVENTS = "hound.events.";
 
 const log = new Logger("RedisASProvider");
 
@@ -242,24 +244,36 @@ export class RedisStorageProvider extends RedisStorageContextualProvider impleme
         return guids.filter((_guid, index) => res[index][1] !== null);
     }
 
-    public async storeHoundActivity(url: string, ...guids: string[]): Promise<void> {
+    public async storeHoundActivity(challengeId: string, ...activityHashes: string[]): Promise<void> {
-        const feedKey = `${HOUND_IDS}${url}`;
+        const key = `${HOUND_GUIDS}${challengeId}`;
-        await this.redis.lpush(feedKey, ...guids);
+        await this.redis.lpush(key, ...activityHashes);
-        await this.redis.ltrim(feedKey, 0, MAX_FEED_ITEMS);
+        await this.redis.ltrim(key, 0, MAX_FEED_ITEMS);
     }
 
-    public async hasSeenHoundActivity(url: string, ...guids: string[]): Promise<string[]> {
+    public async hasSeenHoundActivity(challengeId: string, ...activityHashes: string[]): Promise<string[]> {
         let multi = this.redis.multi();
-        const feedKey = `${HOUND_IDS}${url}`;
+        const key = `${HOUND_GUIDS}${challengeId}`;
 
-        for (const guid of guids) {
+        for (const guid of activityHashes) {
-            multi = multi.lpos(feedKey, guid);
+            multi = multi.lpos(key, guid);
         }
         const res = await multi.exec();
         if (res === null) {
             // Just assume we've seen none.
             return [];
         }
-        return guids.filter((_guid, index) => res[index][1] !== null);
+        return activityHashes.filter((_guid, index) => res[index][1] !== null);
+    }
+
+    public async storeHoundActivityEvent(challengeId: string, activityId: string, eventId: string): Promise<void> {
+        const key = `${HOUND_EVENTS}${challengeId}.${activityId}`;
+        await this.redis.set(key, eventId);
+        this.redis.expire(key, HOUND_EVENT_CACHE).catch((ex) => {
+            log.warn(`Failed to set expiry time on ${key}`, ex);
+        });
+    }
+
+    public async getHoundActivity(challengeId: string, activityId: string): Promise<string|null> {
+        return this.redis.get(`${HOUND_EVENTS}${challengeId}.${activityId}`);
     }
 }

src/Stores/StorageProvider.ts

@@ -9,6 +9,8 @@ import { SerializedGitlabDiscussionThreads } from "../Gitlab/Types";
 // seen from this feed, up to a max of 10,000.
 // Adopted from https://github.com/matrix-org/go-neb/blob/babb74fa729882d7265ff507b09080e732d060ae/services/rssbot/rssbot.go#L304
 export const MAX_FEED_ITEMS = 10_000;
+export const MAX_HOUND_ITEMS = 100;
+
 
 export interface IBridgeStorageProvider extends IAppserviceStorageProvider, IStorageProvider, ProvisioningStore {
     connect?(): Promise<void>;
@@ -28,6 +30,9 @@ export interface IBridgeStorageProvider extends IAppserviceStorageProvider, ISto
     storeFeedGuids(url: string, ...guids: string[]): Promise<void>;
     hasSeenFeed(url: string): Promise<boolean>;
     hasSeenFeedGuids(url: string, ...guids: string[]): Promise<string[]>;
-    storeHoundActivity(id: string, ...guids: string[]): Promise<void>;
+
-    hasSeenHoundActivity(id: string, ...guids: string[]): Promise<string[]>;
+    storeHoundActivity(challengeId: string, ...activityHashes: string[]): Promise<void>;
+    hasSeenHoundActivity(challengeId: string, ...activityHashes: string[]): Promise<string[]>;
+    storeHoundActivityEvent(challengeId: string, activityId: string, eventId: string): Promise<void>;
+    getHoundActivity(challengeId: string, activityId: string): Promise<string|null>;
 }

src/hound/reader.ts

@@ -5,6 +5,7 @@ import { MessageQueue } from "../MessageQueue";
 import { IBridgeStorageProvider } from "../Stores/StorageProvider";
 import { BridgeConfigChallengeHound } from "../config/Config";
 import { Logger } from "matrix-appservice-bridge";
+import { hashId } from "../libRs";
 
 const log = new Logger("HoundReader");
 
@@ -74,12 +75,16 @@ export class HoundReader {
         }
     }
 
+    private static hashActivity(activity: HoundActivity) {
+        return hashId(activity.activityId + activity.name + activity.distanceKilometers + activity.durationSeconds + activity.elevationMeters);
+    }
+
     public async poll(challengeId: string) {
-        const resAct = await this.houndClient.get(`https://api.challengehound.com/challenges/${challengeId}/activities?limit=10`);
+        const resAct = await this.houndClient.get(`https://api.challengehound.com/v1/activities?challengeId=${challengeId}&size=10`);
-        const activites = resAct.data as HoundActivity[];
+        const activites = (resAct.data["results"] as HoundActivity[]).map(a => ({...a, hash: HoundReader.hashActivity(a)}));
-        const seen = await this.storage.hasSeenHoundActivity(challengeId, ...activites.map(a => a.id));
+        const seen = await this.storage.hasSeenHoundActivity(challengeId, ...activites.map(a => a.hash));
         for (const activity of activites) {
-            if (seen.includes(activity.id)) {
+            if (seen.includes(activity.hash)) {
                 continue;
             }
             this.queue.push<HoundPayload>({
@@ -91,7 +96,7 @@ export class HoundReader {
                 }
             });
         }
-        await this.storage.storeHoundActivity(challengeId, ...activites.map(a => a.id))
+        await this.storage.storeHoundActivity(challengeId, ...activites.map(a => a.hash))
     }
 
     public async pollChallenges(): Promise<void> {
@@ -112,6 +117,8 @@ export class HoundReader {
                 if (elapsed > this.sleepingInterval) {
                     log.warn(`It took us longer to update the activities than the expected interval`);
                 }
+            } catch (ex) {
+                log.warn("Failed to poll for challenge", ex);
             } finally {
                 this.challengeIds.splice(0, 0, challengeId);
             }