diff --git a/changelog.d/714.feature b/changelog.d/714.feature new file mode 100644 index 00000000..8a7dc9aa --- /dev/null +++ b/changelog.d/714.feature @@ -0,0 +1 @@ +Use SQLite for file-based crypto stores by default, instead of Sled. diff --git a/docs/advanced/encryption.md b/docs/advanced/encryption.md index 8db6dae2..535f93f1 100644 --- a/docs/advanced/encryption.md +++ b/docs/advanced/encryption.md @@ -11,10 +11,12 @@ Hookshot supports end-to-bridge encryption via [MSC3202](https://github.com/matr ## Enabling encryption in Hookshot In order for hookshot to use encryption, it must be configured as follows: -- The `experimentalEncryption.storagePath` setting must point to a directory that hookshot has permissions to write files into. If running with Docker, this path should be within a volume (for persistency). +- The `experimentalEncryption.storagePath` setting must point to a directory that hookshot has permissions to write files into. If running with Docker, this path should be within a volume (for persistency). Hookshot uses this directory for its crypto store (i.e. long-lived state relating to its encryption keys). + - By default, the crypto store uses an SQLite-based format. To use the legacy Sled-based format, set `experimentalEncryption.useLegacySledStore` to `true`, though this is not expected to be necessary and will be removed in a future release. + - Once a crypto store of a particular type (SQLite or Sled) has been initialized, its files must not be modified, and hookshot cannot be configured to use another crypto store of the same type as one it has used before. If a crypto store's files get lost or corrupted, hookshot may fail to start up, or may be unable to decrypt command messages. To fix such issues, stop hookshot, then reset its crypto store by running `yarn start:resetcrypto`. - [Redis](./workers.md) must be enabled. Note that worker mode is not yet supported with encryption, so `queue.monolithic` must be set to `true`. -If you ever reset your homeserver's state, ensure you also reset hookshot's encryption state. This includes clearing the `encryption.storagePath` directory and all worker state stored in your redis instance. Otherwise, hookshot may fail on start up with registration errors. +If you ever reset your homeserver's state, ensure you also reset hookshot's encryption state. This includes clearing the `experimentalEncryption.storagePath` directory and all worker state stored in your redis instance. Otherwise, hookshot may fail on start up with registration errors. Also ensure that hookshot's appservice registration file contains every line from `registration.sample.yml` that appears after the `If enabling encryption` comment. Note that changing the registration file may require restarting the homeserver that hookshot is connected to. diff --git a/package.json b/package.json index 6ec5525e..22980ad0 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "start:app": "node --require source-map-support/register lib/App/BridgeApp.js", "start:webhooks": "node --require source-map-support/register lib/App/GithubWebhookApp.js", "start:matrixsender": "node --require source-map-support/register lib/App/MatrixSenderApp.js", + "start:resetcrypto": "node --require source-map-support/register lib/App/ResetCryptoStore.js", "test": "mocha -r ts-node/register tests/init.ts tests/*.ts tests/**/*.ts", "test:cover": "nyc --reporter=lcov --reporter=text yarn test", "lint": "yarn run lint:js && yarn run lint:rs", @@ -54,7 +55,7 @@ "jira-client": "^8.0.0", "markdown-it": "^12.3.2", "matrix-appservice-bridge": "^6.0.0", - "matrix-bot-sdk": "npm:@vector-im/matrix-bot-sdk@^0.6.3-element.0", + "matrix-bot-sdk": "npm:@vector-im/matrix-bot-sdk@^0.6.6-element.0", "matrix-widget-api": "^1.0.0", "micromatch": "^4.0.4", "mime": "^3.0.0", diff --git a/src/App/BridgeApp.ts b/src/App/BridgeApp.ts index c359e17e..5c24e52a 100644 --- a/src/App/BridgeApp.ts +++ b/src/App/BridgeApp.ts @@ -45,6 +45,8 @@ async function start() { log.error("Got SIGTERM"); listener.stop(); bridgeApp.stop(); + // Don't care to await this, as the process is about to end + storage.disconnect?.(); }); await bridgeApp.start(); diff --git a/src/App/MatrixSenderApp.ts b/src/App/MatrixSenderApp.ts index 063a4de5..f0f48919 100644 --- a/src/App/MatrixSenderApp.ts +++ b/src/App/MatrixSenderApp.ts @@ -23,7 +23,8 @@ async function start() { LogService.setLogger(Logger.botSdkLogger); const listener = new ListenerService(config.listeners); listener.start(); - const sender = new MatrixSender(config, getAppservice(config, registration).appservice); + const {appservice, storage} = getAppservice(config, registration); + const sender = new MatrixSender(config, appservice); if (config.metrics) { if (!config.metrics.port) { log.warn(`Not running metrics for service, no port specified`); @@ -36,6 +37,7 @@ async function start() { log.error("Got SIGTERM"); sender.stop(); listener.stop(); + storage.disconnect?.(); }); } diff --git a/src/App/ResetCryptoStore.ts b/src/App/ResetCryptoStore.ts new file mode 100644 index 00000000..62780a44 --- /dev/null +++ b/src/App/ResetCryptoStore.ts @@ -0,0 +1,89 @@ +import { rm } from "fs/promises"; + +import { BridgeConfig, parseRegistrationFile } from "../Config/Config"; +import { Logger } from "matrix-appservice-bridge"; +import { LogService, MatrixClient } from "matrix-bot-sdk"; +import { getAppservice } from "../appservice"; +import BotUsersManager from "../Managers/BotUsersManager"; + +const log = new Logger("ResetCryptoStore"); + +async function start() { + const configFile = process.argv[2] || "./config.yml"; + const registrationFile = process.argv[3] || "./registration.yml"; + const config = await BridgeConfig.parseConfig(configFile, process.env); + const registration = await parseRegistrationFile(registrationFile); + Logger.configure({ + console: config.logging.level, + colorize: config.logging.colorize, + json: config.logging.json, + timestampFormat: config.logging.timestampFormat + }); + LogService.setLogger(Logger.botSdkLogger); + + const {appservice, storage, cryptoStorage} = getAppservice(config, registration); + if (!cryptoStorage) { + log.info(`Encryption is not enabled in the configuration file "${configFile}", so there is no encryption state to be reset`); + return; + } + + const botUsersManager = new BotUsersManager(config, appservice); + + for (const botUser of botUsersManager.botUsers) { + try { + const userStorage = storage.storageForUser?.(botUser.userId); + if (!userStorage) { + log.warn(`No storage for ${botUser.userId}`); + continue; + } + const accessToken = await userStorage?.readValue("accessToken"); + if (!accessToken) { + log.debug(`No access token for ${botUser.userId}: no session to remove`); + continue; + } + + const userCryptoStorage = cryptoStorage?.storageForUser(botUser.userId); + if (!userCryptoStorage) { + log.warn(`No crypto storage for ${botUser.userId}`); + continue; + } + const deviceId = await userCryptoStorage?.getDeviceId(); + if (!deviceId) { + log.debug(`No crypto device ID for ${botUser.userId}: no crypto state to remove`); + continue; + } + + const client = new MatrixClient(config.bridge.url, accessToken, userStorage, userCryptoStorage); + await client.doRequest("POST", "/_matrix/client/v3/logout", { + user_id: botUser.userId, + "org.matrix.msc3202.device_id": deviceId, + }); + log.info(`Logged out crypto device for ${botUser.userId}`); + + try { + await userStorage.storeValue("accessToken", ""); + log.info(`Deleted access token for ${botUser.userId}`); + } catch (ex: unknown) { + log.error(`Failed to delete access token for ${botUser.userId}`, ex); + } + + } catch (ex: unknown) { + log.error(`Failed to log out crypto device for ${botUser.userId}`, ex); + } + } + + if (config.encryption?.storagePath) { + try { + await rm(config.encryption.storagePath, { recursive: true, force: true }); + log.info("Removed crypto store from disk"); + } catch (ex) { + log.error("Failed to remove crypto store from disk", ex); + } + } + + await storage.disconnect?.(); +} + +start().catch((ex) => { + log.error("ResetCryptoStore encountered an error and has stopped:", ex); +}); diff --git a/src/Config/Config.ts b/src/Config/Config.ts index c3f9aa66..4a56e9ba 100644 --- a/src/Config/Config.ts +++ b/src/Config/Config.ts @@ -409,6 +409,7 @@ interface BridgeConfigBot { } interface BridgeConfigEncryption { storagePath: string; + useLegacySledStore: boolean; } export interface BridgeConfigServiceBot { diff --git a/src/Stores/RedisStorageProvider.ts b/src/Stores/RedisStorageProvider.ts index 9024519f..7fae2491 100644 --- a/src/Stores/RedisStorageProvider.ts +++ b/src/Stores/RedisStorageProvider.ts @@ -87,6 +87,10 @@ export class RedisStorageProvider extends RedisStorageContextualProvider impleme } } + public async disconnect(): Promise { + await this.redis.quit(); + } + public async addRegisteredUser(userId: string) { this.redis.sadd(REGISTERED_USERS_KEY, [userId]); } diff --git a/src/Stores/StorageProvider.ts b/src/Stores/StorageProvider.ts index 09e5c81e..fe293aae 100644 --- a/src/Stores/StorageProvider.ts +++ b/src/Stores/StorageProvider.ts @@ -4,6 +4,7 @@ import { IssuesGetResponseData } from "../Github/Types"; export interface IBridgeStorageProvider extends IAppserviceStorageProvider, IStorageProvider, ProvisioningStore { connect?(): Promise; + disconnect?(): Promise; setGithubIssue(repo: string, issueNumber: string, data: IssuesGetResponseData, scope?: string): Promise; getGithubIssue(repo: string, issueNumber: string, scope?: string): Promise; setLastNotifCommentUrl(repo: string, issueNumber: string, url: string, scope?: string): Promise; diff --git a/src/appservice.ts b/src/appservice.ts index ff787392..98cbfd6d 100644 --- a/src/appservice.ts +++ b/src/appservice.ts @@ -1,5 +1,5 @@ import { Logger } from "matrix-appservice-bridge"; -import { Appservice, IAppserviceRegistration, RustSdkAppserviceCryptoStorageProvider } from "matrix-bot-sdk"; +import { Appservice, IAppserviceCryptoStorageProvider, IAppserviceRegistration, RustSdkAppserviceCryptoStorageProvider, RustSdkCryptoStoreType } from "matrix-bot-sdk"; import { BridgeConfig } from "./Config/Config"; import Metrics from "./Metrics"; import { MemoryStorageProvider } from "./Stores/MemoryStorageProvider"; @@ -17,7 +17,14 @@ export function getAppservice(config: BridgeConfig, registration: IAppserviceReg storage = new MemoryStorageProvider(); } - const cryptoStorage = config.encryption?.storagePath ? new RustSdkAppserviceCryptoStorageProvider(config.encryption.storagePath) : undefined; + let cryptoStorage: IAppserviceCryptoStorageProvider | undefined; + if (config.encryption?.storagePath) { + log.info('Initialising crypto storage') + cryptoStorage = new RustSdkAppserviceCryptoStorageProvider( + config.encryption.storagePath, + config.encryption.useLegacySledStore ? RustSdkCryptoStoreType.Sled : RustSdkCryptoStoreType.Sqlite + ); + } const appservice = new Appservice({ homeserverName: config.bridge.domain, @@ -45,5 +52,5 @@ export function getAppservice(config: BridgeConfig, registration: IAppserviceReg Metrics.registerMatrixSdkMetrics(appservice); - return {appservice, storage}; + return {appservice, storage, cryptoStorage}; } \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 52d6234c..de369eee 100644 --- a/yarn.lock +++ b/yarn.lock @@ -876,6 +876,14 @@ dependencies: "@lezer/common" "^1.0.0" +"@matrix-org/matrix-sdk-crypto-nodejs@0.1.0-beta.4": + version "0.1.0-beta.4" + resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-nodejs/-/matrix-sdk-crypto-nodejs-0.1.0-beta.4.tgz#80456b2e2cc731982f0d3c6aece80cefa1ebb797" + integrity sha512-XjCp/tG3LRMxMj/MMZfypD5BtW3J1B6oXY2Og8Ed0SyU4uWdglalMwrBUKlDotJr0/Q/2OTspGjD+ytAzCspyw== + dependencies: + https-proxy-agent "^5.0.1" + node-downloader-helper "^2.1.5" + "@matrix-org/matrix-sdk-crypto-nodejs@^0.1.0-beta.1": version "0.1.0-beta.1" resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-nodejs/-/matrix-sdk-crypto-nodejs-0.1.0-beta.1.tgz#8a9058226916a258e5b4e28d76680e895b6203b2" @@ -883,13 +891,6 @@ dependencies: node-downloader-helper "^2.1.1" -"@matrix-org/matrix-sdk-crypto-nodejs@^0.1.0-beta.3": - version "0.1.0-beta.3" - resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-nodejs/-/matrix-sdk-crypto-nodejs-0.1.0-beta.3.tgz#a07225dd180d9d227c24ba62bba439939446d113" - integrity sha512-jHFn6xBeNqfsY5gX60akbss7iFBHZwXycJWMw58Mjz08OwOi7AbTxeS9I2Pa4jX9/M2iinskmGZbzpqOT2fM3A== - dependencies: - node-downloader-helper "^2.1.1" - "@mdn/browser-compat-data@^3.3.14": version "3.3.14" resolved "https://registry.yarnpkg.com/@mdn/browser-compat-data/-/browser-compat-data-3.3.14.tgz#b72a37c654e598f9ae6f8335faaee182bebc6b28" @@ -1668,6 +1669,13 @@ acorn@^8.4.1, acorn@^8.6.0, acorn@^8.7.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30" integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A== +agent-base@6: + version "6.0.2" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" + integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== + dependencies: + debug "4" + aggregate-error@^3.0.0, aggregate-error@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" @@ -2415,6 +2423,13 @@ debug@2.6.9: dependencies: ms "2.0.0" +debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4: + version "4.3.4" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" + integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== + dependencies: + ms "2.1.2" + debug@4.3.1: version "4.3.1" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee" @@ -2422,13 +2437,6 @@ debug@4.3.1: dependencies: ms "2.1.2" -debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4: - version "4.3.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" - integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== - dependencies: - ms "2.1.2" - decamelize@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -2571,7 +2579,7 @@ domhandler@^4.0.0, domhandler@^4.2.0: dependencies: domelementtype "^2.2.0" -domhandler@^5.0.1, domhandler@^5.0.2: +domhandler@^5.0.1, domhandler@^5.0.2, domhandler@^5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31" integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w== @@ -2663,6 +2671,11 @@ entities@^4.2.0: resolved "https://registry.yarnpkg.com/entities/-/entities-4.4.0.tgz#97bdaba170339446495e653cfd2db78962900174" integrity sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA== +entities@^4.4.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48" + integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw== + entities@~2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/entities/-/entities-2.1.0.tgz#992d3129cf7df6870b96c57858c249a120f8b8b5" @@ -3523,6 +3536,16 @@ htmlparser2@^6.0.0, htmlparser2@^6.1.0: domutils "^2.5.2" entities "^2.0.0" +htmlparser2@^8.0.0: + version "8.0.2" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-8.0.2.tgz#f002151705b383e62433b5cf466f5b716edaec21" + integrity sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.3" + domutils "^3.0.1" + entities "^4.4.0" + http-errors@1.8.1: version "1.8.1" resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c" @@ -3568,6 +3591,14 @@ http-status-codes@^2.2.0: resolved "https://registry.yarnpkg.com/http-status-codes/-/http-status-codes-2.2.0.tgz#bb2efe63d941dfc2be18e15f703da525169622be" integrity sha512-feERVo9iWxvnejp3SEfm/+oNG517npqL2/PIA8ORjyOZjGC7TwCRQsZylciLS64i6pJ0wRYz3rkXLRwbtFa8Ng== +https-proxy-agent@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" + integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA== + dependencies: + agent-base "6" + debug "4" + iconv-lite@0.4.24: version "0.4.24" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" @@ -4330,12 +4361,12 @@ matrix-bot-sdk@^0.6.2: request-promise "^4.2.6" sanitize-html "^2.7.0" -"matrix-bot-sdk@npm:@vector-im/matrix-bot-sdk@^0.6.3-element.0": - version "0.6.3-element.0" - resolved "https://registry.yarnpkg.com/@vector-im/matrix-bot-sdk/-/matrix-bot-sdk-0.6.3-element.0.tgz#dfd36d1073145ce95b609b3f78f66a50a9c520e4" - integrity sha512-PuH4vSP2aE0TkJUqLtPsfqU0oEAlW9zmYOd8b5hY8RSkpfitup4lCwTYipWovJIKeoolNv5uMPA2mzqM3mcP6w== +"matrix-bot-sdk@npm:@vector-im/matrix-bot-sdk@^0.6.6-element.0": + version "0.6.6-element.0" + resolved "https://registry.yarnpkg.com/@vector-im/matrix-bot-sdk/-/matrix-bot-sdk-0.6.6-element.0.tgz#96e08bb434939fcbf7f771a47aa4a7d17632f601" + integrity sha512-Mx5SAv1zFXpkG+jbtbkadNMLjaxftnWBGCjKlNFlUDjcJPUIYxKd49jUGst/vdp/ysZYa0Dgk+V+E1qmKUa8bw== dependencies: - "@matrix-org/matrix-sdk-crypto-nodejs" "^0.1.0-beta.3" + "@matrix-org/matrix-sdk-crypto-nodejs" "0.1.0-beta.4" "@types/express" "^4.17.13" another-json "^0.2.0" async-lock "^1.3.2" @@ -4351,7 +4382,7 @@ matrix-bot-sdk@^0.6.2: morgan "^1.10.0" request "^2.88.2" request-promise "^4.2.6" - sanitize-html "^2.7.0" + sanitize-html "^2.8.0" matrix-widget-api@^1.0.0: version "1.0.0" @@ -4592,6 +4623,11 @@ node-downloader-helper@^2.1.1: resolved "https://registry.yarnpkg.com/node-downloader-helper/-/node-downloader-helper-2.1.1.tgz#533427a3cdc163931b106d0fe6d522f83deac7ab" integrity sha512-ouk8MGmJj1gYymbJwi1L8Mr6PdyheJLwfsmyx0KtsvyJ+7Fpf0kBBzM8Gmx8Mt/JBfRWP1PQm6dAGV6x7eNedw== +node-downloader-helper@^2.1.5: + version "2.1.6" + resolved "https://registry.yarnpkg.com/node-downloader-helper/-/node-downloader-helper-2.1.6.tgz#f73ac458e3ac8c21afd0b952a994eab99c64b879" + integrity sha512-VkOvAXIopI3xMuM/MC5UL7NqqnizQ/9QXZt28jR8FPZ6fHLQm4xe4+YXJ9FqsWwLho5BLXrF51nfOQ0QcohRkQ== + node-emoji@^1.11.0: version "1.11.0" resolved "https://registry.yarnpkg.com/node-emoji/-/node-emoji-1.11.0.tgz#69a0150e6946e2f115e9d7ea4df7971e2628301c" @@ -5380,6 +5416,18 @@ sanitize-html@^2.7.0: parse-srcset "^1.0.2" postcss "^8.3.11" +sanitize-html@^2.8.0: + version "2.10.0" + resolved "https://registry.yarnpkg.com/sanitize-html/-/sanitize-html-2.10.0.tgz#74d28848dfcf72c39693139131895c78900ab452" + integrity sha512-JqdovUd81dG4k87vZt6uA6YhDfWkUGruUu/aPmXLxXi45gZExnt9Bnw/qeQU8oGf82vPyaE0vO4aH0PbobB9JQ== + dependencies: + deepmerge "^4.2.2" + escape-string-regexp "^4.0.0" + htmlparser2 "^8.0.0" + is-plain-object "^5.0.0" + parse-srcset "^1.0.2" + postcss "^8.3.11" + sass@^1.51.0: version "1.51.0" resolved "https://registry.yarnpkg.com/sass/-/sass-1.51.0.tgz#25ea36cf819581fe1fe8329e8c3a4eaaf70d2845"