Allow usage of the SQLite-based crypto store (#714)

* Allow usage of the SQLite-based crypto store

Signed-off-by: Andrew Ferrazzutti <andrewf@element.io>

* Add changelog

* Log when crypto storage is initialized

* Add yarn script for resetting crypto state

Also document its usage & when it may be necessary to use it.

* Minor style improvements

- add trailing semicolons
- remove redundant `Promise.resolve`s

* Change logging in crypto reset script

- Move error object to parameter
- Rename logger

* Add method to close the Redis store

and call it to let the process exit cleanly

* Update matrix-bot-sdk

Use release with SQLite crypto store support

---------

Signed-off-by: Andrew Ferrazzutti <andrewf@element.io>
This commit is contained in:
Andrew Ferrazzutti 2023-04-21 23:18:27 +09:00 committed by GitHub
parent a9537c7961
commit f29fac7584
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 186 additions and 28 deletions

1
changelog.d/714.feature Normal file
View File

@ -0,0 +1 @@
Use SQLite for file-based crypto stores by default, instead of Sled.

View File

@ -11,10 +11,12 @@ Hookshot supports end-to-bridge encryption via [MSC3202](https://github.com/matr
## Enabling encryption in Hookshot
In order for hookshot to use encryption, it must be configured as follows:
- The `experimentalEncryption.storagePath` setting must point to a directory that hookshot has permissions to write files into. If running with Docker, this path should be within a volume (for persistency).
- The `experimentalEncryption.storagePath` setting must point to a directory that hookshot has permissions to write files into. If running with Docker, this path should be within a volume (for persistency). Hookshot uses this directory for its crypto store (i.e. long-lived state relating to its encryption keys).
- By default, the crypto store uses an SQLite-based format. To use the legacy Sled-based format, set `experimentalEncryption.useLegacySledStore` to `true`, though this is not expected to be necessary and will be removed in a future release.
- Once a crypto store of a particular type (SQLite or Sled) has been initialized, its files must not be modified, and hookshot cannot be configured to use another crypto store of the same type as one it has used before. If a crypto store's files get lost or corrupted, hookshot may fail to start up, or may be unable to decrypt command messages. To fix such issues, stop hookshot, then reset its crypto store by running `yarn start:resetcrypto`.
- [Redis](./workers.md) must be enabled. Note that worker mode is not yet supported with encryption, so `queue.monolithic` must be set to `true`.
If you ever reset your homeserver's state, ensure you also reset hookshot's encryption state. This includes clearing the `encryption.storagePath` directory and all worker state stored in your redis instance. Otherwise, hookshot may fail on start up with registration errors.
If you ever reset your homeserver's state, ensure you also reset hookshot's encryption state. This includes clearing the `experimentalEncryption.storagePath` directory and all worker state stored in your redis instance. Otherwise, hookshot may fail on start up with registration errors.
Also ensure that hookshot's appservice registration file contains every line from `registration.sample.yml` that appears after the `If enabling encryption` comment. Note that changing the registration file may require restarting the homeserver that hookshot is connected to.

View File

@ -30,6 +30,7 @@
"start:app": "node --require source-map-support/register lib/App/BridgeApp.js",
"start:webhooks": "node --require source-map-support/register lib/App/GithubWebhookApp.js",
"start:matrixsender": "node --require source-map-support/register lib/App/MatrixSenderApp.js",
"start:resetcrypto": "node --require source-map-support/register lib/App/ResetCryptoStore.js",
"test": "mocha -r ts-node/register tests/init.ts tests/*.ts tests/**/*.ts",
"test:cover": "nyc --reporter=lcov --reporter=text yarn test",
"lint": "yarn run lint:js && yarn run lint:rs",
@ -54,7 +55,7 @@
"jira-client": "^8.0.0",
"markdown-it": "^12.3.2",
"matrix-appservice-bridge": "^6.0.0",
"matrix-bot-sdk": "npm:@vector-im/matrix-bot-sdk@^0.6.3-element.0",
"matrix-bot-sdk": "npm:@vector-im/matrix-bot-sdk@^0.6.6-element.0",
"matrix-widget-api": "^1.0.0",
"micromatch": "^4.0.4",
"mime": "^3.0.0",

View File

@ -45,6 +45,8 @@ async function start() {
log.error("Got SIGTERM");
listener.stop();
bridgeApp.stop();
// Don't care to await this, as the process is about to end
storage.disconnect?.();
});
await bridgeApp.start();

View File

@ -23,7 +23,8 @@ async function start() {
LogService.setLogger(Logger.botSdkLogger);
const listener = new ListenerService(config.listeners);
listener.start();
const sender = new MatrixSender(config, getAppservice(config, registration).appservice);
const {appservice, storage} = getAppservice(config, registration);
const sender = new MatrixSender(config, appservice);
if (config.metrics) {
if (!config.metrics.port) {
log.warn(`Not running metrics for service, no port specified`);
@ -36,6 +37,7 @@ async function start() {
log.error("Got SIGTERM");
sender.stop();
listener.stop();
storage.disconnect?.();
});
}

View File

@ -0,0 +1,89 @@
import { rm } from "fs/promises";
import { BridgeConfig, parseRegistrationFile } from "../Config/Config";
import { Logger } from "matrix-appservice-bridge";
import { LogService, MatrixClient } from "matrix-bot-sdk";
import { getAppservice } from "../appservice";
import BotUsersManager from "../Managers/BotUsersManager";
const log = new Logger("ResetCryptoStore");
async function start() {
const configFile = process.argv[2] || "./config.yml";
const registrationFile = process.argv[3] || "./registration.yml";
const config = await BridgeConfig.parseConfig(configFile, process.env);
const registration = await parseRegistrationFile(registrationFile);
Logger.configure({
console: config.logging.level,
colorize: config.logging.colorize,
json: config.logging.json,
timestampFormat: config.logging.timestampFormat
});
LogService.setLogger(Logger.botSdkLogger);
const {appservice, storage, cryptoStorage} = getAppservice(config, registration);
if (!cryptoStorage) {
log.info(`Encryption is not enabled in the configuration file "${configFile}", so there is no encryption state to be reset`);
return;
}
const botUsersManager = new BotUsersManager(config, appservice);
for (const botUser of botUsersManager.botUsers) {
try {
const userStorage = storage.storageForUser?.(botUser.userId);
if (!userStorage) {
log.warn(`No storage for ${botUser.userId}`);
continue;
}
const accessToken = await userStorage?.readValue("accessToken");
if (!accessToken) {
log.debug(`No access token for ${botUser.userId}: no session to remove`);
continue;
}
const userCryptoStorage = cryptoStorage?.storageForUser(botUser.userId);
if (!userCryptoStorage) {
log.warn(`No crypto storage for ${botUser.userId}`);
continue;
}
const deviceId = await userCryptoStorage?.getDeviceId();
if (!deviceId) {
log.debug(`No crypto device ID for ${botUser.userId}: no crypto state to remove`);
continue;
}
const client = new MatrixClient(config.bridge.url, accessToken, userStorage, userCryptoStorage);
await client.doRequest("POST", "/_matrix/client/v3/logout", {
user_id: botUser.userId,
"org.matrix.msc3202.device_id": deviceId,
});
log.info(`Logged out crypto device for ${botUser.userId}`);
try {
await userStorage.storeValue("accessToken", "");
log.info(`Deleted access token for ${botUser.userId}`);
} catch (ex: unknown) {
log.error(`Failed to delete access token for ${botUser.userId}`, ex);
}
} catch (ex: unknown) {
log.error(`Failed to log out crypto device for ${botUser.userId}`, ex);
}
}
if (config.encryption?.storagePath) {
try {
await rm(config.encryption.storagePath, { recursive: true, force: true });
log.info("Removed crypto store from disk");
} catch (ex) {
log.error("Failed to remove crypto store from disk", ex);
}
}
await storage.disconnect?.();
}
start().catch((ex) => {
log.error("ResetCryptoStore encountered an error and has stopped:", ex);
});

View File

@ -409,6 +409,7 @@ interface BridgeConfigBot {
}
interface BridgeConfigEncryption {
storagePath: string;
useLegacySledStore: boolean;
}
export interface BridgeConfigServiceBot {

View File

@ -87,6 +87,10 @@ export class RedisStorageProvider extends RedisStorageContextualProvider impleme
}
}
public async disconnect(): Promise<void> {
await this.redis.quit();
}
public async addRegisteredUser(userId: string) {
this.redis.sadd(REGISTERED_USERS_KEY, [userId]);
}

View File

@ -4,6 +4,7 @@ import { IssuesGetResponseData } from "../Github/Types";
export interface IBridgeStorageProvider extends IAppserviceStorageProvider, IStorageProvider, ProvisioningStore {
connect?(): Promise<void>;
disconnect?(): Promise<void>;
setGithubIssue(repo: string, issueNumber: string, data: IssuesGetResponseData, scope?: string): Promise<void>;
getGithubIssue(repo: string, issueNumber: string, scope?: string): Promise<IssuesGetResponseData|null>;
setLastNotifCommentUrl(repo: string, issueNumber: string, url: string, scope?: string): Promise<void>;

View File

@ -1,5 +1,5 @@
import { Logger } from "matrix-appservice-bridge";
import { Appservice, IAppserviceRegistration, RustSdkAppserviceCryptoStorageProvider } from "matrix-bot-sdk";
import { Appservice, IAppserviceCryptoStorageProvider, IAppserviceRegistration, RustSdkAppserviceCryptoStorageProvider, RustSdkCryptoStoreType } from "matrix-bot-sdk";
import { BridgeConfig } from "./Config/Config";
import Metrics from "./Metrics";
import { MemoryStorageProvider } from "./Stores/MemoryStorageProvider";
@ -17,7 +17,14 @@ export function getAppservice(config: BridgeConfig, registration: IAppserviceReg
storage = new MemoryStorageProvider();
}
const cryptoStorage = config.encryption?.storagePath ? new RustSdkAppserviceCryptoStorageProvider(config.encryption.storagePath) : undefined;
let cryptoStorage: IAppserviceCryptoStorageProvider | undefined;
if (config.encryption?.storagePath) {
log.info('Initialising crypto storage')
cryptoStorage = new RustSdkAppserviceCryptoStorageProvider(
config.encryption.storagePath,
config.encryption.useLegacySledStore ? RustSdkCryptoStoreType.Sled : RustSdkCryptoStoreType.Sqlite
);
}
const appservice = new Appservice({
homeserverName: config.bridge.domain,
@ -45,5 +52,5 @@ export function getAppservice(config: BridgeConfig, registration: IAppserviceReg
Metrics.registerMatrixSdkMetrics(appservice);
return {appservice, storage};
return {appservice, storage, cryptoStorage};
}

View File

@ -876,6 +876,14 @@
dependencies:
"@lezer/common" "^1.0.0"
"@matrix-org/matrix-sdk-crypto-nodejs@0.1.0-beta.4":
version "0.1.0-beta.4"
resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-nodejs/-/matrix-sdk-crypto-nodejs-0.1.0-beta.4.tgz#80456b2e2cc731982f0d3c6aece80cefa1ebb797"
integrity sha512-XjCp/tG3LRMxMj/MMZfypD5BtW3J1B6oXY2Og8Ed0SyU4uWdglalMwrBUKlDotJr0/Q/2OTspGjD+ytAzCspyw==
dependencies:
https-proxy-agent "^5.0.1"
node-downloader-helper "^2.1.5"
"@matrix-org/matrix-sdk-crypto-nodejs@^0.1.0-beta.1":
version "0.1.0-beta.1"
resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-nodejs/-/matrix-sdk-crypto-nodejs-0.1.0-beta.1.tgz#8a9058226916a258e5b4e28d76680e895b6203b2"
@ -883,13 +891,6 @@
dependencies:
node-downloader-helper "^2.1.1"
"@matrix-org/matrix-sdk-crypto-nodejs@^0.1.0-beta.3":
version "0.1.0-beta.3"
resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-nodejs/-/matrix-sdk-crypto-nodejs-0.1.0-beta.3.tgz#a07225dd180d9d227c24ba62bba439939446d113"
integrity sha512-jHFn6xBeNqfsY5gX60akbss7iFBHZwXycJWMw58Mjz08OwOi7AbTxeS9I2Pa4jX9/M2iinskmGZbzpqOT2fM3A==
dependencies:
node-downloader-helper "^2.1.1"
"@mdn/browser-compat-data@^3.3.14":
version "3.3.14"
resolved "https://registry.yarnpkg.com/@mdn/browser-compat-data/-/browser-compat-data-3.3.14.tgz#b72a37c654e598f9ae6f8335faaee182bebc6b28"
@ -1668,6 +1669,13 @@ acorn@^8.4.1, acorn@^8.6.0, acorn@^8.7.0:
resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.1.tgz#0197122c843d1bf6d0a5e83220a788f278f63c30"
integrity sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A==
agent-base@6:
version "6.0.2"
resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77"
integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==
dependencies:
debug "4"
aggregate-error@^3.0.0, aggregate-error@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a"
@ -2415,6 +2423,13 @@ debug@2.6.9:
dependencies:
ms "2.0.0"
debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4:
version "4.3.4"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865"
integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==
dependencies:
ms "2.1.2"
debug@4.3.1:
version "4.3.1"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.1.tgz#f0d229c505e0c6d8c49ac553d1b13dc183f6b2ee"
@ -2422,13 +2437,6 @@ debug@4.3.1:
dependencies:
ms "2.1.2"
debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4:
version "4.3.4"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865"
integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==
dependencies:
ms "2.1.2"
decamelize@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@ -2571,7 +2579,7 @@ domhandler@^4.0.0, domhandler@^4.2.0:
dependencies:
domelementtype "^2.2.0"
domhandler@^5.0.1, domhandler@^5.0.2:
domhandler@^5.0.1, domhandler@^5.0.2, domhandler@^5.0.3:
version "5.0.3"
resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31"
integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==
@ -2663,6 +2671,11 @@ entities@^4.2.0:
resolved "https://registry.yarnpkg.com/entities/-/entities-4.4.0.tgz#97bdaba170339446495e653cfd2db78962900174"
integrity sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==
entities@^4.4.0:
version "4.5.0"
resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48"
integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==
entities@~2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/entities/-/entities-2.1.0.tgz#992d3129cf7df6870b96c57858c249a120f8b8b5"
@ -3523,6 +3536,16 @@ htmlparser2@^6.0.0, htmlparser2@^6.1.0:
domutils "^2.5.2"
entities "^2.0.0"
htmlparser2@^8.0.0:
version "8.0.2"
resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-8.0.2.tgz#f002151705b383e62433b5cf466f5b716edaec21"
integrity sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==
dependencies:
domelementtype "^2.3.0"
domhandler "^5.0.3"
domutils "^3.0.1"
entities "^4.4.0"
http-errors@1.8.1:
version "1.8.1"
resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c"
@ -3568,6 +3591,14 @@ http-status-codes@^2.2.0:
resolved "https://registry.yarnpkg.com/http-status-codes/-/http-status-codes-2.2.0.tgz#bb2efe63d941dfc2be18e15f703da525169622be"
integrity sha512-feERVo9iWxvnejp3SEfm/+oNG517npqL2/PIA8ORjyOZjGC7TwCRQsZylciLS64i6pJ0wRYz3rkXLRwbtFa8Ng==
https-proxy-agent@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6"
integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==
dependencies:
agent-base "6"
debug "4"
iconv-lite@0.4.24:
version "0.4.24"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
@ -4330,12 +4361,12 @@ matrix-bot-sdk@^0.6.2:
request-promise "^4.2.6"
sanitize-html "^2.7.0"
"matrix-bot-sdk@npm:@vector-im/matrix-bot-sdk@^0.6.3-element.0":
version "0.6.3-element.0"
resolved "https://registry.yarnpkg.com/@vector-im/matrix-bot-sdk/-/matrix-bot-sdk-0.6.3-element.0.tgz#dfd36d1073145ce95b609b3f78f66a50a9c520e4"
integrity sha512-PuH4vSP2aE0TkJUqLtPsfqU0oEAlW9zmYOd8b5hY8RSkpfitup4lCwTYipWovJIKeoolNv5uMPA2mzqM3mcP6w==
"matrix-bot-sdk@npm:@vector-im/matrix-bot-sdk@^0.6.6-element.0":
version "0.6.6-element.0"
resolved "https://registry.yarnpkg.com/@vector-im/matrix-bot-sdk/-/matrix-bot-sdk-0.6.6-element.0.tgz#96e08bb434939fcbf7f771a47aa4a7d17632f601"
integrity sha512-Mx5SAv1zFXpkG+jbtbkadNMLjaxftnWBGCjKlNFlUDjcJPUIYxKd49jUGst/vdp/ysZYa0Dgk+V+E1qmKUa8bw==
dependencies:
"@matrix-org/matrix-sdk-crypto-nodejs" "^0.1.0-beta.3"
"@matrix-org/matrix-sdk-crypto-nodejs" "0.1.0-beta.4"
"@types/express" "^4.17.13"
another-json "^0.2.0"
async-lock "^1.3.2"
@ -4351,7 +4382,7 @@ matrix-bot-sdk@^0.6.2:
morgan "^1.10.0"
request "^2.88.2"
request-promise "^4.2.6"
sanitize-html "^2.7.0"
sanitize-html "^2.8.0"
matrix-widget-api@^1.0.0:
version "1.0.0"
@ -4592,6 +4623,11 @@ node-downloader-helper@^2.1.1:
resolved "https://registry.yarnpkg.com/node-downloader-helper/-/node-downloader-helper-2.1.1.tgz#533427a3cdc163931b106d0fe6d522f83deac7ab"
integrity sha512-ouk8MGmJj1gYymbJwi1L8Mr6PdyheJLwfsmyx0KtsvyJ+7Fpf0kBBzM8Gmx8Mt/JBfRWP1PQm6dAGV6x7eNedw==
node-downloader-helper@^2.1.5:
version "2.1.6"
resolved "https://registry.yarnpkg.com/node-downloader-helper/-/node-downloader-helper-2.1.6.tgz#f73ac458e3ac8c21afd0b952a994eab99c64b879"
integrity sha512-VkOvAXIopI3xMuM/MC5UL7NqqnizQ/9QXZt28jR8FPZ6fHLQm4xe4+YXJ9FqsWwLho5BLXrF51nfOQ0QcohRkQ==
node-emoji@^1.11.0:
version "1.11.0"
resolved "https://registry.yarnpkg.com/node-emoji/-/node-emoji-1.11.0.tgz#69a0150e6946e2f115e9d7ea4df7971e2628301c"
@ -5380,6 +5416,18 @@ sanitize-html@^2.7.0:
parse-srcset "^1.0.2"
postcss "^8.3.11"
sanitize-html@^2.8.0:
version "2.10.0"
resolved "https://registry.yarnpkg.com/sanitize-html/-/sanitize-html-2.10.0.tgz#74d28848dfcf72c39693139131895c78900ab452"
integrity sha512-JqdovUd81dG4k87vZt6uA6YhDfWkUGruUu/aPmXLxXi45gZExnt9Bnw/qeQU8oGf82vPyaE0vO4aH0PbobB9JQ==
dependencies:
deepmerge "^4.2.2"
escape-string-regexp "^4.0.0"
htmlparser2 "^8.0.0"
is-plain-object "^5.0.0"
parse-srcset "^1.0.2"
postcss "^8.3.11"
sass@^1.51.0:
version "1.51.0"
resolved "https://registry.yarnpkg.com/sass/-/sass-1.51.0.tgz#25ea36cf819581fe1fe8329e8c3a4eaaf70d2845"