diff --git a/changelog.d/902.removal b/changelog.d/902.removal new file mode 100644 index 00000000..05f7d124 --- /dev/null +++ b/changelog.d/902.removal @@ -0,0 +1,3 @@ +The cache/queue configuration has been changed in this release. The `queue.monolithic` option has been deprecated, in place of a dedicated `cache` +config section. Check the ["Cache configuration" section](https://matrix-org.github.io/matrix-hookshot/latest/setup.html#cache-configuration) for +more information on how to configure Hookshot caches. \ No newline at end of file diff --git a/config.sample.yml b/config.sample.yml index 7182a9bc..9c3aa289 100644 --- a/config.sample.yml +++ b/config.sample.yml @@ -134,12 +134,15 @@ listeners: # # (Optional) Prometheus metrics support # enabled: true +#cache: +# # (Optional) Cache options for large scale deployments. +# # For encryption to work, this must be configured. +# redisUri: redis://localhost:6379 + #queue: -# # (Optional) Message queue / cache configuration options for large scale deployments. -# # For encryption to work, must be set to monolithic mode and have a host & port specified. -# monolithic: true -# port: 6379 -# host: localhost +# # (Optional) Message queue configuration options for large scale deployments. +# # For encryption to work, this must not be configured. +# redisUri: redis://localhost:6379 #widgets: # # (Optional) EXPERIMENTAL support for complimentary widgets diff --git a/docs/advanced/encryption.md b/docs/advanced/encryption.md index 21c334cc..4133f6be 100644 --- a/docs/advanced/encryption.md +++ b/docs/advanced/encryption.md @@ -13,7 +13,7 @@ Hookshot supports end-to-bridge encryption via [MSC3202](https://github.com/matr In order for Hookshot to use encryption, it must be configured as follows: - The `experimentalEncryption.storagePath` setting must point to a directory that Hookshot has permissions to write files into. If running with Docker, this path should be within a volume (for persistency). Hookshot uses this directory for its crypto store (i.e. long-lived state relating to its encryption keys). - Once a crypto store has been initialized, its files must not be modified, and Hookshot cannot be configured to use another crypto store of the same type as one it has used before. If a crypto store's files get lost or corrupted, Hookshot may fail to start up, or may be unable to decrypt command messages. To fix such issues, stop Hookshot, then reset its crypto store by running `yarn start:resetcrypto`. -- [Redis](./workers.md) must be enabled. Note that worker mode is not yet supported with encryption, so `queue.monolithic` must be set to `true`. +- [Redis](./workers.md) must be enabled. Note that worker mode is not yet supported with encryption, so `queue` MUST **NOT be configured**. If you ever reset your homeserver's state, ensure you also reset Hookshot's encryption state. This includes clearing the `experimentalEncryption.storagePath` directory and all worker state stored in your redis instance. Otherwise, Hookshot may fail on start up with registration errors. diff --git a/docs/advanced/workers.md b/docs/advanced/workers.md index c4b59e07..1decc97c 100644 --- a/docs/advanced/workers.md +++ b/docs/advanced/workers.md @@ -11,18 +11,19 @@ This feature is experimental and should only be used when you are reachin You must first have a working Redis instance somewhere which can talk between processes. For example, in Docker you can run: -`docker run --name github-bridge-redis -p 6379:6379 -d redis`. +`docker run --name redis-host -p 6379:6379 -d redis`. The processes should all share the same config, which should contain the correct config to enable Redis: ```yaml queue: - monolithic: false - port: 6379 - host: github-bridge-redis + redisUri: "redis://redis-host:6379" +cache: + redisUri: "redis://redis-host:6379" ``` -Note that if [encryption](./encryption.md) is enabled, `queue.monolithic` must be set to `true`, as worker mode is not yet supported with encryption. +Note that if [encryption](./encryption.md) is enabled, you MUST enable the `cache` config but NOT the `queue` config. Workers require persistent +storage in Redis, but cannot make use of worker-mode queues. Once that is done, you can simply start the processes by name using yarn: ``` diff --git a/docs/setup.md b/docs/setup.md index caf71662..004c2f71 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -227,6 +227,20 @@ Please note that the appservice HTTP listener is configured separatelythis issue for details. +### Cache configuration + +You can optionally enable a Redis-backed cache for Hookshot. This is generally a good thing to enable if you can +afford to, as it will generally improve startup times. Some features such as resuming RSS/Atom feeds between restarts +is also only possible with a external cache. + +To enable, simply set: + +```yaml +cache: + redisUri: "redis://redis-host:3679" +``` + + ### Services configuration You will need to configure some services. Each service has its own documentation file inside the setup subdirectory. diff --git a/helm/hookshot/values.yaml b/helm/hookshot/values.yaml index a4b1302b..6a8e0c21 100644 --- a/helm/hookshot/values.yaml +++ b/helm/hookshot/values.yaml @@ -3,7 +3,6 @@ # # -- Number of replicas to deploy. Consequences of using multiple Hookshot replicas currently unknown. replicaCount: 1 - image: # -- Repository to pull hookshot image from repository: halfshot/matrix-hookshot @@ -11,16 +10,12 @@ image: pullPolicy: IfNotPresent # -- Image tag to pull. Defaults to chart's appVersion value as set in Chart.yaml tag: - # -- List of names of k8s secrets to be used as ImagePullSecrets for the pod imagePullSecrets: [] - # -- Name override for helm chart nameOverride: "" - # -- Full name override for helm chart fullnameOverride: "" - serviceAccount: # -- Specifies whether a service account should be created create: true @@ -28,10 +23,8 @@ serviceAccount: annotations: {} # -- The name of the service account to use. If not set and create is true, a name is generated using the fullname template name: "" - # -- Extra annotations for Hookshot pod podAnnotations: {} - # -- Pod security context settings podSecurityContext: {} # fsGroup: 2000 @@ -54,7 +47,6 @@ service: annotations: {} # -- Extra labels for service labels: {} - webhook: # -- Webhook port as configured in container port: 9000 @@ -64,7 +56,6 @@ service: appservice: # -- Appservice port as configured in container port: 9002 - ingress: webhook: # -- Enable ingress for webhook @@ -77,7 +68,6 @@ ingress: hosts: [] # -- TLS configuration for webhook ingress tls: [] - appservice: # -- Enable ingress for appservice enabled: false @@ -89,7 +79,6 @@ ingress: hosts: [] # -- TLS configuration for appservice ingress tls: [] - # -- Pod resource requests / limits resources: {} # We usually recommend not to specify default resources and to leave this as a conscious @@ -105,189 +94,40 @@ resources: {} autoscaling: enabled: false - # -- Node selector parameters nodeSelector: {} # -- Tolerations for deployment tolerations: [] - # -- Affinity settings for deployment affinity: {} - hookshot: # -- Name of existing ConfigMap with valid Hookshot configuration existingConfigMap: - # -- Raw Hookshot configuration. Gets templated into a YAML file and then loaded unless an existingConfigMap is specified. config: + # This is an example configuration file bridge: # Basic homeserver configuration - # domain: example.com url: http://localhost:8008 mediaUrl: https://example.com port: 9993 bindAddress: 127.0.0.1 - # github: - # (Optional) Configure this to enable GitHub support - # - # auth: - # Authentication for the GitHub App. - # - # id: 123 - # privateKeyFile: github-key.pem - # webhook: - # Webhook settings for the GitHub app. - # - # secret: secrettoken - # oauth: - # (Optional) Settings for allowing users to sign in via OAuth. - # - # client_id: foo - # client_secret: bar - # redirect_uri: https://example.com/bridge_oauth/ - # defaultOptions: - # (Optional) Default options for GitHub connections. - # - # showIssueRoomLink: false - # hotlinkIssues: - # prefix: "#" - # userIdPrefix: _github_ - # (Optional) Prefix used when creating ghost users for GitHub accounts. - # - # gitlab: - # (Optional) Configure this to enable GitLab support - # - # instances: - # gitlab.com: - # url: https://gitlab.com - # webhook: - # secret: secrettoken - # publicUrl: https://example.com/hookshot/ - # userIdPrefix: _gitlab_ - # (Optional) Prefix used when creating ghost users for GitLab accounts. - # - # figma: - # (Optional) Configure this to enable Figma support - # - # publicUrl: https://example.com/hookshot/ - # instances: - # your-instance: - # teamId: your-team-id - # accessToken: your-personal-access-token - # passcode: your-webhook-passcode - # jira: - # (Optional) Configure this to enable Jira support. Only specify `url` if you are using a On Premise install (i.e. not atlassian.com) - # - # webhook: - # Webhook settings for JIRA - # - # secret: secrettoken - # oauth: - # (Optional) OAuth settings for connecting users to JIRA. See documentation for more information - # - # client_id: foo - # client_secret: bar - # redirect_uri: https://example.com/bridge_oauth/ - generic: - # (Optional) Support for generic webhook events. - #'allowJsTransformationFunctions' will allow users to write short transformation snippets in code, and thus is unsafe in untrusted environments - # - # - enabled: false - enableHttpGet: false - urlPrefix: https://example.com/webhook/ - userIdPrefix: _webhooks_ - allowJsTransformationFunctions: false - waitForComplete: false - feeds: - # (Optional) Configure this to enable RSS/Atom feed support - # - enabled: false - pollIntervalSeconds: 600 - pollTimeoutSeconds: 30 - # provisioning: - # (Optional) Provisioning API for integration managers - # - # secret: "!secretToken" passFile: passkey.pem # A passkey used to encrypt tokens stored inside the bridge. # Run openssl genpkey -out passkey.pem -outform PEM -algorithm RSA -pkeyopt rsa_keygen_bits:4096 to generate - # - # bot: - # (Optional) Define profile information for the bot user - # - # displayname: Hookshot Bot - # avatar: mxc://half-shot.uk/2876e89ccade4cb615e210c458e2a7a6883fe17d - # serviceBots: - # (Optional) Define additional bot users for specific services - # - # - localpart: feeds - # displayname: Feeds - # avatar: mxc://half-shot.uk/2876e89ccade4cb615e210c458e2a7a6883fe17d - # prefix: "!feeds" - # service: feeds - metrics: - # (Optional) Prometheus metrics support - # - enabled: true - # queue: - # (Optional) Message queue / cache configuration options for large scale deployments. - # For encryption to work, must be set to monolithic mode and have a host & port specified. - # - # monolithic: true - # port: 6379 - # host: localhost logging: - # (Optional) Logging settings. You can have a severity debug,info,warn,error - # + # Logging settings. You can have a severity debug,info,warn,error level: info colorize: true json: false timestampFormat: HH:mm:ss:SSS - # widgets: - # (Optional) EXPERIMENTAL support for complimentary widgets - # - # addToAdminRooms: false - # disallowedIpRanges: - # - 127.0.0.0/8 - # - 10.0.0.0/8 - # - 172.16.0.0/12 - # - 192.168.0.0/16 - # - 100.64.0.0/10 - # - 192.0.0.0/24 - # - 169.254.0.0/16 - # - 192.88.99.0/24 - # - 198.18.0.0/15 - # - 192.0.2.0/24 - # - 198.51.100.0/24 - # - 203.0.113.0/24 - # - 224.0.0.0/4 - # - ::1/128 - # - fe80::/10 - # - fc00::/7 - # - 2001:db8::/32 - # - ff00::/8 - # - fec0::/10 - # roomSetupWidget: - # addOnInvite: false - # publicUrl: https://example.com/widgetapi/v1/static/ - # branding: - # widgetTitle: Hookshot Configuration - # permissions: - # (Optional) Permissions for using the bridge. See docs/setup.md#permissions for help - # - # - actor: example.com - # services: - # - service: "*" - # level: admin listeners: - # (Optional) HTTP Listener configuration. + # HTTP Listener configuration. # Bind resource endpoints to ports and addresses. # 'port' must be specified. Each listener must listen on a unique port. # 'bindAddress' will default to '127.0.0.1' if not specified, which may not be suited to Docker environments. # 'resources' may be any of webhooks, widgets, metrics, provisioning - # - port: 9000 bindAddress: 0.0.0.0 resources: @@ -302,6 +142,153 @@ hookshot: resources: - widgets registration: + #github: + # # (Optional) Configure this to enable GitHub support + # auth: + # # Authentication for the GitHub App. + # id: 123 + # privateKeyFile: github-key.pem + # webhook: + # # Webhook settings for the GitHub app. + # secret: secrettoken + # oauth: + # # (Optional) Settings for allowing users to sign in via OAuth. + # client_id: foo + # client_secret: bar + # redirect_uri: https://example.com/oauth/ + # defaultOptions: + # # (Optional) Default options for GitHub connections. + # showIssueRoomLink: false + # hotlinkIssues: + # prefix: "#" + # userIdPrefix: + # # (Optional) Prefix used when creating ghost users for GitHub accounts. + # _github_ + + #gitlab: + # # (Optional) Configure this to enable GitLab support + # instances: + # gitlab.com: + # url: https://gitlab.com + # webhook: + # secret: secrettoken + # publicUrl: https://example.com/hookshot/ + # userIdPrefix: + # # (Optional) Prefix used when creating ghost users for GitLab accounts. + # _gitlab_ + # commentDebounceMs: + # # (Optional) Aggregate comments by waiting this many miliseconds before posting them to Matrix. Defaults to 5000 (5 seconds) + # 5000 + + #figma: + # # (Optional) Configure this to enable Figma support + # publicUrl: https://example.com/hookshot/ + # instances: + # your-instance: + # teamId: your-team-id + # accessToken: your-personal-access-token + # passcode: your-webhook-passcode + + #jira: + # # (Optional) Configure this to enable Jira support. Only specify `url` if you are using a On Premise install (i.e. not atlassian.com) + # webhook: + # # Webhook settings for JIRA + # secret: secrettoken + # oauth: + # # (Optional) OAuth settings for connecting users to JIRA. See documentation for more information + # client_id: foo + # client_secret: bar + # redirect_uri: https://example.com/oauth/ + + #generic: + # # (Optional) Support for generic webhook events. + # #'allowJsTransformationFunctions' will allow users to write short transformation snippets in code, and thus is unsafe in untrusted environments + + # enabled: false + # enableHttpGet: false + # urlPrefix: https://example.com/webhook/ + # userIdPrefix: _webhooks_ + # allowJsTransformationFunctions: false + # waitForComplete: false + + #feeds: + # # (Optional) Configure this to enable RSS/Atom feed support + # enabled: false + # pollConcurrency: 4 + # pollIntervalSeconds: 600 + # pollTimeoutSeconds: 30 + + #provisioning: + # # (Optional) Provisioning API for integration managers + # secret: "!secretToken" + + #bot: + # # (Optional) Define profile information for the bot user + # displayname: Hookshot Bot + # avatar: mxc://half-shot.uk/2876e89ccade4cb615e210c458e2a7a6883fe17d + + #serviceBots: + # # (Optional) Define additional bot users for specific services + # - localpart: feeds + # displayname: Feeds + # avatar: ./assets/feeds_avatar.png + # prefix: "!feeds" + # service: feeds + + #metrics: + # # (Optional) Prometheus metrics support + # enabled: true + + #cache: + # # (Optional) Cache options for large scale deployments. + # # For encryption to work, this must be configured. + # redisUri: redis://localhost:6379 + + #queue: + # # (Optional) Message queue configuration options for large scale deployments. + # # For encryption to work, this must not be configured. + # redisUri: redis://localhost:6379 + + #widgets: + # # (Optional) EXPERIMENTAL support for complimentary widgets + # addToAdminRooms: false + # disallowedIpRanges: + # - 127.0.0.0/8 + # - 10.0.0.0/8 + # - 172.16.0.0/12 + # - 192.168.0.0/16 + # - 100.64.0.0/10 + # - 192.0.0.0/24 + # - 169.254.0.0/16 + # - 192.88.99.0/24 + # - 198.18.0.0/15 + # - 192.0.2.0/24 + # - 198.51.100.0/24 + # - 203.0.113.0/24 + # - 224.0.0.0/4 + # - ::1/128 + # - fe80::/10 + # - fc00::/7 + # - 2001:db8::/32 + # - ff00::/8 + # - fec0::/10 + # roomSetupWidget: + # addOnInvite: false + # publicUrl: https://example.com/widgetapi/v1/static/ + # branding: + # widgetTitle: Hookshot Configuration + + #sentry: + # # (Optional) Configure Sentry error reporting + # dsn: https://examplePublicKey@o0.ingest.sentry.io/0 + # environment: production + + #permissions: + # # (Optional) Permissions for using the bridge. See docs/setup.md#permissions for help + # - actor: example.com + # services: + # - service: "*" + # level: admin id: matrix-hookshot as_token: "" hs_token: "" diff --git a/spec/util/e2e-test.ts b/spec/util/e2e-test.ts index 20cf38c2..28631077 100644 --- a/spec/util/e2e-test.ts +++ b/spec/util/e2e-test.ts @@ -203,9 +203,6 @@ export class E2ETestEnv { logging: { level: 'info', }, - queue: { - monolithic: true, - }, // Always enable webhooks so that hookshot starts. generic: { enabled: true, diff --git a/src/App/BridgeApp.ts b/src/App/BridgeApp.ts index a21049e4..1c40b23e 100644 --- a/src/App/BridgeApp.ts +++ b/src/App/BridgeApp.ts @@ -1,6 +1,5 @@ import { Bridge } from "../Bridge"; import { BridgeConfig, parseRegistrationFile } from "../config/Config"; -import { Webhooks } from "../Webhooks"; import { MatrixSender } from "../MatrixSender"; import { UserNotificationWatcher } from "../Notifications/UserNotificationWatcher"; import { ListenerService } from "../ListenerService"; @@ -27,7 +26,7 @@ export async function start(config: BridgeConfig, registration: IAppserviceRegis const {appservice, storage} = getAppservice(config, registration); - if (config.queue.monolithic) { + if (!config.queue) { const matrixSender = new MatrixSender(config, appservice); matrixSender.listen(); const userNotificationWatcher = new UserNotificationWatcher(config); diff --git a/src/MessageQueue/MessageQueue.ts b/src/MessageQueue/MessageQueue.ts index 476e6d86..03dc1d17 100644 --- a/src/MessageQueue/MessageQueue.ts +++ b/src/MessageQueue/MessageQueue.ts @@ -1,4 +1,4 @@ -import { BridgeConfigQueue } from "../config/Config"; +import { BridgeConfigQueue } from "../config/sections"; import { LocalMQ } from "./LocalMQ"; import { RedisMQ } from "./RedisQueue"; import { MessageQueue } from "./Types"; @@ -6,8 +6,8 @@ import { MessageQueue } from "./Types"; const staticLocalMq = new LocalMQ(); let staticRedisMq: RedisMQ|null = null; -export function createMessageQueue(config: BridgeConfigQueue): MessageQueue { - if (config.monolithic) { +export function createMessageQueue(config?: BridgeConfigQueue): MessageQueue { + if (!config) { return staticLocalMq; } if (staticRedisMq === null) { diff --git a/src/MessageQueue/RedisQueue.ts b/src/MessageQueue/RedisQueue.ts index 2fbda2f3..0800da89 100644 --- a/src/MessageQueue/RedisQueue.ts +++ b/src/MessageQueue/RedisQueue.ts @@ -1,7 +1,7 @@ import { MessageQueue, MessageQueueMessage, DEFAULT_RES_TIMEOUT, MessageQueueMessageOut } from "./Types"; import { Redis, default as redis } from "ioredis"; -import { BridgeConfigQueue } from "../config/Config"; +import { BridgeConfigQueue } from "../config/sections/queue"; import { EventEmitter } from "events"; import { Logger } from "matrix-appservice-bridge"; import { randomUUID } from 'node:crypto'; @@ -22,9 +22,10 @@ export class RedisMQ extends EventEmitter implements MessageQueue { private myUuid: string; constructor(config: BridgeConfigQueue) { super(); - this.redisSub = new redis(config.port ?? 6379, config.host ?? "localhost"); - this.redisPub = new redis(config.port ?? 6379, config.host ?? "localhost"); - this.redis = new redis(config.port ?? 6379, config.host ?? "localhost"); + const uri = 'redisUri' in config ? config.redisUri : `redis://${config.host ?? 'localhost'}:${config.port ?? 6379}`; + this.redisSub = new redis(uri); + this.redisPub = new redis(uri); + this.redis = new redis(uri); this.myUuid = randomUUID(); this.redisSub.on("pmessage", (_: string, channel: string, message: string) => { const msg = JSON.parse(message) as MessageQueueMessageOut; diff --git a/src/Stores/RedisStorageProvider.ts b/src/Stores/RedisStorageProvider.ts index 12c42a79..49fac70f 100644 --- a/src/Stores/RedisStorageProvider.ts +++ b/src/Stores/RedisStorageProvider.ts @@ -6,6 +6,7 @@ import { IBridgeStorageProvider, MAX_FEED_ITEMS } from "./StorageProvider"; import { IFilterInfo, IStorageProvider } from "matrix-bot-sdk"; import { ProvisionSession } from "matrix-appservice-bridge"; import { SerializedGitlabDiscussionThreads } from "../Gitlab/Types"; +import { BridgeConfigCache } from "../config/sections"; const BOT_SYNC_TOKEN_KEY = "bot.sync_token."; const BOT_FILTER_KEY = "bot.filter."; @@ -68,8 +69,8 @@ export class RedisStorageContextualProvider implements IStorageProvider { export class RedisStorageProvider extends RedisStorageContextualProvider implements IBridgeStorageProvider { - constructor(host: string, port: number, contextSuffix = '') { - super(new redis(port, host), contextSuffix); + constructor(cacheConfig: BridgeConfigCache, contextSuffix = '') { + super(new redis(cacheConfig.redisUri), contextSuffix); this.redis.expire(COMPLETED_TRANSACTIONS_KEY, COMPLETED_TRANSACTIONS_EXPIRE_AFTER).catch((ex) => { log.warn("Failed to set expiry time on as.completed_transactions", ex); }); diff --git a/src/appservice.ts b/src/appservice.ts index 2bd97742..9fbd5f34 100644 --- a/src/appservice.ts +++ b/src/appservice.ts @@ -9,9 +9,9 @@ const log = new Logger("Appservice"); export function getAppservice(config: BridgeConfig, registration: IAppserviceRegistration) { let storage: IBridgeStorageProvider; - if (config.queue.host && config.queue.port) { - log.info(`Initialising Redis storage (on ${config.queue.host}:${config.queue.port})`); - storage = new RedisStorageProvider(config.queue.host, config.queue.port); + if (config.cache) { + log.info(`Initialising Redis storage`); + storage = new RedisStorageProvider(config.cache); } else { log.info('Initialising memory storage'); storage = new MemoryStorageProvider(); diff --git a/src/config/Config.ts b/src/config/Config.ts index 374c295a..beb73857 100644 --- a/src/config/Config.ts +++ b/src/config/Config.ts @@ -10,6 +10,8 @@ import { ConfigError } from "../errors"; import { ApiError, ErrCode } from "../api"; import { GithubInstance, GITHUB_CLOUD_URL } from "../github/GithubInstance"; import { Logger } from "matrix-appservice-bridge"; +import { BridgeConfigCache } from "./sections/cache"; +import { BridgeConfigQueue } from "./sections"; const log = new Logger("Config"); @@ -407,12 +409,6 @@ interface BridgeConfigWebhook { bindAddress?: string; } -export interface BridgeConfigQueue { - monolithic: boolean; - port?: number; - host?: string; -} - export interface BridgeConfigLogging { level: "debug"|"info"|"warn"|"error"|"trace"; json?: boolean; @@ -454,40 +450,45 @@ export interface BridgeConfigSentry { environment?: string; } + export interface BridgeConfigRoot { bot?: BridgeConfigBot; - serviceBots?: BridgeConfigServiceBot[]; bridge: BridgeConfigBridge; + cache?: BridgeConfigCache; experimentalEncryption?: BridgeConfigEncryption; - figma?: BridgeConfigFigma; feeds?: BridgeConfigFeedsYAML; + figma?: BridgeConfigFigma; generic?: BridgeGenericWebhooksConfigYAML; github?: BridgeConfigGitHubYAML; gitlab?: BridgeConfigGitLabYAML; + jira?: BridgeConfigJiraYAML; + listeners?: BridgeConfigListener[]; + logging: BridgeConfigLogging; + metrics?: BridgeConfigMetrics; + passFile: string; permissions?: BridgeConfigActorPermission[]; provisioning?: BridgeConfigProvisioning; - jira?: BridgeConfigJiraYAML; - logging: BridgeConfigLogging; - passFile: string; - queue: BridgeConfigQueue; + queue?: BridgeConfigQueue; + sentry?: BridgeConfigSentry; + serviceBots?: BridgeConfigServiceBot[]; webhook?: BridgeConfigWebhook; widgets?: BridgeWidgetConfigYAML; - metrics?: BridgeConfigMetrics; - listeners?: BridgeConfigListener[]; - sentry?: BridgeConfigSentry; } export class BridgeConfig { @configKey("Basic homeserver configuration") public readonly bridge: BridgeConfigBridge; + @configKey(`Cache options for large scale deployments. + For encryption to work, this must be configured.`, true) + public readonly cache?: BridgeConfigCache; @configKey(`Configuration for encryption support in the bridge. If omitted, encryption support will be disabled. This feature is HIGHLY EXPERIMENTAL AND SUBJECT TO CHANGE. For more details, see https://github.com/matrix-org/matrix-hookshot/issues/594.`, true) public readonly encryption?: BridgeConfigEncryption; - @configKey(`Message queue / cache configuration options for large scale deployments. - For encryption to work, must be set to monolithic mode and have a host & port specified.`, true) - public readonly queue: BridgeConfigQueue; + @configKey(`Message queue configuration options for large scale deployments. + For encryption to work, this must not be configured.`, true) + public readonly queue?: Omit; @configKey("Logging settings. You can have a severity debug,info,warn,error") public readonly logging: BridgeConfigLogging; @configKey(`Permissions for using the bridge. See docs/setup.md#permissions for help`, true) @@ -553,9 +554,34 @@ export class BridgeConfig { this.bot = configData.bot; this.serviceBots = configData.serviceBots; this.metrics = configData.metrics; - this.queue = configData.queue || { - monolithic: true, - }; + + // TODO: Formalize env support + if (env?.CFG_QUEUE_MONOLITHIC && ["false", "off", "no"].includes(env.CFG_QUEUE_MONOLITHIC)) { + if (!env?.CFG_QUEUE_HOST) { + throw new ConfigError("env:CFG_QUEUE_HOST", "CFG_QUEUE_MONOLITHIC was defined but host was not"); + } + configData.queue = { + monolithic: false, + host: env?.CFG_QUEUE_HOST, + port: env?.CFG_QUEUE_POST ? parseInt(env?.CFG_QUEUE_POST, 10) : undefined, + } + } + + this.cache = configData.cache; + this.queue = configData.queue; + + if (configData.queue?.monolithic !== undefined) { + log.warn("The `queue.monolithic` config option is deprecated. Instead, configure the `cache` section."); + this.cache = { + redisUri: 'redisUri' in configData.queue ? configData.queue.redisUri + : `redis://${configData.queue.host ?? 'localhost'}:${configData.queue.port ?? 6379}` + }; + // If monolithic, disable the redis queue. + if (configData.queue.monolithic === true) { + this.queue = undefined; + } + } + this.encryption = configData.experimentalEncryption; @@ -589,13 +615,6 @@ export class BridgeConfig { throw Error("Config is not valid: At least one of GitHub, GitLab, JIRA, Figma, feeds or generic hooks must be configured"); } - // TODO: Formalize env support - if (env?.CFG_QUEUE_MONOLITHIC && ["false", "off", "no"].includes(env.CFG_QUEUE_MONOLITHIC)) { - this.queue.monolithic = false; - this.queue.host = env?.CFG_QUEUE_HOST; - this.queue.port = env?.CFG_QUEUE_POST ? parseInt(env?.CFG_QUEUE_POST, 10) : undefined; - } - if ('goNebMigrator' in configData) { log.warn(`The GoNEB migrator has been removed from this release. You should remove the 'goNebMigrator' from your config.`); } @@ -677,12 +696,12 @@ Please back up your crypto store at ${this.encryption.storagePath}, remove "useLegacySledStore" from your configuration file, and restart Hookshot. `); } - if (!this.queue.monolithic) { - throw new ConfigError("queue.monolithic", "Encryption is not supported in worker mode yet."); + if (!this.cache) { + throw new ConfigError("cache", "Encryption requires the Redis cache to be enabled."); } - if (!this.queue.port) { - throw new ConfigError("queue.port", "You must enable redis support for encryption to work."); + if (this.queue) { + throw new ConfigError("queue", "Encryption does not support message queues."); } } diff --git a/src/config/Defaults.ts b/src/config/Defaults.ts index 70522e8e..190f47b7 100644 --- a/src/config/Defaults.ts +++ b/src/config/Defaults.ts @@ -16,9 +16,10 @@ export const DefaultConfigRoot: BridgeConfigRoot = { bindAddress: "127.0.0.1", }, queue: { - monolithic: true, - port: 6379, - host: "localhost", + redisUri: "redis://localhost:6379", + }, + cache: { + redisUri: "redis://localhost:6379", }, logging: { level: "info", diff --git a/src/config/sections/cache.ts b/src/config/sections/cache.ts new file mode 100644 index 00000000..2da69c73 --- /dev/null +++ b/src/config/sections/cache.ts @@ -0,0 +1,7 @@ +export interface BridgeConfigCache { + /** + * A redis URI string + * @example `redis://user:password@host:port/dbnum` + */ + redisUri: string; +} \ No newline at end of file diff --git a/src/config/sections/index.ts b/src/config/sections/index.ts new file mode 100644 index 00000000..e4fb0283 --- /dev/null +++ b/src/config/sections/index.ts @@ -0,0 +1,2 @@ +export * from "./cache"; +export * from "./queue"; \ No newline at end of file diff --git a/src/config/sections/queue.ts b/src/config/sections/queue.ts new file mode 100644 index 00000000..605450e7 --- /dev/null +++ b/src/config/sections/queue.ts @@ -0,0 +1,22 @@ +/** + * Configuration for the message queue. + */ +interface BridgeConfigQueueBase { + /** + * Controls whether the queue config is used just for the cache (monolithic), + * or the message queue as well. + * @deprecated Use the `cache` config instead to control this seperately. + */ + monolithic?: boolean; +} + +interface BridgeConfigQueueUri extends BridgeConfigQueueBase { + redisUri: string; +} + +interface BridgeConfigQueueLegacyOptions extends BridgeConfigQueueBase { + port?: number; + host?: string; +} + +export type BridgeConfigQueue = BridgeConfigQueueUri|BridgeConfigQueueLegacyOptions \ No newline at end of file diff --git a/src/format_util.rs b/src/format_util.rs index 2146381c..dc11b7ff 100644 --- a/src/format_util.rs +++ b/src/format_util.rs @@ -1,7 +1,7 @@ use crate::github::types::*; use crate::jira; use crate::jira::types::{JiraIssue, JiraIssueLight, JiraIssueMessageBody, JiraIssueSimpleItem}; -use contrast; +use contrast::contrast; use md5::{Digest, Md5}; use napi::bindgen_prelude::*; use napi_derive::napi; @@ -87,12 +87,11 @@ pub fn format_labels(array: Vec) -> Result(color_rgb, RGB::new(0, 0, 0)) > 4.5 { - "#000000" - } else { - "#FFFFFF" - }; + let contrast_color = if contrast::(color_rgb, RGB::new(0, 0, 0)) > 4.5 { + "#000000" + } else { + "#FFFFFF" + }; write!(html, " data-mx-color=\"{}\"", contrast_color).unwrap(); } if let Some(description) = label.description { diff --git a/tests/MessageQueueTest.ts b/tests/MessageQueueTest.ts index a3fa0d63..f2cd6e45 100644 --- a/tests/MessageQueueTest.ts +++ b/tests/MessageQueueTest.ts @@ -1,9 +1,7 @@ import { expect } from "chai"; import { createMessageQueue } from "../src/MessageQueue/MessageQueue"; -const mq = createMessageQueue({ - monolithic: true, -}); +const mq = createMessageQueue(); describe("MessageQueueTest", () => { describe("LocalMq", () => { diff --git a/tests/config/config.ts b/tests/config/config.ts new file mode 100644 index 00000000..22a1a752 --- /dev/null +++ b/tests/config/config.ts @@ -0,0 +1,76 @@ +import { BridgeConfig } from "../../src/config/Config"; +import { DefaultConfigRoot } from "../../src/config/Defaults"; +import { expect } from "chai"; + + +describe("Config/BridgeConfig", () => { + describe("will handle the legacy queue.monolitihc option", () => { + it("with no parameters", () => { + const config = new BridgeConfig({ ...DefaultConfigRoot, queue: { + monolithic: true + }}); + expect(config.queue).to.be.undefined; + expect(config.cache?.redisUri).to.equal("redis://localhost:6379"); + }); + it("with a host parameter", () => { + const config = new BridgeConfig({ ...DefaultConfigRoot, queue: { + monolithic: true, + host: 'bark' + }}); + expect(config.queue).to.be.undefined; + expect(config.cache?.redisUri).to.equal("redis://bark:6379"); + }); + it("with a port parameter", () => { + const config = new BridgeConfig({ ...DefaultConfigRoot, queue: { + monolithic: true, + port: 6379, + }}); + expect(config.queue).to.be.undefined; + expect(config.cache?.redisUri).to.equal("redis://localhost:6379"); + }); + it("with a host and port parameter", () => { + const config = new BridgeConfig({ ...DefaultConfigRoot, queue: { + monolithic: true, + host: 'bark', + port: 6379, + }}); + expect(config.queue).to.be.undefined; + expect(config.cache?.redisUri).to.equal("redis://bark:6379"); + }); + it("with monolithic disabled", () => { + const config = new BridgeConfig({ ...DefaultConfigRoot, queue: { + monolithic: false + }}); + expect(config.queue).to.deep.equal({ + monolithic: false, + }); + expect(config.cache?.redisUri).to.equal("redis://localhost:6379"); + }); + }); + describe("will handle the queue option", () => { + it("with redisUri", () => { + const config = new BridgeConfig({ ...DefaultConfigRoot, queue: { + redisUri: "redis://localhost:6379" + }, cache: undefined}); + expect(config.queue).to.deep.equal({ + redisUri: "redis://localhost:6379" + }); + expect(config.cache).to.be.undefined; + }); + }); + describe("will handle the cache option", () => { + it("with redisUri", () => { + const config = new BridgeConfig({ + ...DefaultConfigRoot, + cache: { + redisUri: "redis://localhost:6379" + }, + queue: undefined, + }); + expect(config.cache).to.deep.equal({ + redisUri: "redis://localhost:6379" + }); + expect(config.queue).to.be.undefined; + }); + }); +}) \ No newline at end of file diff --git a/tests/connections/GithubRepoTest.ts b/tests/connections/GithubRepoTest.ts index e686f2a5..8c1728d4 100644 --- a/tests/connections/GithubRepoTest.ts +++ b/tests/connections/GithubRepoTest.ts @@ -37,9 +37,7 @@ const GITHUB_ISSUE_CREATED_PAYLOAD = { }; function createConnection(state: Record = {}, isExistingState=false) { - const mq = createMessageQueue({ - monolithic: true - }); + const mq = createMessageQueue(); mq.subscribe('*'); const as = AppserviceMock.create(); const intent = as.getIntentForUserId('@github:example.test'); diff --git a/tests/connections/GitlabRepoTest.ts b/tests/connections/GitlabRepoTest.ts index ee87d765..b2376fe2 100644 --- a/tests/connections/GitlabRepoTest.ts +++ b/tests/connections/GitlabRepoTest.ts @@ -56,9 +56,7 @@ const GITLAB_MR_COMMENT = { const COMMENT_DEBOUNCE_MS = 25; function createConnection(state: Record = {}, isExistingState=false): { connection: GitLabRepoConnection, intent: IntentMock } { - const mq = createMessageQueue({ - monolithic: true - }); + const mq = createMessageQueue(); mq.subscribe('*'); const as = AppserviceMock.create(); const intent = as.getIntentForUserId('@gitlab:example.test');