Add support for prometheus (#99)

* Add support for prometheus metrics

* Automatically build metrics documentation

* Slight linter tidyup

* add changelog

* fix workflow

* hush yarn
This commit is contained in:
Will Hunt 2021-12-16 15:05:03 +00:00 committed by GitHub
parent a45cbb781d
commit ffcb41a0e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 302 additions and 6 deletions

View File

@ -24,4 +24,4 @@ jobs:
github_token: ${{ secrets.GITHUB_TOKEN }}
keep_files: true
publish_dir: ./book
destination_dir: ./
destination_dir: ./

View File

@ -39,6 +39,18 @@ jobs:
node-version: 16
- run: yarn --ignore-scripts && yarn build:app
- run: node lib/Config/Defaults.js --config | diff config.sample.yml -
metrics-docs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Use Node.js
uses: actions/setup-node@v1
with:
node-version: 16
- run: yarn --ignore-scripts
- run: yarn --silent ts-node ./scripts/build-metrics-docs.ts | diff docs/metrics.md -
test:
runs-on: ubuntu-latest
strategy:

1
changelog.d/99.feature Normal file
View File

@ -0,0 +1 @@
Add support for exporting [Prometheus](https://prometheus.io) metrics.

View File

@ -75,6 +75,12 @@ bot:
#
displayname: GitHub Bot
avatar: mxc://half-shot.uk/2876e89ccade4cb615e210c458e2a7a6883fe17d
metrics:
# (Optional) Prometheus metrics support
#
enabled: true
bindAddress: 127.0.0.1
port: 9002
queue:
# (Optional) Message queue / cache configuration options for large scale deployments
#

View File

@ -6,4 +6,5 @@
- [GitHub](./setup/github.md)
- [GitLab](./setup/gitlab.md)
- [JIRA](./setup/jira.md)
- [Webhooks](./setup/webhooks.md)
- [Webhooks](./setup/webhooks.md)
- [Metrics](./metrics.md)

55
docs/metrics.md Normal file
View File

@ -0,0 +1,55 @@
Prometheus Metrics
==================
Below is the generated list of prometheus metrics for hookshot.
## hookshot
| Metric | Help | Labels |
|--------|------|--------|
| hookshot_webhooks_http_request | Number of requests made to the hookshot webhooks handler | path, method |
| hookshot_provisioning_http_request | Number of requests made to the hookshot webhooks handler | path, method |
| hookshot_queue_event_pushes | Number of events pushed through the queue | event |
| hookshot_notifications_push | Number of notifications pushed | service |
| hookshot_notifications_service_up | Is the notification service up or down | service |
| hookshot_notifications_watchers | Number of notifications watchers running | service |
## matrix
| Metric | Help | Labels |
|--------|------|--------|
| matrix_api_calls | The number of Matrix client API calls made | method |
| matrix_api_calls_failed | The number of Matrix client API calls which failed | method |
| matrix_appservice_events | The number of events sent over the AS API | |
## process
| Metric | Help | Labels |
|--------|------|--------|
| process_cpu_user_seconds_total | Total user CPU time spent in seconds. | |
| process_cpu_system_seconds_total | Total system CPU time spent in seconds. | |
| process_cpu_seconds_total | Total user and system CPU time spent in seconds. | |
| process_start_time_seconds | Start time of the process since unix epoch in seconds. | |
| process_resident_memory_bytes | Resident memory size in bytes. | |
| process_virtual_memory_bytes | Virtual memory size in bytes. | |
| process_heap_bytes | Process heap size in bytes. | |
| process_open_fds | Number of open file descriptors. | |
| process_max_fds | Maximum number of open file descriptors. | |
## nodejs
| Metric | Help | Labels |
|--------|------|--------|
| nodejs_eventloop_lag_seconds | Lag of event loop in seconds. | |
| nodejs_eventloop_lag_min_seconds | The minimum recorded event loop delay. | |
| nodejs_eventloop_lag_max_seconds | The maximum recorded event loop delay. | |
| nodejs_eventloop_lag_mean_seconds | The mean of the recorded event loop delays. | |
| nodejs_eventloop_lag_stddev_seconds | The standard deviation of the recorded event loop delays. | |
| nodejs_eventloop_lag_p50_seconds | The 50th percentile of the recorded event loop delays. | |
| nodejs_eventloop_lag_p90_seconds | The 90th percentile of the recorded event loop delays. | |
| nodejs_eventloop_lag_p99_seconds | The 99th percentile of the recorded event loop delays. | |
| nodejs_active_handles | Number of active libuv handles grouped by handle type. Every handle type is C++ class name. | type |
| nodejs_active_handles_total | Total number of active handles. | |
| nodejs_active_requests | Number of active libuv requests grouped by request type. Every request type is C++ class name. | type |
| nodejs_active_requests_total | Total number of active requests. | |
| nodejs_heap_size_total_bytes | Process heap size from Node.js in bytes. | |
| nodejs_heap_size_used_bytes | Process heap size used from Node.js in bytes. | |
| nodejs_external_memory_bytes | Node.js external memory size in bytes. | |
| nodejs_heap_space_size_total_bytes | Process heap space size total from Node.js in bytes. | space |
| nodejs_heap_space_size_used_bytes | Process heap space size used from Node.js in bytes. | space |
| nodejs_heap_space_size_available_bytes | Process heap space size available from Node.js in bytes. | space |
| nodejs_version_info | Node.js version info. | version, major, minor, patch |
| nodejs_gc_duration_seconds | Garbage collection duration by kind, one of major, minor, incremental or weakcb. | kind |

View File

@ -17,6 +17,7 @@
"build:web": "snowpack build",
"build:app": "tsc --project tsconfig.json",
"build:app:rs": "napi build --release ./lib",
"build:docs": "ts-node scripts/build-metrics-docs.ts > docs/metrics.md && mdbook build",
"dev:web": "snowpack dev",
"build": "yarn run build:web && yarn run build:app:rs && yarn run build:app",
"prepare": "yarn build",
@ -47,6 +48,7 @@
"micromatch": "^4.0.4",
"mime": "^3.0.0",
"node-emoji": "^1.11.0",
"prom-client": "^14.0.1",
"reflect-metadata": "^0.1.13",
"source-map-support": "^0.5.21",
"string-argv": "^0.3.1",
@ -63,6 +65,7 @@
"@types/cors": "^2.8.12",
"@types/express": "^4.17.13",
"@types/ioredis": "^4.28.1",
"@types/jira-client": "^7.1.0",
"@types/markdown-it": "^12.2.3",
"@types/micromatch": "^4.0.1",
"@types/mime": "^2.0.3",
@ -70,7 +73,6 @@
"@types/node": "^12",
"@types/node-emoji": "^1.8.1",
"@types/uuid": "^8.3.3",
"@types/jira-client": "^7.1.0",
"@typescript-eslint/eslint-plugin": "^5.4.0",
"@typescript-eslint/parser": "^5.4.0",
"chai": "^4.3.4",

View File

@ -0,0 +1,37 @@
/* eslint-disable no-console */
import Metrics from "../src/Metrics";
import { register } from "prom-client";
// This is just used to ensure we create a singleton.
Metrics.getMetrics();
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const anyRegister = register as any as {_metrics: {[metricName: string]: {labelNames: string[], name: string, help: string}}};
const categories: {[title: string]: {name: string, labels: string[], help: string}[]} = {};
Object.entries(anyRegister._metrics).map(
([key, value]) => {
const [categoryName] = key.split('_');
categories[categoryName] = categories[categoryName] || [];
categories[categoryName].push({
name: key,
labels: value.labelNames,
help: value.help,
});
});
// Generate some markdown
console.log(`Prometheus Metrics
==================
Below is the generated list of prometheus metrics for hookshot.
`)
Object.entries(categories).forEach(([name, entries]) => {
console.log(`## ${name}`);
console.log('| Metric | Help | Labels |');
console.log('|--------|------|--------|');
entries.forEach((e) => console.log(`| ${e.name} | ${e.help} | ${e.labels.join(', ')} |`));
});

View File

@ -2,6 +2,7 @@ import { BridgeConfig } from "../Config/Config";
import { Webhooks } from "../Webhooks";
import LogWrapper from "../LogWrapper";
import { UserNotificationWatcher } from "../Notifications/UserNotificationWatcher";
import Metrics from "../Metrics";
const log = new LogWrapper("App");
@ -10,6 +11,13 @@ async function start() {
const configFile = process.argv[2] || "./config.yml";
const config = await BridgeConfig.parseConfig(configFile, process.env);
LogWrapper.configureLogging(config.logging.level);
if (config.metrics) {
if (!config.metrics.port) {
log.warn(`Not running metrics for service, no port specified`);
} else {
Metrics.start(config.metrics);
}
}
const webhookHandler = new Webhooks(config);
webhookHandler.listen();
const userWatcher = new UserNotificationWatcher(config);
@ -18,6 +26,7 @@ async function start() {
log.error("Got SIGTERM");
webhookHandler.stop();
userWatcher.stop();
Metrics.stop();
});
}

View File

@ -1,6 +1,7 @@
import { BridgeConfig, parseRegistrationFile } from "../Config/Config";
import { MatrixSender } from "../MatrixSender";
import LogWrapper from "../LogWrapper";
import Metrics from "../Metrics";
const log = new LogWrapper("App");
@ -12,10 +13,18 @@ async function start() {
const registration = await parseRegistrationFile(registrationFile);
LogWrapper.configureLogging(config.logging.level);
const sender = new MatrixSender(config, registration);
if (config.metrics) {
if (!config.metrics.port) {
log.warn(`Not running metrics for service, no port specified`);
} else {
Metrics.start(config.metrics);
}
}
sender.listen();
process.once("SIGTERM", () => {
log.error("Got SIGTERM");
sender.stop();
Metrics.stop();
});
}

View File

@ -34,6 +34,7 @@ import { GitHubProvisionerRouter } from "./Github/Router";
import { OAuthRequest } from "./WebhookTypes";
import { promises as fs } from "fs";
import { SetupConnection } from "./Connections/SetupConnection";
import Metrics from "./Metrics";
const log = new LogWrapper("Bridge");
export function getAppservice(config: BridgeConfig, registration: IAppserviceRegistration, storage: IAppserviceStorageProvider) {
@ -94,6 +95,7 @@ export class Bridge {
public stop() {
this.as.stop();
Metrics.stop();
if (this.queue.stop) this.queue.stop();
if (this.widgetApi) this.widgetApi.stop();
if (this.provisioningApi) this.provisioningApi.stop();
@ -178,6 +180,7 @@ export class Bridge {
});
this.as.on("room.event", async (roomId, event) => {
Metrics.matrixAppserviceEvents.inc();
return this.onRoomEvent(roomId, event);
});
@ -580,6 +583,9 @@ export class Bridge {
if (this.provisioningApi) {
await this.provisioningApi.listen();
}
if (this.config.metrics?.enabled) {
Metrics.start(this.config.metrics, this.as);
}
await this.as.begin();
log.info("Started bridge");
this.ready = true;

View File

@ -139,6 +139,13 @@ export interface BridgeConfigProvisioning {
secret: string;
}
export interface BridgeConfigMetrics {
enabled: boolean;
bindAddress?: string;
port?: number;
}
interface BridgeConfigRoot {
bot?: BridgeConfigBot;
bridge: BridgeConfigBridge;
@ -152,6 +159,7 @@ interface BridgeConfigRoot {
queue: BridgeConfigQueue;
webhook: BridgeConfigWebhook;
widgets?: BridgeWidgetConfig;
metrics?: BridgeConfigMetrics;
}
export class BridgeConfig {
@ -180,6 +188,8 @@ export class BridgeConfig {
public readonly widgets?: BridgeWidgetConfig;
@configKey("Provisioning API for integration managers", true)
public readonly provisioning?: BridgeConfigProvisioning;
@configKey("Prometheus metrics support", true)
public readonly metrics?: BridgeConfigMetrics;
constructor(configData: BridgeConfigRoot, env: {[key: string]: string|undefined}) {
this.bridge = configData.bridge;
@ -198,6 +208,7 @@ export class BridgeConfig {
this.provisioning = configData.provisioning;
this.passFile = configData.passFile;
this.bot = configData.bot;
this.metrics = configData.metrics;
assert.ok(this.webhook);
this.queue = configData.queue || {
monolithic: true,

View File

@ -81,6 +81,11 @@ export const DefaultConfig = new BridgeConfig({
bindAddress: "127.0.0.1",
port: 9001,
secret: "!secretToken"
},
metrics: {
enabled: true,
bindAddress: "127.0.0.1",
port: 9002,
}
}, {});

View File

@ -2,6 +2,7 @@ import { EventEmitter } from "events";
import { MessageQueue, MessageQueueMessage, DEFAULT_RES_TIMEOUT } from "./Types";
import micromatch from "micromatch";
import {v4 as uuid} from "uuid";
import Metrics from "../Metrics";
export class LocalMQ extends EventEmitter implements MessageQueue {
private subs: Set<string>;
@ -19,6 +20,7 @@ export class LocalMQ extends EventEmitter implements MessageQueue {
}
public async push<T>(message: MessageQueueMessage<T>) {
Metrics.messageQueuePushes.inc({event: message.eventName});
if (!micromatch.match([...this.subs], message.eventName)) {
return;
}

106
src/Metrics.ts Normal file
View File

@ -0,0 +1,106 @@
import { Appservice, FunctionCallContext, METRIC_MATRIX_CLIENT_FAILED_FUNCTION_CALL, METRIC_MATRIX_CLIENT_SUCCESSFUL_FUNCTION_CALL } from "matrix-bot-sdk";
import { collectDefaultMetrics, Counter, Gauge, register, Registry } from "prom-client";
import { BridgeConfigMetrics } from "./Config/Config";
import { Response, default as expressApp } from "express";
import LogWrapper from "./LogWrapper";
import { Server } from "http";
const log = new LogWrapper("Metrics");
export class Metrics {
private httpServer?: Server;
public readonly webhooksHttpRequest = new Counter({ name: "hookshot_webhooks_http_request", help: "Number of requests made to the hookshot webhooks handler", labelNames: ["path", "method"], registers: [this.registry]});
public readonly provisioningHttpRequest = new Counter({ name: "hookshot_provisioning_http_request", help: "Number of requests made to the hookshot webhooks handler", labelNames: ["path", "method"], registers: [this.registry]});
public readonly messageQueuePushes = new Counter({ name: "hookshot_queue_event_pushes", help: "Number of events pushed through the queue", labelNames: ["event"], registers: [this.registry]});
public readonly notificationsPush = new Counter({ name: "hookshot_notifications_push", help: "Number of notifications pushed", labelNames: ["service"], registers: [this.registry]});
public readonly notificationsServiceUp = new Gauge({ name: "hookshot_notifications_service_up", help: "Is the notification service up or down", labelNames: ["service"], registers: [this.registry]});
public readonly notificationsWatchers = new Gauge({ name: "hookshot_notifications_watchers", help: "Number of notifications watchers running", labelNames: ["service"], registers: [this.registry]});
private readonly matrixApiCalls = new Counter({ name: "matrix_api_calls", help: "The number of Matrix client API calls made", labelNames: ["method"], registers: [this.registry]});
private readonly matrixApiCallsFailed = new Counter({ name: "matrix_api_calls_failed", help: "The number of Matrix client API calls which failed", labelNames: ["method"], registers: [this.registry]});
public readonly matrixAppserviceEvents = new Counter({ name: "matrix_appservice_events", help: "The number of events sent over the AS API", labelNames: [], registers: [this.registry]});
constructor(private registry: Registry = register) {
collectDefaultMetrics({
register: this.registry
})
}
public async getMetrics() {
return this.registry.metrics();
}
/**
* Registers some exported metrics that relate to operations of the embedded
* matrix-js-sdk. In particular, a metric is added that counts the number of
* calls to client API endpoints made by the client library.
*/
public registerMatrixSdkMetrics(appservice: Appservice): void {
appservice.metrics.registerListener({
onStartMetric: () => {
// Not used yet.
},
onEndMetric: () => {
// Not used yet.
},
onIncrement: (metricName, context) => {
if (metricName === METRIC_MATRIX_CLIENT_SUCCESSFUL_FUNCTION_CALL) {
const ctx = context as FunctionCallContext;
this.matrixApiCalls.inc({method: ctx.functionName});
}
if (metricName === METRIC_MATRIX_CLIENT_FAILED_FUNCTION_CALL) {
const ctx = context as FunctionCallContext;
this.matrixApiCallsFailed.inc({method: ctx.functionName});
}
},
onDecrement: () => {
// Not used yet.
},
onReset: (metricName) => {
if (metricName === METRIC_MATRIX_CLIENT_SUCCESSFUL_FUNCTION_CALL) {
this.matrixApiCalls.reset();
}
if (metricName === METRIC_MATRIX_CLIENT_FAILED_FUNCTION_CALL) {
this.matrixApiCallsFailed.reset();
}
},
})
}
private metricsFunc(_req: unknown, res: Response) {
this.getMetrics().then(
(m) => res.type('text/plain').send((m))
).catch((err) => {
log.error('Failed to fetch metrics: ', err);
res.status(500).send('Could not fetch metrics due to an error');
});
}
public start(config: BridgeConfigMetrics, as?: Appservice) {
if (!config.port) {
if (!as) {
throw Error("No metric port defined in config, and service doesn't run a appservice");
}
as.expressAppInstance.get('/metrics', this.metricsFunc.bind(this));
return;
}
const app = expressApp();
app.get('/metrics', this.metricsFunc.bind(this));
this.httpServer = app.listen(config.port, config.bindAddress || "127.0.0.1");
}
public async stop() {
if (!this.httpServer) {
return;
}
return new Promise<void>((res, rej) => this.httpServer?.close(err => err ? rej(err) : res()));
}
}
const singleton = new Metrics();
export default singleton;

View File

@ -6,7 +6,7 @@ import { NotificationWatcherTask } from "./NotificationWatcherTask";
import { RequestError } from "@octokit/request-error";
import { GitHubUserNotification } from "../Github/Types";
import { OctokitResponse } from "@octokit/types";
import Metrics from "../Metrics";
const log = new LogWrapper("GitHubWatcher");
const GH_API_THRESHOLD = 50;
@ -27,6 +27,7 @@ export class GitHubWatcher extends EventEmitter implements NotificationWatcherTa
this.globalRetryIn = Date.now() + GH_API_RETRY_IN;
}
log.warn(`API Failure limit reached, holding off new requests for ${GH_API_RETRY_IN / 1000}s`);
Metrics.notificationsServiceUp.set({service: "github"}, 0);
}
private octoKit: Octokit;
@ -80,6 +81,7 @@ export class GitHubWatcher extends EventEmitter implements NotificationWatcherTa
response = await this.octoKit.request(
`/notifications?participating=${this.participating}${since}`,
);
Metrics.notificationsServiceUp.set({service: "github"}, 1);
// We were succesful, clear any timeouts.
GitHubWatcher.globalRetryIn = 0;
// To avoid a bouncing issue, gradually reduce the failure count.
@ -129,6 +131,7 @@ export class GitHubWatcher extends EventEmitter implements NotificationWatcherTa
// We still push
}
log.debug(`Pushing ${rawEvent.id}`);
Metrics.notificationsPush.inc({service: "github"});
this.emit("new_events", {
eventName: "notifications.user.events",
data: {

View File

@ -7,7 +7,7 @@ import { GitHubWatcher } from "./GitHubWatcher";
import { GitHubUserNotification } from "../Github/Types";
import { GitLabWatcher } from "./GitLabWatcher";
import { BridgeConfig } from "../Config/Config";
import Metrics from "../Metrics";
export interface UserNotificationsEvent {
roomId: string;
lastReadTs: number;
@ -59,6 +59,7 @@ export class UserNotificationWatcher {
this.userIntervals.delete(key);
log.info(`Removed ${key} from the notif queue`);
}
Metrics.notificationsWatchers.set({service: type}, this.userIntervals.size);
}
private onFetchFailure(task: NotificationWatcherTask) {
@ -89,6 +90,7 @@ Check your token is still valid, and then turn notifications back on.`, "m.notic
this.queue.push<UserNotificationsEvent>(payload);
});
this.userIntervals.set(key, task);
Metrics.notificationsWatchers.set({service: data.type}, this.userIntervals.size);
log.info(`Inserted ${key} into the notif queue`);
}
}

View File

@ -12,6 +12,7 @@ import { IJiraWebhookEvent } from "./Jira/WebhookTypes";
import { JiraWebhooksRouter } from "./Jira/Router";
import { OAuthRequest } from "./WebhookTypes";
import { GitHubOAuthTokenResponse } from "./Github/Types";
import Metrics from "./Metrics";
const log = new LogWrapper("GithubWebhooks");
export interface GenericWebhookEvent {
@ -43,6 +44,10 @@ export class Webhooks extends EventEmitter {
constructor(private config: BridgeConfig) {
super();
this.expressApp = express();
this.expressApp.use((req, _res, next) => {
Metrics.webhooksHttpRequest.inc({path: req.path, method: req.method});
next();
});
if (this.config.github?.webhook.secret) {
this.ghWebhooks = new OctokitWebhooks({
secret: config.github?.webhook.secret as string,

View File

@ -5,6 +5,7 @@ import LogWrapper from "../LogWrapper";
import { Server } from "http";
import { ApiError, ErrCode, GetConnectionsResponseItem, GetConnectionTypeResponseItem } from "./api";
import { Intent, MembershipEventContent, PowerLevelsEventContent } from "matrix-bot-sdk";
import Metrics from "../Metrics";
const log = new LogWrapper("Provisioner");
@ -28,6 +29,10 @@ export class Provisioner {
throw Error('Missing port in provisioning config');
}
this.expressApp = express();
this.expressApp.use((req, _res, next) => {
Metrics.provisioningHttpRequest.inc({path: req.path, method: req.method});
next();
});
this.expressApp.get("/v1/health", this.getHealth);
this.expressApp.use(this.checkAuth.bind(this));
this.expressApp.use(express.json());
@ -257,4 +262,4 @@ export class Provisioner {
this.server.close();
}
}
}
}

View File

@ -1357,6 +1357,11 @@ binary-extensions@^2.0.0:
resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==
bintrees@1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.1.tgz#0e655c9b9c2435eaab68bf4027226d2b55a34524"
integrity sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ=
bl@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a"
@ -4537,6 +4542,13 @@ progress@^2.0.0:
resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
prom-client@^14.0.1:
version "14.0.1"
resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-14.0.1.tgz#bdd9583e02ec95429677c0e013712d42ef1f86a8"
integrity sha512-HxTArb6fkOntQHoRGvv4qd/BkorjliiuO2uSWC2KC17MUTKYttWdDoXX/vxOhQdkoECEM9BBH0pj2l8G8kev6w==
dependencies:
tdigest "^0.1.1"
promise-all-reject-late@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/promise-all-reject-late/-/promise-all-reject-late-1.0.1.tgz#f8ebf13483e5ca91ad809ccc2fcf25f26f8643c2"
@ -5298,6 +5310,13 @@ tar@^6.0.2, tar@^6.1.0:
mkdirp "^1.0.3"
yallist "^4.0.0"
tdigest@^0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.1.tgz#2e3cb2c39ea449e55d1e6cd91117accca4588021"
integrity sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=
dependencies:
bintrees "1.0.1"
text-hex@1.0.x:
version "1.0.0"
resolved "https://registry.yarnpkg.com/text-hex/-/text-hex-1.0.0.tgz#69dc9c1b17446ee79a92bf5b884bb4b9127506f5"