From 3217b9eecf2f522e1b47a21200ff5cb8eeba3598 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Wed, 28 Jun 2023 16:29:54 +0100 Subject: [PATCH] Performance improvements to Feeds (#786) * Various smaller changes * Drop account data entirely * Use max feed items * Commit known working improvements * Better status handlingh * changelog * Update changelog * Add a note on Redis. * Add proper HTTP tests * Linty lint * Tweaks * New metrics woah * Tweaks --- Cargo.lock | 757 +++++++++++++++++++++++++++- Cargo.toml | 3 +- changelog.d/786.bugfix | 5 + docs/metrics.md | 2 + docs/setup/feeds.md | 3 + package.json | 6 +- src/Bridge.ts | 3 +- src/Connections/FeedConnection.ts | 12 +- src/Metrics.ts | 61 ++- src/Stores/MemoryStorageProvider.ts | 24 +- src/Stores/RedisStorageProvider.ts | 21 +- src/Stores/StorageProvider.ts | 10 + src/feeds/FeedReader.ts | 255 +++------- src/feeds/parser.rs | 85 +++- tests/FeedReader.spec.ts | 71 ++- yarn.lock | 32 +- 16 files changed, 1086 insertions(+), 264 deletions(-) create mode 100644 changelog.d/786.bugfix diff --git a/Cargo.lock b/Cargo.lock index 371650ac..b19c7460 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,6 +75,12 @@ dependencies = [ "byte-tools", ] +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + [[package]] name = "byte-tools" version = "0.3.1" @@ -99,6 +105,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + [[package]] name = "cfg-if" version = "1.0.0" @@ -134,6 +146,22 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + [[package]] name = "ctor" version = "0.2.0" @@ -237,12 +265,57 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -262,6 +335,45 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + [[package]] name = "generic-array" version = "0.12.4" @@ -282,12 +394,46 @@ dependencies = [ "wasi", ] +[[package]] +name = "h2" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "hex" version = "0.4.3" @@ -308,6 +454,77 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -335,12 +552,47 @@ dependencies = [ "serde", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "ipnet" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" + [[package]] name = "itoa" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "js_int" version = "0.2.2" @@ -381,6 +633,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "984e109462d46ad18314f10e392c286c3d47bce203088a09012de1015b45b737" +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.142" @@ -397,6 +655,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + [[package]] name = "lock_api" version = "0.4.9" @@ -447,6 +711,7 @@ dependencies = [ "napi", "napi-build", "napi-derive", + "reqwest", "rgb", "rss", "ruma", @@ -473,6 +738,23 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + [[package]] name = "napi" version = "2.12.4" @@ -486,6 +768,7 @@ dependencies = [ "once_cell", "serde", "serde_json", + "tokio", ] [[package]] @@ -531,6 +814,24 @@ dependencies = [ "libloading", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "never" version = "0.1.0" @@ -562,6 +863,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi 0.2.6", + "libc", +] + [[package]] name = "once_cell" version = "1.17.1" @@ -574,6 +885,50 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +[[package]] +name = "openssl" +version = "0.10.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.14", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -592,9 +947,9 @@ checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -689,6 +1044,18 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -778,6 +1145,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "regex" version = "1.7.3" @@ -795,6 +1171,43 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "reqwest" +version = "0.11.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rgb" version = "0.8.36" @@ -882,18 +1295,64 @@ dependencies = [ "toml", ] +[[package]] +name = "rustix" +version = "0.37.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + [[package]] name = "ryu" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "schannel" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +dependencies = [ + "windows-sys 0.42.0", +] + [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "security-framework" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.17" @@ -953,18 +1412,49 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "siphasher" version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "string_cache" version = "0.8.7" @@ -1019,6 +1509,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +dependencies = [ + "autocfg", + "cfg-if", + "fastrand", + "redox_syscall 0.3.5", + "rustix", + "windows-sys 0.48.0", +] + [[package]] name = "tendril" version = "0.4.3" @@ -1065,6 +1569,46 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokio" +version = "1.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +dependencies = [ + "autocfg", + "bytes", + "libc", + "mio", + "num_cpus", + "pin-project-lite", + "socket2", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml" version = "0.7.3" @@ -1099,6 +1643,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -1131,6 +1681,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "typenum" version = "1.16.0" @@ -1181,12 +1737,103 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.14", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.14", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wildmatch" version = "2.1.1" @@ -1215,13 +1862,37 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-sys" version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets", + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", ] [[package]] @@ -1230,13 +1901,28 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] @@ -1245,42 +1931,84 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + [[package]] name = "windows_i686_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + [[package]] name = "windows_i686_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + [[package]] name = "winnow" version = "0.4.6" @@ -1289,3 +2017,12 @@ checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" dependencies = [ "memchr", ] + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] diff --git a/Cargo.toml b/Cargo.toml index 8fd5ebc5..ab4306d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" crate-type = ["cdylib"] [dependencies] -napi = {version="2", features=["serde-json"]} +napi = {version="2", features=["serde-json", "async"]} napi-derive = "2" url = "2" serde_json = "1" @@ -20,6 +20,7 @@ hex = "0.4.3" rss = "2.0.3" atom_syndication = "0.12" ruma = { version = "0.8.2", features = ["events", "unstable-sanitize"] } +reqwest = "0.11" [build-dependencies] napi-build = "1" diff --git a/changelog.d/786.bugfix b/changelog.d/786.bugfix new file mode 100644 index 00000000..358c1fef --- /dev/null +++ b/changelog.d/786.bugfix @@ -0,0 +1,5 @@ +Refactor Hookshot to use Redis for caching of feed information, massively improving memory usage. + +Please note that this is a behavioural change: Hookshots configured to use in-memory caching (not Redis), +will no longer bridge any RSS entries it may have missed during downtime, and will instead perform an initial +sync (not reporting any entries) instead. \ No newline at end of file diff --git a/docs/metrics.md b/docs/metrics.md index 68444735..db01bc40 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -70,6 +70,8 @@ Below is the generated list of Prometheus metrics for Hookshot. | nodejs_eventloop_lag_p50_seconds | The 50th percentile of the recorded event loop delays. | | | nodejs_eventloop_lag_p90_seconds | The 90th percentile of the recorded event loop delays. | | | nodejs_eventloop_lag_p99_seconds | The 99th percentile of the recorded event loop delays. | | +| nodejs_active_resources | Number of active resources that are currently keeping the event loop alive, grouped by async resource type. | type | +| nodejs_active_resources_total | Total number of active resources. | | | nodejs_active_handles | Number of active libuv handles grouped by handle type. Every handle type is C++ class name. | type | | nodejs_active_handles_total | Total number of active handles. | | | nodejs_active_requests | Number of active libuv requests grouped by request type. Every request type is C++ class name. | type | diff --git a/docs/setup/feeds.md b/docs/setup/feeds.md index fff83c4a..6ace4021 100644 --- a/docs/setup/feeds.md +++ b/docs/setup/feeds.md @@ -19,6 +19,9 @@ Each feed will only be checked once, regardless of the number of rooms to which No entries will be bridged upon the “initial sync” -- all entries that exist at the moment of setup will be considered to be already seen. +Please note that Hookshot **must** be configued with Redis to retain seen entries between restarts. By default, Hookshot will +run an "initial sync" on each startup and will not process any entries from feeds from before the first sync. + ## Usage ### Adding new feeds diff --git a/package.json b/package.json index 4d1acc77..7af3c05a 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "node-emoji": "^1.11.0", "nyc": "^15.1.0", "p-queue": "^6.6.2", - "prom-client": "^14.0.1", + "prom-client": "^14.2.0", "reflect-metadata": "^0.1.13", "source-map-support": "^0.5.21", "string-argv": "^0.3.1", @@ -76,7 +76,7 @@ }, "devDependencies": { "@codemirror/lang-javascript": "^6.0.2", - "@napi-rs/cli": "^2.2.0", + "@napi-rs/cli": "^2.13.2", "@preact/preset-vite": "^2.2.0", "@tsconfig/node18": "^2.0.0", "@types/ajv": "^1.0.0", @@ -105,7 +105,7 @@ "rimraf": "^3.0.2", "sass": "^1.51.0", "ts-node": "^10.9.1", - "typescript": "^5.0.4", + "typescript": "^5.1.3", "vite": "^4.1.5", "vite-svg-loader": "^4.0.0" } diff --git a/src/Bridge.ts b/src/Bridge.ts index 89537659..91b4dee9 100644 --- a/src/Bridge.ts +++ b/src/Bridge.ts @@ -776,8 +776,7 @@ export class Bridge { this.config.feeds, this.connectionManager, this.queue, - // Use default bot when storing account data - this.as.botClient, + this.storage, ); } diff --git a/src/Connections/FeedConnection.ts b/src/Connections/FeedConnection.ts index c746546c..26802693 100644 --- a/src/Connections/FeedConnection.ts +++ b/src/Connections/FeedConnection.ts @@ -1,13 +1,14 @@ import {Intent, StateEvent} from "matrix-bot-sdk"; import { IConnection, IConnectionState, InstantiateConnectionOpts } from "."; import { ApiError, ErrCode } from "../api"; -import { FeedEntry, FeedError, FeedReader} from "../feeds/FeedReader"; +import { FeedEntry, FeedError} from "../feeds/FeedReader"; import { Logger } from "matrix-appservice-bridge"; import { BaseConnection } from "./BaseConnection"; import markdown from "markdown-it"; import { Connection, ProvisionConnectionOpts } from "./IConnection"; import { GetConnectionsResponseItem } from "../provisioning/api"; -import { sanitizeHtml } from "../libRs"; +import { readFeed, sanitizeHtml } from "../libRs"; +import UserAgent from "../UserAgent"; const log = new Logger("FeedConnection"); const md = new markdown({ html: true, @@ -38,7 +39,7 @@ export interface FeedConnectionSecrets { export type FeedResponseItem = GetConnectionsResponseItem; const MAX_LAST_RESULT_ITEMS = 5; -const VALIDATION_FETCH_TIMEOUT_MS = 5000; +const VALIDATION_FETCH_TIMEOUT_S = 5; const MAX_SUMMARY_LENGTH = 512; const MAX_TEMPLATE_LENGTH = 1024; @@ -68,7 +69,10 @@ export class FeedConnection extends BaseConnection implements IConnection { } try { - await FeedReader.fetchFeed(url, {}, VALIDATION_FETCH_TIMEOUT_MS); + await readFeed(url, { + userAgent: UserAgent, + pollTimeoutSeconds: VALIDATION_FETCH_TIMEOUT_S, + }); } catch (ex) { throw new ApiError(`Could not read feed from URL: ${ex.message}`, ErrCode.BadValue); } diff --git a/src/Metrics.ts b/src/Metrics.ts index 8e06c69c..cd5d0c52 100644 --- a/src/Metrics.ts +++ b/src/Metrics.ts @@ -7,33 +7,58 @@ const log = new Logger("Metrics"); export class Metrics { public readonly expressRouter = Router(); - public readonly webhooksHttpRequest = new Counter({ name: "hookshot_webhooks_http_request", help: "Number of requests made to the hookshot webhooks handler", labelNames: ["path", "method"], registers: [this.registry]}); - public readonly provisioningHttpRequest = new Counter({ name: "hookshot_provisioning_http_request", help: "Number of requests made to the hookshot webhooks handler", labelNames: ["path", "method"], registers: [this.registry]}); + public readonly webhooksHttpRequest; + public readonly provisioningHttpRequest; - public readonly messageQueuePushes = new Counter({ name: "hookshot_queue_event_pushes", help: "Number of events pushed through the queue", labelNames: ["event"], registers: [this.registry]}); - public readonly connectionsEventFailed = new Counter({ name: "hookshot_connection_event_failed", help: "The number of events that failed to process", labelNames: ["event", "connectionId"], registers: [this.registry]}); - public readonly connections = new Gauge({ name: "hookshot_connections", help: "The number of active hookshot connections", labelNames: ["service"], registers: [this.registry]}); + public readonly messageQueuePushes; + public readonly connectionsEventFailed; + public readonly connections; - public readonly notificationsPush = new Counter({ name: "hookshot_notifications_push", help: "Number of notifications pushed", labelNames: ["service"], registers: [this.registry]}); - public readonly notificationsServiceUp = new Gauge({ name: "hookshot_notifications_service_up", help: "Is the notification service up or down", labelNames: ["service"], registers: [this.registry]}); - public readonly notificationsWatchers = new Gauge({ name: "hookshot_notifications_watchers", help: "Number of notifications watchers running", labelNames: ["service"], registers: [this.registry]}); + public readonly notificationsPush; + public readonly notificationsServiceUp; + public readonly notificationsWatchers; - private readonly matrixApiCalls = new Counter({ name: "matrix_api_calls", help: "The number of Matrix client API calls made", labelNames: ["method"], registers: [this.registry]}); - private readonly matrixApiCallsFailed = new Counter({ name: "matrix_api_calls_failed", help: "The number of Matrix client API calls which failed", labelNames: ["method"], registers: [this.registry]}); + private readonly matrixApiCalls; + private readonly matrixApiCallsFailed; - public readonly matrixAppserviceEvents = new Counter({ name: "matrix_appservice_events", help: "The number of events sent over the AS API", labelNames: [], registers: [this.registry]}); - public readonly matrixAppserviceDecryptionFailed = new Counter({ name: "matrix_appservice_decryption_failed", help: "The number of events sent over the AS API that failed to decrypt", registers: [this.registry]}); + public readonly matrixAppserviceEvents; + public readonly matrixAppserviceDecryptionFailed; - public readonly feedsCount = new Gauge({ name: "hookshot_feeds_count", help: "The number of RSS feeds that hookshot is subscribed to", labelNames: [], registers: [this.registry]}); - public readonly feedFetchMs = new Gauge({ name: "hookshot_feeds_fetch_ms", help: "The time taken for hookshot to fetch all feeds", labelNames: [], registers: [this.registry]}); - public readonly feedsFailing = new Gauge({ name: "hookshot_feeds_failing", help: "The number of RSS feeds that hookshot is failing to read", labelNames: ["reason"], registers: [this.registry]}); - public readonly feedsCountDeprecated = new Gauge({ name: "feed_count", help: "(Deprecated) The number of RSS feeds that hookshot is subscribed to", labelNames: [], registers: [this.registry]}); - public readonly feedsFetchMsDeprecated = new Gauge({ name: "feed_fetch_ms", help: "(Deprecated) The time taken for hookshot to fetch all feeds", labelNames: [], registers: [this.registry]}); - public readonly feedsFailingDeprecated = new Gauge({ name: "feed_failing", help: "(Deprecated) The number of RSS feeds that hookshot is failing to read", labelNames: ["reason"], registers: [this.registry]}); + public readonly feedsCount; + public readonly feedFetchMs; + public readonly feedsFailing; + public readonly feedsCountDeprecated; + public readonly feedsFetchMsDeprecated; + public readonly feedsFailingDeprecated; constructor(private registry: Registry = register) { this.expressRouter.get('/metrics', this.metricsFunc.bind(this)); + + this.webhooksHttpRequest = new Counter({ name: "hookshot_webhooks_http_request", help: "Number of requests made to the hookshot webhooks handler", labelNames: ["path", "method"], registers: [this.registry]}); + this.provisioningHttpRequest = new Counter({ name: "hookshot_provisioning_http_request", help: "Number of requests made to the hookshot webhooks handler", labelNames: ["path", "method"], registers: [this.registry]}); + + this.messageQueuePushes = new Counter({ name: "hookshot_queue_event_pushes", help: "Number of events pushed through the queue", labelNames: ["event"], registers: [this.registry]}); + this.connectionsEventFailed = new Counter({ name: "hookshot_connection_event_failed", help: "The number of events that failed to process", labelNames: ["event", "connectionId"], registers: [this.registry]}); + this.connections = new Gauge({ name: "hookshot_connections", help: "The number of active hookshot connections", labelNames: ["service"], registers: [this.registry]}); + + this.notificationsPush = new Counter({ name: "hookshot_notifications_push", help: "Number of notifications pushed", labelNames: ["service"], registers: [this.registry]}); + this.notificationsServiceUp = new Gauge({ name: "hookshot_notifications_service_up", help: "Is the notification service up or down", labelNames: ["service"], registers: [this.registry]}); + this.notificationsWatchers = new Gauge({ name: "hookshot_notifications_watchers", help: "Number of notifications watchers running", labelNames: ["service"], registers: [this.registry]}); + + this.matrixApiCalls = new Counter({ name: "matrix_api_calls", help: "The number of Matrix client API calls made", labelNames: ["method"], registers: [this.registry]}); + this.matrixApiCallsFailed = new Counter({ name: "matrix_api_calls_failed", help: "The number of Matrix client API calls which failed", labelNames: ["method"], registers: [this.registry]}); + + this.matrixAppserviceEvents = new Counter({ name: "matrix_appservice_events", help: "The number of events sent over the AS API", labelNames: [], registers: [this.registry]}); + this.matrixAppserviceDecryptionFailed = new Counter({ name: "matrix_appservice_decryption_failed", help: "The number of events sent over the AS API that failed to decrypt", registers: [this.registry]}); + + this.feedsCount = new Gauge({ name: "hookshot_feeds_count", help: "The number of RSS feeds that hookshot is subscribed to", labelNames: [], registers: [this.registry]}); + this.feedFetchMs = new Gauge({ name: "hookshot_feeds_fetch_ms", help: "The time taken for hookshot to fetch all feeds", labelNames: [], registers: [this.registry]}); + this.feedsFailing = new Gauge({ name: "hookshot_feeds_failing", help: "The number of RSS feeds that hookshot is failing to read", labelNames: ["reason"], registers: [this.registry]}); + this.feedsCountDeprecated = new Gauge({ name: "feed_count", help: "(Deprecated) The number of RSS feeds that hookshot is subscribed to", labelNames: [], registers: [this.registry]}); + this.feedsFetchMsDeprecated = new Gauge({ name: "feed_fetch_ms", help: "(Deprecated) The time taken for hookshot to fetch all feeds", labelNames: [], registers: [this.registry]}); + this.feedsFailingDeprecated = new Gauge({ name: "feed_failing", help: "(Deprecated) The number of RSS feeds that hookshot is failing to read", labelNames: ["reason"], registers: [this.registry]}); + collectDefaultMetrics({ register: this.registry, }) diff --git a/src/Stores/MemoryStorageProvider.ts b/src/Stores/MemoryStorageProvider.ts index b4bcf894..b81b6fe5 100644 --- a/src/Stores/MemoryStorageProvider.ts +++ b/src/Stores/MemoryStorageProvider.ts @@ -1,5 +1,5 @@ import { MemoryStorageProvider as MSP } from "matrix-bot-sdk"; -import { IBridgeStorageProvider } from "./StorageProvider"; +import { IBridgeStorageProvider, MAX_FEED_ITEMS } from "./StorageProvider"; import { IssuesGetResponseData } from "../github/Types"; import { ProvisionSession } from "matrix-appservice-bridge"; import QuickLRU from "@alloc/quick-lru"; @@ -11,10 +11,32 @@ export class MemoryStorageProvider extends MSP implements IBridgeStorageProvider private figmaCommentIds: Map = new Map(); private widgetSessions: Map = new Map(); private storedFiles = new QuickLRU({ maxSize: 128 }); + private feedGuids = new Map>(); constructor() { super(); } + + async storeFeedGuids(url: string, ...guids: string[]): Promise { + let set = this.feedGuids.get(url); + if (!set) { + set = [] + this.feedGuids.set(url, set); + } + set.unshift(...guids); + while (set.length > MAX_FEED_ITEMS) { + set.pop(); + } + } + + async hasSeenFeed(url: string): Promise { + return this.feedGuids.has(url); + } + + async hasSeenFeedGuid(url: string, guid: string): Promise { + return this.feedGuids.get(url)?.includes(guid) ?? false; + } + public async setGithubIssue(repo: string, issueNumber: string, data: IssuesGetResponseData, scope = "") { this.issues.set(`${scope}${repo}/${issueNumber}`, data); } diff --git a/src/Stores/RedisStorageProvider.ts b/src/Stores/RedisStorageProvider.ts index 5b5d027e..859ded1f 100644 --- a/src/Stores/RedisStorageProvider.ts +++ b/src/Stores/RedisStorageProvider.ts @@ -2,7 +2,7 @@ import { IssuesGetResponseData } from "../github/Types"; import { Redis, default as redis } from "ioredis"; import { Logger } from "matrix-appservice-bridge"; -import { IBridgeStorageProvider } from "./StorageProvider"; +import { IBridgeStorageProvider, MAX_FEED_ITEMS } from "./StorageProvider"; import { IFilterInfo, IStorageProvider } from "matrix-bot-sdk"; import { ProvisionSession } from "matrix-appservice-bridge"; @@ -25,6 +25,10 @@ const ISSUES_LAST_COMMENT_EXPIRE_AFTER = 14 * 24 * 60 * 60; // 7 days const WIDGET_TOKENS = "widgets.tokens."; const WIDGET_USER_TOKENS = "widgets.user-tokens."; +const FEED_GUIDS = "feeds.guids."; + + + const log = new Logger("RedisASProvider"); export class RedisStorageContextualProvider implements IStorageProvider { @@ -61,6 +65,7 @@ export class RedisStorageContextualProvider implements IStorageProvider { } + export class RedisStorageProvider extends RedisStorageContextualProvider implements IBridgeStorageProvider { constructor(host: string, port: number, contextSuffix = '') { super(new redis(port, host), contextSuffix); @@ -198,4 +203,18 @@ export class RedisStorageProvider extends RedisStorageContextualProvider impleme public async setStoredTempFile(key: string, value: string) { await this.redis.set(STORED_FILES_KEY + key, value); } + + public async storeFeedGuids(url: string, ...guid: string[]): Promise { + const feedKey = `${FEED_GUIDS}${url}`; + await this.redis.lpush(feedKey, ...guid); + await this.redis.ltrim(feedKey, 0, MAX_FEED_ITEMS); + } + + public async hasSeenFeed(url: string): Promise { + return (await this.redis.exists(`${FEED_GUIDS}${url}`)) === 1; + } + + public async hasSeenFeedGuid(url: string, guid: string): Promise { + return (await this.redis.lpos(`${FEED_GUIDS}${url}`, guid)) != null; + } } diff --git a/src/Stores/StorageProvider.ts b/src/Stores/StorageProvider.ts index 447fb84a..af3ee7b7 100644 --- a/src/Stores/StorageProvider.ts +++ b/src/Stores/StorageProvider.ts @@ -2,6 +2,13 @@ import { ProvisioningStore } from "matrix-appservice-bridge"; import { IAppserviceStorageProvider, IStorageProvider } from "matrix-bot-sdk"; import { IssuesGetResponseData } from "../github/Types"; +// Some RSS feeds can return a very small number of items then bounce +// back to their "normal" size, so we cannot just clobber the recent GUID list per request or else we'll +// forget what we sent and resend it. Instead, we'll keep 2x the max number of items that we've ever +// seen from this feed, up to a max of 10,000. +// Adopted from https://github.com/matrix-org/go-neb/blob/babb74fa729882d7265ff507b09080e732d060ae/services/rssbot/rssbot.go#L304 +export const MAX_FEED_ITEMS = 10_000; + export interface IBridgeStorageProvider extends IAppserviceStorageProvider, IStorageProvider, ProvisioningStore { connect?(): Promise; disconnect?(): Promise; @@ -15,4 +22,7 @@ export interface IBridgeStorageProvider extends IAppserviceStorageProvider, ISto getFigmaCommentEventId(roomId: string, figmaCommentId: string): Promise; getStoredTempFile(key: string): Promise; setStoredTempFile(key: string, value: string): Promise; + storeFeedGuids(url: string, ...guid: string[]): Promise; + hasSeenFeed(url: string, ...guid: string[]): Promise; + hasSeenFeedGuid(url: string, guid: string): Promise; } \ No newline at end of file diff --git a/src/feeds/FeedReader.ts b/src/feeds/FeedReader.ts index 3c07cc77..c2760cad 100644 --- a/src/feeds/FeedReader.ts +++ b/src/feeds/FeedReader.ts @@ -1,21 +1,16 @@ -import { MatrixError } from "matrix-bot-sdk"; import { BridgeConfigFeeds } from "../config/Config"; import { ConnectionManager } from "../ConnectionManager"; import { FeedConnection } from "../Connections"; import { Logger } from "matrix-appservice-bridge"; import { MessageQueue } from "../MessageQueue"; - -import Ajv from "ajv"; -import axios, { AxiosResponse } from "axios"; +import axios from "axios"; import Metrics from "../Metrics"; -import UserAgent from "../UserAgent"; import { randomUUID } from "crypto"; -import { StatusCodes } from "http-status-codes"; -import { FormatUtil } from "../FormatUtil"; -import { JsRssChannel, parseFeed } from "../libRs"; +import { readFeed } from "../libRs"; +import { IBridgeStorageProvider } from "../Stores/StorageProvider"; +import UserAgent from "../UserAgent"; const log = new Logger("FeedReader"); - export class FeedError extends Error { constructor( public url: string, @@ -64,28 +59,6 @@ export interface FeedSuccess { url: string, } -interface AccountData { - [url: string]: string[], -} - -interface AccountDataStore { - getAccountData(type: string): Promise; - setAccountData(type: string, data: T): Promise; -} - -const accountDataSchema = { - type: 'object', - patternProperties: { - "https?://.+": { - type: 'array', - items: { type: 'string' }, - } - }, - additionalProperties: false, -}; -const ajv = new Ajv(); -const validateAccountData = ajv.compile(accountDataSchema); - function isNonEmptyString(input: unknown): input is string { return Boolean(input) && typeof input === 'string'; } @@ -109,35 +82,6 @@ function shuffle(array: T[]): T[] { } export class FeedReader { - /** - * Read a feed URL and parse it into a set of items. - * @param url The feed URL. - * @param headers Any headers to provide. - * @param timeoutMs How long to wait for the response, in milliseconds. - * @param parser The parser instance. If not provided, this creates a new parser. - * @returns The raw axios response, and the parsed feed. - */ - public static async fetchFeed( - url: string, - headers: Record, - timeoutMs: number, - httpClient = axios, - ): Promise<{ response: AxiosResponse, feed: JsRssChannel }> { - const response = await httpClient.get(url, { - headers: { - 'User-Agent': UserAgent, - ...headers, - }, - // We don't want to wait forever for the feed. - timeout: timeoutMs, - }); - - if (typeof response.data !== "string") { - throw Error('Unexpected response type'); - } - const feed = parseFeed(response.data); - return { response, feed }; - } private connections: FeedConnection[]; // ts should notice that we do in fact initialize it in constructor, but it doesn't (in this version) @@ -145,7 +89,6 @@ export class FeedReader { private feedQueue: string[] = []; - private seenEntries: Map = new Map(); // A set of last modified times for each url. private cacheTimes: Map = new Map(); @@ -156,19 +99,26 @@ export class FeedReader { static readonly seenEntriesEventType = "uk.half-shot.matrix-hookshot.feed.reader.seenEntries"; private shouldRun = true; - private timeout?: NodeJS.Timeout; + private readonly timeouts: (NodeJS.Timeout|undefined)[]; get sleepingInterval() { - return (this.config.pollIntervalSeconds * 1000) / (this.feedQueue.length || 1); + return ( + // Calculate the number of MS to wait in between feeds. + (this.config.pollIntervalSeconds * 1000) / (this.feedQueue.length || 1) + // And multiply by the number of concurrent readers + ) * this.config.pollConcurrency; } constructor( private readonly config: BridgeConfigFeeds, private readonly connectionManager: ConnectionManager, private readonly queue: MessageQueue, - private readonly accountDataStore: AccountDataStore, - private readonly httpClient = axios, + private readonly storage: IBridgeStorageProvider, ) { + // Ensure a fixed length array, + this.timeouts = new Array(config.pollConcurrency); + this.timeouts.fill(undefined); + Object.seal(this.timeouts); this.connections = this.connectionManager.getAllConnectionsOfType(FeedConnection); this.calculateFeedUrls(); connectionManager.on('new-connection', c => { @@ -187,16 +137,14 @@ export class FeedReader { log.debug('Loaded feed URLs:', this.observedFeedUrls); - void this.loadSeenEntries().then(() => { - for (let i = 0; i < config.pollConcurrency; i++) { - void this.pollFeeds(i); - } - }); + for (let i = 0; i < config.pollConcurrency; i++) { + void this.pollFeeds(i); + } } public stop() { - clearTimeout(this.timeout); this.shouldRun = false; + this.timeouts.forEach(t => clearTimeout(t)); } private calculateFeedUrls(): void { @@ -216,36 +164,6 @@ export class FeedReader { Metrics.feedsCountDeprecated.set(this.observedFeedUrls.size); } - private async loadSeenEntries(): Promise { - try { - const accountData = await this.accountDataStore.getAccountData(FeedReader.seenEntriesEventType).catch((err: MatrixError|unknown) => { - if (err instanceof MatrixError && err.statusCode === 404) { - return {} as AccountData; - } else { - throw err; - } - }); - if (!validateAccountData(accountData)) { - const errors = validateAccountData.errors?.map(e => `${e.instancePath} ${e.message}`) || ['No error reported']; - throw new Error(`Invalid account data: ${errors.join(', ')}`); - } - for (const url in accountData) { - this.seenEntries.set(url, accountData[url]); - } - } catch (err: unknown) { - log.error(`Failed to load seen feed entries from accountData: ${err}. This may result in skipped entries`); - // no need to wipe it manually, next saveSeenEntries() will make it right - } - } - - private async saveSeenEntries(): Promise { - const accountData: AccountData = {}; - for (const [url, guids] of this.seenEntries.entries()) { - accountData[url.toString()] = guids; - } - await this.accountDataStore.setAccountData(FeedReader.seenEntriesEventType, accountData); - } - /** * Poll a given feed URL for data, pushing any entries found into the message queue. * We also check the `cacheTimes` cache to see if the feed has recent entries that we can @@ -260,95 +178,80 @@ export class FeedReader { const { etag, lastModified } = this.cacheTimes.get(url) || {}; log.debug(`Checking for updates in ${url} (${etag ?? lastModified})`); try { - const { response, feed } = await FeedReader.fetchFeed( - url, - { - ...(lastModified && { 'If-Modified-Since': lastModified}), - ...(etag && { 'If-None-Match': etag}), - }, - // We don't want to wait forever for the feed. - this.config.pollTimeoutSeconds * 1000, - this.httpClient, - ); + const result = await readFeed(url, { + pollTimeoutSeconds: this.config.pollTimeoutSeconds, + etag, + lastModified, + userAgent: UserAgent, + }); // Store any entity tags/cache times. - if (response.headers.ETag) { - this.cacheTimes.set(url, { etag: response.headers.ETag}); - } else if (response.headers['Last-Modified']) { - this.cacheTimes.set(url, { lastModified: response.headers['Last-Modified'] }); + if (result.etag) { + this.cacheTimes.set(url, { etag: result.etag }); + } else if (result.lastModified) { + this.cacheTimes.set(url, { lastModified: result.lastModified }); } + const { feed } = result; let initialSync = false; - let seenGuids = this.seenEntries.get(url); - if (!seenGuids) { + if (!await this.storage.hasSeenFeed(url)) { initialSync = true; - seenGuids = []; seenEntriesChanged = true; // to ensure we only treat it as an initialSync once } - // migrate legacy, cleartext guids to their md5-hashed counterparts - seenGuids = seenGuids.map(guid => guid.startsWith('md5:') ? guid : this.hashGuid(guid)); - const seenGuidsSet = new Set(seenGuids); const newGuids = []; - log.debug(`Found ${feed.items.length} entries in ${url}`); + if (feed) { + // If undefined, we got a not-modified. + log.debug(`Found ${feed.items.length} entries in ${url}`); - for (const item of feed.items) { - // Find the first guid-like that looks like a string. - // Some feeds have a nasty habit of leading a empty tag there, making us parse it as garbage. - if (!item.hashId) { - log.error(`Could not determine guid for entry in ${url}, skipping`); - continue; + for (const item of feed.items) { + // Some feeds have a nasty habit of leading a empty tag there, making us parse it as garbage. + if (!item.hashId) { + log.error(`Could not determine guid for entry in ${url}, skipping`); + continue; + } + const hashId = `md5:${item.hashId}`; + newGuids.push(hashId); + + if (initialSync) { + log.debug(`Skipping entry ${item.id ?? hashId} since we're performing an initial sync`); + continue; + } + if (await this.storage.hasSeenFeedGuid(url, hashId)) { + log.debug('Skipping already seen entry', item.id ?? hashId); + continue; + } + const entry = { + feed: { + title: isNonEmptyString(feed.title) ? stripHtml(feed.title) : null, + url: url, + }, + title: isNonEmptyString(item.title) ? stripHtml(item.title) : null, + pubdate: item.pubdate ?? null, + summary: item.summary ?? null, + author: item.author ?? null, + link: item.link ?? null, + fetchKey + }; + + log.debug('New entry:', entry); + seenEntriesChanged = true; + + this.queue.push({ eventName: 'feed.entry', sender: 'FeedReader', data: entry }); } - const hashId = `md5:${item.hashId}`; - newGuids.push(hashId); - - if (initialSync) { - log.debug(`Skipping entry ${item.id ?? hashId} since we're performing an initial sync`); - continue; + + if (seenEntriesChanged) { + await this.storage.storeFeedGuids(url, ...newGuids); } - if (seenGuidsSet.has(hashId)) { - log.debug('Skipping already seen entry', item.id ?? hashId); - continue; - } - const entry = { - feed: { - title: isNonEmptyString(feed.title) ? stripHtml(feed.title) : null, - url: url, - }, - title: isNonEmptyString(item.title) ? stripHtml(item.title) : null, - pubdate: item.pubdate ?? null, - summary: item.summary ?? null, - author: item.author ?? null, - link: item.link ?? null, - fetchKey - }; - - log.debug('New entry:', entry); - seenEntriesChanged = true; - - this.queue.push({ eventName: 'feed.entry', sender: 'FeedReader', data: entry }); + } - - if (seenEntriesChanged) { - // Some RSS feeds can return a very small number of items then bounce - // back to their "normal" size, so we cannot just clobber the recent GUID list per request or else we'll - // forget what we sent and resend it. Instead, we'll keep 2x the max number of items that we've ever - // seen from this feed, up to a max of 10,000. - // Adopted from https://github.com/matrix-org/go-neb/blob/babb74fa729882d7265ff507b09080e732d060ae/services/rssbot/rssbot.go#L304 - const maxGuids = Math.min(Math.max(2 * newGuids.length, seenGuids.length), 10_000); - const newSeenItems = Array.from(new Set([ ...newGuids, ...seenGuids ]).values()).slice(0, maxGuids); - this.seenEntries.set(url, newSeenItems); - } - this.queue.push({ eventName: 'feed.success', sender: 'FeedReader', data: { url: url } }); + this.queue.push({ eventName: 'feed.success', sender: 'FeedReader', data: { url } }); // Clear any feed failures this.feedsFailingHttp.delete(url); this.feedsFailingParsing.delete(url); } catch (err: unknown) { - if (axios.isAxiosError(err)) { - // No new feed items, skip. - if (err.response?.status === StatusCodes.NOT_MODIFIED) { - return false; - } + // TODO: Proper Rust Type error. + if ((err as Error).message.includes('Failed to fetch feed due to HTTP')) { this.feedsFailingHttp.add(url); } else { this.feedsFailingParsing.add(url); @@ -383,7 +286,7 @@ export class FeedReader { if (url) { if (await this.pollFeed(url)) { - await this.saveSeenEntries(); + log.debug(`Feed changed and will be saved`); } const elapsed = Date.now() - fetchingStarted; Metrics.feedFetchMs.set(elapsed); @@ -399,15 +302,11 @@ export class FeedReader { log.debug(`No feeds available to poll for worker ${workerId}`); } - this.timeout = setTimeout(() => { + this.timeouts[workerId] = setTimeout(() => { if (!this.shouldRun) { return; } void this.pollFeeds(workerId); }, sleepFor); } - - private hashGuid(guid: string): string { - return `md5:${FormatUtil.hashId(guid)}`; - } } diff --git a/src/feeds/parser.rs b/src/feeds/parser.rs index c7f59ec8..623df0c6 100644 --- a/src/feeds/parser.rs +++ b/src/feeds/parser.rs @@ -1,7 +1,11 @@ -use std::str::FromStr; +use std::{str::FromStr, time::Duration}; use atom_syndication::{Error as AtomError, Feed, Person}; use napi::bindgen_prelude::{Error as JsError, Status}; +use reqwest::{ + header::{HeaderMap, HeaderValue}, + Method, StatusCode, +}; use rss::{Channel, Error as RssError}; use crate::format_util::hash_id; @@ -26,6 +30,23 @@ pub struct JsRssChannel { pub items: Vec, } +#[derive(Serialize, Debug, Deserialize)] +#[napi(object)] +pub struct ReadFeedOptions { + pub last_modified: Option, + pub etag: Option, + pub poll_timeout_seconds: i64, + pub user_agent: String, +} + +#[derive(Serialize, Debug, Deserialize)] +#[napi(object)] +pub struct FeedResult { + pub feed: Option, + pub etag: Option, + pub last_modified: Option, +} + fn parse_channel_to_js_result(channel: &Channel) -> JsRssChannel { JsRssChannel { title: channel.title().to_string(), @@ -145,3 +166,65 @@ pub fn js_parse_feed(xml: String) -> Result { Err(RssError::Eof) => Err(JsError::new(Status::Unknown, "Unexpected end of input")), } } + +#[napi(js_name = "readFeed")] +pub async fn js_read_feed(url: String, options: ReadFeedOptions) -> Result { + let client = reqwest::Client::new(); + let req = client + .request(Method::GET, url) + .timeout(Duration::from_secs( + options.poll_timeout_seconds.try_into().unwrap(), + )); + + let mut headers: HeaderMap = HeaderMap::new(); + + headers.append( + "User-Agent", + HeaderValue::from_str(&options.user_agent).unwrap(), + ); + + if let Some(last_modifed) = options.last_modified { + headers.append( + "If-Modified-Since", + HeaderValue::from_str(&last_modifed).unwrap(), + ); + } + if let Some(etag) = options.etag { + headers.append("If-None-Match", HeaderValue::from_str(&etag).unwrap()); + } + + match req.headers(headers).send().await { + Ok(res) => { + let res_headers = res.headers().clone(); + match res.status() { + StatusCode::OK => match res.text().await { + Ok(body) => match js_parse_feed(body) { + Ok(feed) => Ok(FeedResult { + feed: Some(feed), + etag: res_headers + .get("ETag") + .map(|v| v.to_str().unwrap()) + .map(|v| v.to_string()), + last_modified: res_headers + .get("Last-Modified") + .map(|v| v.to_str().unwrap()) + .map(|v| v.to_string()), + }), + Err(err) => Err(err), + }, + Err(err) => Err(JsError::new(Status::Unknown, err)), + }, + StatusCode::NOT_MODIFIED => Ok(FeedResult { + feed: None, + etag: None, + last_modified: None, + }), + status => Err(JsError::new( + Status::Unknown, + format!("Failed to fetch feed due to HTTP {}", status), + )), + } + } + Err(err) => Err(JsError::new(Status::Unknown, err)), + } +} diff --git a/tests/FeedReader.spec.ts b/tests/FeedReader.spec.ts index d1a37590..9b2de2fb 100644 --- a/tests/FeedReader.spec.ts +++ b/tests/FeedReader.spec.ts @@ -1,4 +1,3 @@ -import { AxiosResponse, AxiosStatic } from "axios"; import { expect } from "chai"; import EventEmitter from "events"; import { BridgeConfigFeeds } from "../src/config/Config"; @@ -6,6 +5,9 @@ import { ConnectionManager } from "../src/ConnectionManager"; import { IConnection } from "../src/Connections"; import { FeedEntry, FeedReader } from "../src/feeds/FeedReader"; import { MessageQueue, MessageQueueMessage } from "../src/MessageQueue"; +import { MemoryStorageProvider } from "../src/Stores/MemoryStorageProvider"; +import { Server, createServer } from 'http'; +import { AddressInfo } from "net"; class MockConnectionManager extends EventEmitter { constructor( @@ -37,38 +39,41 @@ class MockMessageQueue extends EventEmitter implements MessageQueue { } } -class MockHttpClient { - constructor(public response: AxiosResponse) {} - - get(): Promise { - return Promise.resolve(this.response); - } -} - -const FEED_URL = 'http://test/'; - -function constructFeedReader(feedResponse: () => {headers: Record, data: string}) { +async function constructFeedReader(feedResponse: () => {headers: Record, data: string}) { + const httpServer = await new Promise(resolve => { + const srv = createServer((_req, res) => { + res.writeHead(200); + const { headers, data } = feedResponse(); + Object.entries(headers).forEach(([key,value]) => { + res.setHeader(key, value); + }); + res.write(data); + res.end(); + }).listen(0, '127.0.0.1', () => { + resolve(srv); + }); + }); + const address = httpServer.address() as AddressInfo; + const feedUrl = `http://127.0.0.1:${address.port}/` const config = new BridgeConfigFeeds({ enabled: true, pollIntervalSeconds: 1, pollTimeoutSeconds: 1, }); - const cm = new MockConnectionManager([{ feedUrl: FEED_URL } as unknown as IConnection]) as unknown as ConnectionManager + const cm = new MockConnectionManager([{ feedUrl } as unknown as IConnection]) as unknown as ConnectionManager const mq = new MockMessageQueue(); + const storage = new MemoryStorageProvider(); + // Ensure we don't initial sync by storing a guid. + await storage.storeFeedGuids(feedUrl, '-test-guid-'); const feedReader = new FeedReader( - config, cm, mq, - { - getAccountData: () => Promise.resolve({ [FEED_URL]: [] } as unknown as T), - setAccountData: () => Promise.resolve(), - }, - new MockHttpClient({ ...feedResponse() } as AxiosResponse) as unknown as AxiosStatic, + config, cm, mq, storage, ); - return {config, cm, mq, feedReader}; + return {config, cm, mq, feedReader, feedUrl, httpServer}; } describe("FeedReader", () => { it("should correctly handle empty titles", async () => { - const { mq, feedReader} = constructFeedReader(() => ({ + const { mq, feedReader, httpServer } = await constructFeedReader(() => ({ headers: {}, data: ` @@ -84,6 +89,8 @@ describe("FeedReader", () => { ` })); + after(() => httpServer.close()); + const event: any = await new Promise((resolve) => { mq.on('pushed', (data) => { resolve(data); feedReader.stop() }); }); @@ -93,7 +100,7 @@ describe("FeedReader", () => { expect(event.data.title).to.equal(null); }); it("should handle RSS 2.0 feeds", async () => { - const { mq, feedReader} = constructFeedReader(() => ({ + const { mq, feedReader, httpServer } = await constructFeedReader(() => ({ headers: {}, data: ` @@ -118,6 +125,8 @@ describe("FeedReader", () => { ` })); + after(() => httpServer.close()); + const event: MessageQueueMessage = await new Promise((resolve) => { mq.on('pushed', (data) => { resolve(data); feedReader.stop() }); }); @@ -131,7 +140,7 @@ describe("FeedReader", () => { expect(event.data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000'); }); it("should handle RSS feeds with a permalink url", async () => { - const { mq, feedReader} = constructFeedReader(() => ({ + const { mq, feedReader, httpServer } = await constructFeedReader(() => ({ headers: {}, data: ` @@ -155,6 +164,8 @@ describe("FeedReader", () => { ` })); + after(() => httpServer.close()); + const event: MessageQueueMessage = await new Promise((resolve) => { mq.on('pushed', (data) => { resolve(data); feedReader.stop() }); }); @@ -168,7 +179,7 @@ describe("FeedReader", () => { expect(event.data.pubdate).to.equal('Sun, 6 Sep 2009 16:20:00 +0000'); }); it("should handle Atom feeds", async () => { - const { mq, feedReader} = constructFeedReader(() => ({ + const { mq, feedReader, httpServer } = await constructFeedReader(() => ({ headers: {}, data: ` @@ -196,6 +207,8 @@ describe("FeedReader", () => { ` })); + after(() => httpServer.close()); + const event: MessageQueueMessage = await new Promise((resolve) => { mq.on('pushed', (data) => { resolve(data); feedReader.stop() }); }); @@ -209,7 +222,7 @@ describe("FeedReader", () => { expect(event.data.pubdate).to.equal('Sat, 13 Dec 2003 18:30:02 +0000'); }); it("should not duplicate feed entries", async () => { - const { mq, feedReader} = constructFeedReader(() => ({ + const { mq, feedReader, httpServer, feedUrl } = await constructFeedReader(() => ({ headers: {}, data: ` @@ -227,11 +240,13 @@ describe("FeedReader", () => { ` })); + after(() => httpServer.close()); + const events: MessageQueueMessage[] = []; mq.on('pushed', (data) => { if (data.eventName === 'feed.entry') {events.push(data);} }); - await feedReader.pollFeed(FEED_URL); - await feedReader.pollFeed(FEED_URL); - await feedReader.pollFeed(FEED_URL); + await feedReader.pollFeed(feedUrl); + await feedReader.pollFeed(feedUrl); + await feedReader.pollFeed(feedUrl); feedReader.stop(); expect(events).to.have.lengthOf(1); }); diff --git a/yarn.lock b/yarn.lock index 7277a2a6..3a20b019 100644 --- a/yarn.lock +++ b/yarn.lock @@ -919,10 +919,10 @@ resolved "https://registry.yarnpkg.com/@mdn/browser-compat-data/-/browser-compat-data-4.1.14.tgz#45b45f2fcd8fe766950e5abc40efde94b4348efd" integrity sha512-pndsgd4jXIGcgWKPXkN5AL1rdwhgQpLXWyK25jb42SUaeujs/GhRK8+Q4W97RTiCirf/DoaahcTI/3Op6+/gfw== -"@napi-rs/cli@^2.2.0": - version "2.2.0" - resolved "https://registry.yarnpkg.com/@napi-rs/cli/-/cli-2.2.0.tgz#0129406192c2dfff6e8fc3de0c8be1d2ec286e3f" - integrity sha512-lXOKq0EZWztzHIlpXhKG0Nrv/PDZAl/yBsqQTG0aDfdjGCJudtPgWLR7zzaJoYzkkdFJo0r+teYYzgC+cXB4KQ== +"@napi-rs/cli@^2.13.2": + version "2.16.1" + resolved "https://registry.yarnpkg.com/@napi-rs/cli/-/cli-2.16.1.tgz#912e1169be6ff8bb5e1e22bb702adcc5e73e232b" + integrity sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA== "@nodelib/fs.scandir@2.1.5": version "2.1.5" @@ -3303,11 +3303,16 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@^1.14.0, follow-redirects@^1.14.4: +follow-redirects@^1.14.0: version "1.14.8" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.8.tgz#016996fb9a11a100566398b1c6839337d7bfa8fc" integrity sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA== +follow-redirects@^1.14.4: + version "1.15.2" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" + integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== + follow-redirects@^1.14.9: version "1.15.1" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.1.tgz#0ca6a452306c9b276e4d3127483e29575e207ad5" @@ -5145,14 +5150,7 @@ process-on-spawn@^1.0.0: dependencies: fromentries "^1.2.0" -prom-client@^14.0.1: - version "14.0.1" - resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-14.0.1.tgz#bdd9583e02ec95429677c0e013712d42ef1f86a8" - integrity sha512-HxTArb6fkOntQHoRGvv4qd/BkorjliiuO2uSWC2KC17MUTKYttWdDoXX/vxOhQdkoECEM9BBH0pj2l8G8kev6w== - dependencies: - tdigest "^0.1.1" - -prom-client@^14.1.0: +prom-client@^14.1.0, prom-client@^14.2.0: version "14.2.0" resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-14.2.0.tgz#ca94504e64156f6506574c25fb1c34df7812cf11" integrity sha512-sF308EhTenb/pDRPakm+WgiN+VdM/T1RaHj1x+MvAuT8UiQP8JmOEbxVqtkbfR4LrvOg5n7ic01kRBDGXjYikA== @@ -6027,10 +6025,10 @@ typedarray-to-buffer@^3.1.5: dependencies: is-typedarray "^1.0.0" -typescript@^5.0.4: - version "5.0.4" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.0.4.tgz#b217fd20119bd61a94d4011274e0ab369058da3b" - integrity sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw== +typescript@^5.1.3: + version "5.1.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.1.3.tgz#8d84219244a6b40b6fb2b33cc1c062f715b9e826" + integrity sha512-XH627E9vkeqhlZFQuL+UsyAXEnibT0kWR2FWONlr4sTjvxyJYnyefgrkyECLzM5NenmKzRAy2rR/OlYLA1HkZw== uc.micro@^1.0.1, uc.micro@^1.0.5: version "1.0.6"