sliding-sync/internal/required_state_map.go

94 lines
2.8 KiB
Go
Raw Permalink Normal View History

add extensions for typing and receipts; bugfixes and additional perf improvements Features: - Add `typing` extension. - Add `receipts` extension. - Add comprehensive prometheus `/metrics` activated via `SYNCV3_PROM`. - Add `SYNCV3_PPROF` support. - Add `by_notification_level` sort order. - Add `include_old_rooms` support. - Add support for `$ME` and `$LAZY`. - Add correct filtering when `*,*` is used as `required_state`. - Add `num_live` to each room response to indicate how many timeline entries are live. Bug fixes: - Use a stricter comparison function on ranges: fixes an issue whereby UTs fail on go1.19 due to change in sorting algorithm. - Send back an `errcode` on HTTP errors (e.g expired sessions). - Remove `unsigned.txn_id` on insertion into the DB. Otherwise other users would see other users txn IDs :( - Improve range delta algorithm: previously it didn't handle cases like `[0,20] -> [20,30]` and would panic. - Send HTTP 400 for invalid range requests. - Don't publish no-op unread counts which just adds extra noise. - Fix leaking DB connections which could eventually consume all available connections. - Ensure we always unblock WaitUntilInitialSync even on invalid access tokens. Other code relies on WaitUntilInitialSync() actually returning at _some_ point e.g on startup we have N workers which bound the number of concurrent pollers made at any one time, we need to not just hog a worker forever. Improvements: - Greatly improve startup times of sync3 handlers by improving `JoinedRoomsTracker`: a modest amount of data would take ~28s to create the handler, now it takes 4s. - Massively improve initial initial v3 sync times, by refactoring `JoinedRoomsTracker`, from ~47s to <1s. - Add `SlidingSyncUntil...` in tests to reduce races. - Tweak the API shape of JoinedUsersForRoom to reduce state block processing time for large rooms from 63s to 39s. - Add trace task for initial syncs. - Include the proxy version in UA strings. - HTTP errors now wait 1s before returning to stop clients tight-looping on error. - Pending event buffer is now 2000. - Index the room ID first to cull the most events when returning timeline entries. Speeds up `SelectLatestEventsBetween` by a factor of 8. - Remove cancelled `m.room_key_requests` from the to-device inbox. Cuts down the amount of events in the inbox by ~94% for very large (20k+) inboxes, ~50% for moderate sized (200 events) inboxes. Adds book-keeping to remember the unacked to-device position for each client.
2022-12-14 18:53:55 +00:00
package internal
const StateKeyLazy = "$LAZY"
type RequiredStateMap struct {
eventTypesWithWildcardStateKeys map[string]struct{}
stateKeysForWildcardEventType []string
eventTypeToStateKeys map[string][]string
allState bool
lazyLoading bool
}
func NewRequiredStateMap(eventTypesWithWildcardStateKeys map[string]struct{},
stateKeysForWildcardEventType []string,
eventTypeToStateKeys map[string][]string,
allState, lazyLoading bool) *RequiredStateMap {
return &RequiredStateMap{
eventTypesWithWildcardStateKeys: eventTypesWithWildcardStateKeys,
stateKeysForWildcardEventType: stateKeysForWildcardEventType,
eventTypeToStateKeys: eventTypeToStateKeys,
allState: allState,
lazyLoading: lazyLoading,
}
}
func (rsm *RequiredStateMap) IsLazyLoading() bool {
return rsm.lazyLoading
}
func (rsm *RequiredStateMap) Include(evType, stateKey string) bool {
if rsm.allState {
// "additional entries FILTER OUT the returned set of state events. These additional entries cannot use '*' themselves."
includedStateKeys := rsm.eventTypeToStateKeys[evType]
if len(includedStateKeys) > 0 {
for _, sk := range includedStateKeys {
if sk == stateKey {
return true
}
}
return false
}
return true
}
// check if we should include this event due to wildcard event types
for _, sk := range rsm.stateKeysForWildcardEventType {
if sk == stateKey || sk == "*" {
return true
}
}
// check if we should include this event due to wildcard state keys
for et := range rsm.eventTypesWithWildcardStateKeys {
if et == evType {
return true
}
}
// check if we should include this event due to exact type/state key match
for _, sk := range rsm.eventTypeToStateKeys[evType] {
if sk == stateKey {
return true
}
}
return false
}
func (rsm *RequiredStateMap) Empty() bool {
return !rsm.allState && !rsm.lazyLoading &&
len(rsm.eventTypeToStateKeys) == 0 &&
len(rsm.stateKeysForWildcardEventType) == 0 &&
len(rsm.eventTypesWithWildcardStateKeys) == 0
}
add extensions for typing and receipts; bugfixes and additional perf improvements Features: - Add `typing` extension. - Add `receipts` extension. - Add comprehensive prometheus `/metrics` activated via `SYNCV3_PROM`. - Add `SYNCV3_PPROF` support. - Add `by_notification_level` sort order. - Add `include_old_rooms` support. - Add support for `$ME` and `$LAZY`. - Add correct filtering when `*,*` is used as `required_state`. - Add `num_live` to each room response to indicate how many timeline entries are live. Bug fixes: - Use a stricter comparison function on ranges: fixes an issue whereby UTs fail on go1.19 due to change in sorting algorithm. - Send back an `errcode` on HTTP errors (e.g expired sessions). - Remove `unsigned.txn_id` on insertion into the DB. Otherwise other users would see other users txn IDs :( - Improve range delta algorithm: previously it didn't handle cases like `[0,20] -> [20,30]` and would panic. - Send HTTP 400 for invalid range requests. - Don't publish no-op unread counts which just adds extra noise. - Fix leaking DB connections which could eventually consume all available connections. - Ensure we always unblock WaitUntilInitialSync even on invalid access tokens. Other code relies on WaitUntilInitialSync() actually returning at _some_ point e.g on startup we have N workers which bound the number of concurrent pollers made at any one time, we need to not just hog a worker forever. Improvements: - Greatly improve startup times of sync3 handlers by improving `JoinedRoomsTracker`: a modest amount of data would take ~28s to create the handler, now it takes 4s. - Massively improve initial initial v3 sync times, by refactoring `JoinedRoomsTracker`, from ~47s to <1s. - Add `SlidingSyncUntil...` in tests to reduce races. - Tweak the API shape of JoinedUsersForRoom to reduce state block processing time for large rooms from 63s to 39s. - Add trace task for initial syncs. - Include the proxy version in UA strings. - HTTP errors now wait 1s before returning to stop clients tight-looping on error. - Pending event buffer is now 2000. - Index the room ID first to cull the most events when returning timeline entries. Speeds up `SelectLatestEventsBetween` by a factor of 8. - Remove cancelled `m.room_key_requests` from the to-device inbox. Cuts down the amount of events in the inbox by ~94% for very large (20k+) inboxes, ~50% for moderate sized (200 events) inboxes. Adds book-keeping to remember the unacked to-device position for each client.
2022-12-14 18:53:55 +00:00
// work out what to ask the storage layer: if we have wildcard event types we need to pull all
// room state and cannot only pull out certain event types. If we have wildcard state keys we
// need to use an empty list for state keys.
func (rsm *RequiredStateMap) QueryStateMap() map[string][]string {
queryStateMap := make(map[string][]string)
if rsm.allState {
return queryStateMap
}
if len(rsm.stateKeysForWildcardEventType) == 0 { // no wildcard event types
for evType, stateKeys := range rsm.eventTypeToStateKeys {
if evType == "m.room.member" && rsm.lazyLoading {
queryStateMap[evType] = nil
} else {
queryStateMap[evType] = stateKeys
}
}
for evType := range rsm.eventTypesWithWildcardStateKeys {
queryStateMap[evType] = nil
}
}
return queryStateMap
}