sliding-sync/state/snapshot_table.go

96 lines
2.7 KiB
Go
Raw Normal View History

2021-05-26 20:01:56 +01:00
package state
import (
"fmt"
2021-05-26 20:01:56 +01:00
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
)
type SnapshotRow struct {
SnapshotID int64 `db:"snapshot_id"`
RoomID string `db:"room_id"`
OtherEvents pq.Int64Array `db:"events"`
MembershipEvents pq.Int64Array `db:"membership_events"`
2021-05-26 20:01:56 +01:00
}
2021-05-27 19:20:36 +01:00
// SnapshotTable stores room state snapshots. Each snapshot has a unique numeric ID.
// Not every event will be associated with a snapshot.
2021-05-26 20:01:56 +01:00
type SnapshotTable struct {
db *sqlx.DB
2021-05-26 20:01:56 +01:00
}
2021-05-27 19:20:36 +01:00
func NewSnapshotsTable(db *sqlx.DB) *SnapshotTable {
2021-05-26 20:01:56 +01:00
// make sure tables are made
db.MustExec(`
CREATE SEQUENCE IF NOT EXISTS syncv3_snapshots_seq;
CREATE TABLE IF NOT EXISTS syncv3_snapshots (
snapshot_id BIGINT PRIMARY KEY DEFAULT nextval('syncv3_snapshots_seq'),
room_id TEXT NOT NULL,
2021-07-21 10:30:04 +01:00
events BIGINT[] NOT NULL,
membership_events BIGINT[] NOT NULL,
2021-07-21 10:30:04 +01:00
UNIQUE(snapshot_id, room_id)
2021-05-26 20:01:56 +01:00
);
`)
return &SnapshotTable{db}
}
add extensions for typing and receipts; bugfixes and additional perf improvements Features: - Add `typing` extension. - Add `receipts` extension. - Add comprehensive prometheus `/metrics` activated via `SYNCV3_PROM`. - Add `SYNCV3_PPROF` support. - Add `by_notification_level` sort order. - Add `include_old_rooms` support. - Add support for `$ME` and `$LAZY`. - Add correct filtering when `*,*` is used as `required_state`. - Add `num_live` to each room response to indicate how many timeline entries are live. Bug fixes: - Use a stricter comparison function on ranges: fixes an issue whereby UTs fail on go1.19 due to change in sorting algorithm. - Send back an `errcode` on HTTP errors (e.g expired sessions). - Remove `unsigned.txn_id` on insertion into the DB. Otherwise other users would see other users txn IDs :( - Improve range delta algorithm: previously it didn't handle cases like `[0,20] -> [20,30]` and would panic. - Send HTTP 400 for invalid range requests. - Don't publish no-op unread counts which just adds extra noise. - Fix leaking DB connections which could eventually consume all available connections. - Ensure we always unblock WaitUntilInitialSync even on invalid access tokens. Other code relies on WaitUntilInitialSync() actually returning at _some_ point e.g on startup we have N workers which bound the number of concurrent pollers made at any one time, we need to not just hog a worker forever. Improvements: - Greatly improve startup times of sync3 handlers by improving `JoinedRoomsTracker`: a modest amount of data would take ~28s to create the handler, now it takes 4s. - Massively improve initial initial v3 sync times, by refactoring `JoinedRoomsTracker`, from ~47s to <1s. - Add `SlidingSyncUntil...` in tests to reduce races. - Tweak the API shape of JoinedUsersForRoom to reduce state block processing time for large rooms from 63s to 39s. - Add trace task for initial syncs. - Include the proxy version in UA strings. - HTTP errors now wait 1s before returning to stop clients tight-looping on error. - Pending event buffer is now 2000. - Index the room ID first to cull the most events when returning timeline entries. Speeds up `SelectLatestEventsBetween` by a factor of 8. - Remove cancelled `m.room_key_requests` from the to-device inbox. Cuts down the amount of events in the inbox by ~94% for very large (20k+) inboxes, ~50% for moderate sized (200 events) inboxes. Adds book-keeping to remember the unacked to-device position for each client.
2022-12-14 18:53:55 +00:00
func (t *SnapshotTable) CurrentSnapshots(txn *sqlx.Tx) (map[string][]int64, error) {
rows, err := txn.Query(
`SELECT syncv3_rooms.room_id, events, membership_events FROM syncv3_snapshots JOIN syncv3_rooms ON syncv3_snapshots.snapshot_id = syncv3_rooms.current_snapshot_id`,
)
if err != nil {
return nil, err
}
result := make(map[string][]int64)
defer rows.Close()
for rows.Next() {
var eventNIDs pq.Int64Array
var memberEventNIDs pq.Int64Array
var roomID string
if err = rows.Scan(&roomID, &eventNIDs, &memberEventNIDs); err != nil {
return nil, err
}
result[roomID] = append(eventNIDs, memberEventNIDs...)
}
return result, nil
2021-05-26 20:01:56 +01:00
}
// Select a row based on its snapshot ID.
2021-08-06 15:39:36 +01:00
func (s *SnapshotTable) Select(txn *sqlx.Tx, snapshotID int64) (row SnapshotRow, err error) {
if snapshotID == 0 {
err = fmt.Errorf("SnapshotTable.Select: snapshot ID requested is 0")
return
}
2021-05-27 19:20:36 +01:00
err = txn.Get(&row, `SELECT * FROM syncv3_snapshots WHERE snapshot_id = $1`, snapshotID)
2021-05-26 20:01:56 +01:00
return
}
// Insert the row. Modifies SnapshotID to be the inserted primary key.
2021-05-27 19:20:36 +01:00
func (s *SnapshotTable) Insert(txn *sqlx.Tx, row *SnapshotRow) error {
2021-08-06 15:39:36 +01:00
var id int64
if row.MembershipEvents == nil {
row.MembershipEvents = []int64{}
}
if row.OtherEvents == nil {
row.OtherEvents = []int64{}
}
err := txn.QueryRow(
`INSERT INTO syncv3_snapshots(room_id, events, membership_events) VALUES($1, $2, $3) RETURNING snapshot_id`,
row.RoomID, row.OtherEvents, row.MembershipEvents,
).Scan(&id)
2021-05-26 20:01:56 +01:00
row.SnapshotID = id
return err
}
2021-05-27 19:20:36 +01:00
// Delete the snapshot IDs given
2021-08-06 15:39:36 +01:00
func (s *SnapshotTable) Delete(txn *sqlx.Tx, snapshotIDs []int64) error {
query, args, err := sqlx.In(`DELETE FROM syncv3_snapshots WHERE snapshot_id = ANY(?)`, pq.Int64Array(snapshotIDs))
2021-05-27 19:20:36 +01:00
if err != nil {
return err
}
query = txn.Rebind(query)
_, err = txn.Exec(query, args...)
return err
}