go/sliding-sync
go/sliding-sync
1
0
Fork 0
mirror of https://github.com/matrix-org/sliding-sync.git synced 2025-03-10 13:37:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate data to JSONB, update device data query

Browse Source
This commit is contained in:
Till Faelligen 2023-08-02 14:00:42 +02:00
parent a085e1fc04
commit 7b3ef7ee28
No known key found for this signature in database
GPG Key ID: ACCDC9606D472758
3 changed files with 32 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
state/device_data_table.go
View File

@ -28,7 +28,7 @@ func NewDeviceDataTable(db *sqlx.DB) *DeviceDataTable {
CREATE TABLE IF NOT EXISTS syncv3_device_data (
user_id TEXT NOT NULL,
device_id TEXT NOT NULL,
data BYTEA NOT NULL,
data JSONB NOT NULL,
UNIQUE(user_id, device_id)
);
-- Set the fillfactor to 90%, to allow for HOT updates (e.g. we only
@ -77,12 +77,12 @@ func (t *DeviceDataTable) Select(userID, deviceID string, swap bool) (result *in
return nil
}
// re-marshal and write
data, err := json.Marshal(writeBack)
if err != nil {
return err
}
_, err = txn.Exec(`UPDATE syncv3_device_data SET data=$1 WHERE user_id=$2 AND device_id=$3`, data, userID, deviceID)
// Some JSON juggling in Postgres ahead. This is to avoid pushing
// DeviceLists.Sent -> DeviceLists.New over the wire again.
_, err = txn.Exec(`UPDATE syncv3_device_data SET data = jsonb_set(
jsonb_set(data, '{dl,s}', data->'dl'->'n', false), -- move 'dl.n' -> 'dl.s'
'{dl,n}', '{}', false) || '{"c":0}' -- clear 'dl.n' and set the changed bits to 0
WHERE user_id = $1 AND device_id = $2`, userID, deviceID)
return err
})
return

15
state/migrations/20230802121023_device_data_jsonb.sql Normal file
View File

@ -0,0 +1,15 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE IF EXISTS syncv3_device_data ADD COLUMN IF NOT EXISTS dataj JSONB;
UPDATE syncv3_device_data SET dataj = encode(data, 'escape')::JSONB;
ALTER TABLE IF EXISTS syncv3_device_data DROP COLUMN IF EXISTS data;
ALTER TABLE IF EXISTS syncv3_device_data RENAME COLUMN dataj TO data;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE IF EXISTS syncv3_device_data ADD COLUMN IF NOT EXISTS datab BYTEA;
UPDATE syncv3_device_data SET datab = (data::TEXT)::BYTEA;
ALTER TABLE IF EXISTS syncv3_device_data DROP COLUMN IF EXISTS data;
ALTER TABLE IF EXISTS syncv3_device_data RENAME COLUMN datab TO data;
-- +goose StatementEnd

20
v3.go
View File

@ -10,16 +10,15 @@ import (
"time"
"github.com/getsentry/sentry-go"
"github.com/jmoiron/sqlx"
"github.com/pressly/goose/v3"
"github.com/gorilla/mux"
"github.com/jmoiron/sqlx"
"github.com/matrix-org/sliding-sync/internal"
"github.com/matrix-org/sliding-sync/pubsub"
"github.com/matrix-org/sliding-sync/state"
"github.com/matrix-org/sliding-sync/sync2"
"github.com/matrix-org/sliding-sync/sync2/handler2"
"github.com/matrix-org/sliding-sync/sync3/handler"
"github.com/pressly/goose/v3"
"github.com/rs/zerolog"
"github.com/rs/zerolog/hlog"
)
@ -93,13 +92,6 @@ func Setup(destHomeserver, postgresURI, secret string, opts Opts) (*handler2.Han
logger.Panic().Err(err).Str("uri", postgresURI).Msg("failed to open SQL DB")
}
// Automatically execute migrations
goose.SetBaseFS(EmbedMigrations)
err = goose.Up(db.DB, "state/migrations", goose.WithAllowMissing())
if err != nil {
logger.Panic().Err(err).Msg("failed to execute migrations")
}
if opts.DBMaxConns > 0 {
// https://github.com/go-sql-driver/mysql#important-settings
// "db.SetMaxIdleConns() is recommended to be set same to db.SetMaxOpenConns(). When it is smaller
@ -112,6 +104,14 @@ func Setup(destHomeserver, postgresURI, secret string, opts Opts) (*handler2.Han
}
store := state.NewStorageWithDB(db)
storev2 := sync2.NewStoreWithDB(db, secret)
// Automatically execute migrations
goose.SetBaseFS(EmbedMigrations)
err = goose.Up(db.DB, "state/migrations", goose.WithAllowMissing())
if err != nil {
logger.Panic().Err(err).Msg("failed to execute migrations")
}
bufferSize := 50
deviceDataUpdateFrequency := time.Second
if opts.TestingSynchronousPubsub {