Add filter tables; remove alice dep

This commit is contained in:
Kegan Dougal 2021-07-21 12:12:57 +01:00
parent 1178f7381e
commit 7f8d84d79e
7 changed files with 121 additions and 25 deletions

View File

@ -123,13 +123,18 @@ in the first 5 room members for example, then you can set `limit: 5` and ignore
clients which are keeping up-to-date with room members and those who are just interested in the first page
of results, clients MUST specify `p.since = "1"` to indicate the first page of results. This makes `"1"` a
sentinel constant value in pagination to indicate "the first page of results". If you don't do this, the server
will just send back deltas from the last `since` value, which won't contain the first 5 members.
will just send back deltas from the last `since` value, which won't necessarily contain the first 5 members
of your chosen sort order (e.g alphabetically).
## Streams
### Typing Stream
TODO
Request Parameters:
- `room_id` (string): The room ID to track typing notifications in.
Response fields:
- `events` ([]EDU): A list of typing EDUs.
### To Device Stream

1
go.mod
View File

@ -5,7 +5,6 @@ go 1.14
require (
github.com/gorilla/mux v1.8.0
github.com/jmoiron/sqlx v1.3.3
github.com/justinas/alice v1.2.0
github.com/lib/pq v1.10.1
github.com/matrix-org/gomatrixserverlib v0.0.0-20210510192107-124228cb9548
github.com/rs/zerolog v1.21.0

18
go.sum
View File

@ -1,24 +1,28 @@
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k=
github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
github.com/jmoiron/sqlx v1.3.3 h1:j82X0bf7oQ27XeqxicSZsTU5suPwKElg3oyxNn43iTk=
github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
github.com/justinas/alice v1.2.0 h1:+MHSA/vccVCF4Uq37S42jwlkvI2Xzl7zTPCN5BnZNVo=
github.com/justinas/alice v1.2.0/go.mod h1:fN5HRH/reO/zrUflLfTN43t3vXvKzvZIENsNEe7i7qA=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo=
@ -29,10 +33,14 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210510192107-124228cb9548 h1:h4
github.com/matrix-org/gomatrixserverlib v0.0.0-20210510192107-124228cb9548/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU=
github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo=
github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA=
github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
@ -43,6 +51,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc=
github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
@ -64,9 +73,11 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -85,8 +96,11 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM=
gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

View File

@ -27,7 +27,36 @@ type FilterRoomList struct {
// Clients should limit the size to how much they can display (e.g 70 chars)
RoomNameSize int
// True to include the MXC URI for the room avatar, if it has one.
IncludeRoomAvatarMXC bool
IncludeRoomAvatarMXC *bool
}
// Combine two filters together. A new filter is returned.
func (f *FilterRoomList) Combine(new *FilterRoomList) *FilterRoomList {
combined := &FilterRoomList{}
// nil slice != 0 length slice
if new.SummaryEventTypes != nil {
combined.SummaryEventTypes = new.SummaryEventTypes
} else {
combined.SummaryEventTypes = f.SummaryEventTypes
}
if new.EntriesPerBatch != 0 {
combined.EntriesPerBatch = new.EntriesPerBatch
} else {
combined.EntriesPerBatch = f.EntriesPerBatch
}
if new.RoomNameSize != 0 {
combined.RoomNameSize = new.RoomNameSize
} else {
combined.RoomNameSize = f.RoomNameSize
}
// pointer to bool to indicate absence
if new.IncludeRoomAvatarMXC != nil {
combined.IncludeRoomAvatarMXC = new.IncludeRoomAvatarMXC
} else {
combined.IncludeRoomAvatarMXC = f.IncludeRoomAvatarMXC
}
return combined
}
type ControlMessageRoomList struct {

11
sync3/request.go Normal file
View File

@ -0,0 +1,11 @@
package sync3
import "github.com/matrix-org/sync-v3/streams"
// Request represents a sync v3 request
//
// A request is made by the combination of the client HTTP request parameters and the stored filters
// on the server.
type Request struct {
RoomList streams.FilterRoomList `json:"room_list"`
}

View File

@ -2,6 +2,7 @@ package sync3
import (
"database/sql"
"encoding/json"
"os"
"github.com/jmoiron/sqlx"
@ -54,6 +55,13 @@ func NewSessions(postgresURI string) *Sessions {
device_id TEXT PRIMARY KEY,
since TEXT NOT NULL
);
CREATE SEQUENCE IF NOT EXISTS syncv3_filter_id_seq;
CREATE TABLE IF NOT EXISTS syncv3_filters (
filter_id BIGINT PRIMARY KEY DEFAULT nextval('syncv3_filter_id_seq'),
session_id BIGINT NOT NULL,
api_name TEXT NOT NULL,
req_params TEXT NOT NULL
);
`)
return &Sessions{
db: db,
@ -152,3 +160,18 @@ func (s *Sessions) UpdateUserIDForDevice(deviceID, userID string) error {
_, err := s.db.Exec(`UPDATE syncv3_sessions_v2devices SET user_id = $1 WHERE device_id = $2`, userID, deviceID)
return err
}
// Update a filter with the latest delta from the client. If filters with this apiName already exists for this session,
// the filter will be updated and a new filter ID will be returned. If the filter does not exist yet, a new filter ID
// will be made. This ensures we never delete filter IDs for a session to allow for replayability of requests. Filters
// are only deleted when the session is deleted.
func (s *Sessions) UpdateFilter(sessionID int64, apiName string, jsonBlob json.RawMessage) (filterID int64, err error) {
return 0, nil
}
// Filters returns the filters for the session ID and filter IDs given. If a filter ID is given which is unknown, an
// error is returned as filters should always be known to the server. Callers should call UpdateFilter first with the
// clients new filters before calling this function, to ensure that their filter deltas have been applied and persisted.
func (s *Sessions) Filters(sessionID int64, filterIDs []int64) (*Request, error) {
return nil, nil
}

53
v3.go
View File

@ -12,7 +12,6 @@ import (
"time"
"github.com/gorilla/mux"
"github.com/justinas/alice"
"github.com/matrix-org/sync-v3/state"
"github.com/matrix-org/sync-v3/streams"
"github.com/matrix-org/sync-v3/sync2"
@ -26,22 +25,21 @@ var log = zerolog.New(os.Stdout).With().Timestamp().Logger().Output(zerolog.Cons
TimeFormat: "15:04:05",
})
type server struct {
chain []func(next http.Handler) http.Handler
final http.Handler
}
func (s *server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
h := s.final
for i := range s.chain {
h = s.chain[len(s.chain)-1-i](h)
}
h.ServeHTTP(w, req)
}
// RunSyncV3Server is the main entry point to the server
func RunSyncV3Server(destinationServer, bindAddr, postgresDBURI string) {
// setup logging
c := alice.New()
c = c.Append(hlog.NewHandler(log))
c = c.Append(hlog.AccessHandler(func(r *http.Request, status, size int, duration time.Duration) {
hlog.FromRequest(r).Info().
Str("method", r.Method).
Int("status", status).
Int("size", size).
Dur("duration", duration).
Str("since", r.URL.Query().Get("since")).
Msg("")
}))
c = c.Append(hlog.RemoteAddrHandler("ip"))
// dependency inject all components together
sh := &SyncV3Handler{
V2: &sync2.Client{
@ -59,11 +57,27 @@ func RunSyncV3Server(destinationServer, bindAddr, postgresDBURI string) {
// HTTP path routing
r := mux.NewRouter()
r.Handle("/_matrix/client/v3/sync", sh)
handler := c.Then(r)
srv := &server{
chain: []func(next http.Handler) http.Handler{
hlog.NewHandler(log),
hlog.AccessHandler(func(r *http.Request, status, size int, duration time.Duration) {
hlog.FromRequest(r).Info().
Str("method", r.Method).
Int("status", status).
Int("size", size).
Dur("duration", duration).
Str("since", r.URL.Query().Get("since")).
Msg("")
}),
hlog.RemoteAddrHandler("ip"),
},
final: r,
}
// Block forever
log.Info().Msgf("listening on %s", bindAddr)
if err := http.ListenAndServe(bindAddr, handler); err != nil {
if err := http.ListenAndServe(bindAddr, srv); err != nil {
log.Fatal().Err(err).Msg("failed to listen and serve")
}
}
@ -123,10 +137,11 @@ func (h *SyncV3Handler) serve(w http.ResponseWriter, req *http.Request) *handler
// TODO: read filters
// TODO: read streams
f := false
filter := &streams.FilterRoomList{
EntriesPerBatch: 5,
RoomNameSize: 70,
IncludeRoomAvatarMXC: false,
IncludeRoomAvatarMXC: &f,
SummaryEventTypes: []string{"m.room.message", "m.room.member"},
}
stream := streams.NewRoomList(h.Storage)
@ -185,7 +200,7 @@ func (h *SyncV3Handler) getOrCreateSession(req *http.Request) (*sync3.Session, *
// we need to work out the user ID to do membership queries
userID, err := h.userIDFromRequest(req)
if err != nil {
log.Warn().Err(err).Msg("failed to work out user ID from request")
log.Warn().Err(err).Msg("failed to work out user ID from request, is the authorization header valid?")
return nil, nil, &handlerError{400, err}
}
session.UserID = userID