From c500a2be38afcbb5688537d97c7c3ee30a57dba4 Mon Sep 17 00:00:00 2001 From: Julio Capote Date: Fri, 30 Dec 2022 21:20:02 -0500 Subject: parse and persist feeds from handlers --- README.md | 12 ++++- cgi/servers.go | 38 ++++---------- go.mod | 4 ++ go.sum | 22 ++++++++ http/server.go | 13 ++--- models/activity_streams_object.go | 54 ------------------- models/outbox_item.go | 56 ++++++++++++++++++++ models/persister.go | 12 +++-- registry/registry.go | 10 ++-- sample-cgi-handler.sh | 17 +++--- views/outbox.go | 107 +++++++++++++++----------------------- 11 files changed, 167 insertions(+), 178 deletions(-) delete mode 100644 models/activity_streams_object.go create mode 100644 models/outbox_item.go diff --git a/README.md b/README.md index 3c2dcd0..094e6ce 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,19 @@ Communique uses CGI as the API between itself and your scripts. These scripts ar ## Response API +> The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL +> NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and +> "OPTIONAL" in this document are to be interpreted as described in +> RFC 2119. + Your CGI script MUST emit a valid CGI response. -Your CGI script reponse MUST be of content type `application/activity+json`. +Your CGI script reponse MUST be of one of these content types: + +* `application/atom+xml` +* `application/rss+xml` -Your CGI script reponse MUST be an `OrderedCollection` of ActivityStream `Object`s. +Your CGI script reponse MUST be an RSS or ATOM feed (or at least something [gofeed](https://github.com/mmcdole/gofeed) can parse) See `sample-cgi-handler.sh` for an example diff --git a/cgi/servers.go b/cgi/servers.go index f5652ed..c751135 100644 --- a/cgi/servers.go +++ b/cgi/servers.go @@ -2,7 +2,6 @@ package cgi import ( "context" - "encoding/json" "fmt" "io/ioutil" "net" @@ -14,8 +13,7 @@ import ( "git.capotej.com/capotej/communique/config" "git.capotej.com/capotej/communique/models" - "github.com/go-fed/activity/streams" - "github.com/go-fed/activity/streams/vocab" + "github.com/mmcdole/gofeed" "go.uber.org/zap" ) @@ -83,35 +81,17 @@ func startTicker(h config.Handler, persister *models.Persister, log *zap.Sugared } func processTick(h config.Handler, output []byte, persister *models.Persister, log *zap.SugaredLogger) error { - var m map[string]interface{} - err := json.Unmarshal(output, &m) + fp := gofeed.NewParser() + fp.ParseString(string(output)) + feed, err := fp.ParseString(string(output)) if err != nil { - return fmt.Errorf("could not unmarshal JSON: %w", err) + return err } - log.Debugf("processTick map is %+v", m) - var coll vocab.ActivityStreamsOrderedCollection - resolver, err := streams.NewJSONResolver(func(c context.Context, a vocab.ActivityStreamsOrderedCollection) error { - // Example: store the article in the enclosing scope, for later. - coll = a - // We could pass an error back up, if desired. - return nil - }) - ctx := context.Background() - err = resolver.Resolve(ctx, m) - if err != nil { - return fmt.Errorf("could not resolve JSON: %w", err) - } - orderedProp := coll.GetActivityStreamsOrderedItems() - for iter := orderedProp.Begin(); iter != orderedProp.End(); iter = iter.Next() { - contentType := iter.GetType().GetTypeName() - contentNote := iter.GetActivityStreamsNote() // TODO make this configurable since it cant be dynamic - - jsonMap, _ := streams.Serialize(contentNote) - jsonIter, _ := json.Marshal(jsonMap) - log.Debugf("found object %s of type %s in activity stream", jsonIter, contentType) - localNote := models.NewActivityStreamsObject(contentNote, h) - err = persister.Store(localNote) + for _, v := range feed.Items { + log.Debugf("found content '%s'", v.Content) + outboxItem := models.CreateOutboxItem(h, []byte(v.Content)) + err = persister.Store(outboxItem) if err != nil { return err } diff --git a/go.mod b/go.mod index 8c8e5b9..c8e5410 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require github.com/BurntSushi/toml v1.2.1 require ( github.com/Jeffail/gabs/v2 v2.6.1 // indirect + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dgraph-io/badger/v3 v3.2103.5 // indirect @@ -28,6 +30,8 @@ require ( github.com/klauspost/compress v1.15.13 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mmcdole/gofeed v1.1.3 // indirect + github.com/mmcdole/goxpp v1.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect diff --git a/go.sum b/go.sum index 9ff94e0..faf8240 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,12 @@ github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi github.com/Jeffail/gabs/v2 v2.6.1 h1:wwbE6nTQTwIMsMxzi6XFQQYRZ6wDc1mSdxoAN+9U4Gk= github.com/Jeffail/gabs/v2 v2.6.1/go.mod h1:xCn81vdHKxFUuWWAaD5jCTQDNPBMh5pPs9IJ+NcziBI= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -18,6 +24,7 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dave/jennifer v1.3.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -93,6 +100,7 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -113,9 +121,15 @@ github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peK github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o= +github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8= +github.com/mmcdole/goxpp v1.0.0 h1:/eu75G4jwH/LaugmPVB0FFC8LdKw00UMrpo6N7ym45o= +github.com/mmcdole/goxpp v1.0.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= @@ -130,8 +144,10 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -156,6 +172,7 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -182,15 +199,19 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= @@ -218,6 +239,7 @@ golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= diff --git a/http/server.go b/http/server.go index 78d916c..98b4083 100644 --- a/http/server.go +++ b/http/server.go @@ -62,17 +62,14 @@ func (s *Server) Start(zapWriter io.Writer) { // Outbox router.GET("/actors/:actor/outbox", func(c *gin.Context) { actorParam := c.Param("actor") - var found bool + var resource map[string]interface{} if c.Query("page") == "true" { - resource, _ := s.registry.OutboxCollection(actorParam) - c.String(http.StatusOK, resource) - found = true + resource, _ = s.registry.OutboxCollection(actorParam) } else { - resource, _ := s.registry.Outbox(actorParam) - c.JSON(http.StatusOK, resource) - found = true + resource, _ = s.registry.Outbox(actorParam) } - if found { + if resource != nil { + c.JSON(http.StatusOK, resource) c.Writer.Header().Set("Content-Type", "application/activity+json") } else { c.JSON(http.StatusNotFound, nil) diff --git a/models/activity_streams_object.go b/models/activity_streams_object.go deleted file mode 100644 index 98c69bb..0000000 --- a/models/activity_streams_object.go +++ /dev/null @@ -1,54 +0,0 @@ -package models - -import ( - "bytes" - "encoding/json" - "fmt" - "time" - - "git.capotej.com/capotej/communique/config" - "github.com/dgraph-io/badger/v3" - "github.com/go-fed/activity/streams/vocab" - "github.com/segmentio/ksuid" -) - -// ActivityStreamsObject is our internal model for storing activity streams objects -// For every object we receive, we create 2+N copies of it in Badger: -// - the original object -// - a copy for dedupe with an optional TTL (specified by handler response) TODO -// - a temporary copy for every subscriber until it is delivered TODO -type ActivityStreamsObject struct { - wrappedObject vocab.ActivityStreamsNote // OPTIONAL, not always wrapped (think .count) - handler config.Handler -} - -func NewActivityStreamsObject(obj vocab.ActivityStreamsNote, h config.Handler) *ActivityStreamsObject { - aso := &ActivityStreamsObject{wrappedObject: obj, handler: h} - return aso -} - -func (a *ActivityStreamsObject) keyName() []byte { - k, _ := ksuid.NewRandomWithTime(time.Now()) - key := fmt.Sprintf("%s:%s", a.Keybase(), k.String()) - return []byte(key) -} - -func (a *ActivityStreamsObject) Keybase() string { - keyBase := fmt.Sprintf("outboxes:%s", a.handler.Name) - return keyBase -} - -func (a *ActivityStreamsObject) content() []byte { - objMap, _ := a.wrappedObject.Serialize() - buffer := &bytes.Buffer{} - encoder := json.NewEncoder(buffer) - encoder.SetEscapeHTML(false) - encoder.Encode(objMap) - return bytes.TrimRight(buffer.Bytes(), "\n") -} - -func (a *ActivityStreamsObject) Save(txn *badger.Txn) error { - content := a.content() - e := badger.NewEntry(a.keyName(), content) - return txn.SetEntry(e) -} diff --git a/models/outbox_item.go b/models/outbox_item.go new file mode 100644 index 0000000..ed74130 --- /dev/null +++ b/models/outbox_item.go @@ -0,0 +1,56 @@ +package models + +import ( + "fmt" + "time" + + "git.capotej.com/capotej/communique/config" + "github.com/dgraph-io/badger/v3" + "github.com/segmentio/ksuid" +) + +type OutboxItem struct { + handler config.Handler + content []byte + id []byte + createdAt time.Time +} + +// used for lookup purposes (count, collect) +func NewOutboxItem(h config.Handler) *OutboxItem { + aso := &OutboxItem{handler: h} + return aso +} + +func CreateOutboxItem(h config.Handler, content []byte) *OutboxItem { + t := time.Now() + k, _ := ksuid.NewRandomWithTime(t) + aso := &OutboxItem{ + handler: h, + createdAt: t, + content: content, + id: k.Bytes(), + } + return aso +} + +func (a *OutboxItem) keyName() []byte { + key := fmt.Sprintf("%s:%s", a.Keybase(), a.id) + return []byte(key) +} + +func (a *OutboxItem) Keybase() string { + keyBase := fmt.Sprintf("outboxes:%s", a.handler.Name) + return keyBase +} + +func (a *OutboxItem) Save(txn *badger.Txn) error { + if len(a.content) == 0 { + return fmt.Errorf("content not set") + } + if len(a.id) == 0 { + return fmt.Errorf("id not set") + } + e := badger.NewEntry(a.keyName(), a.content) + return txn.SetEntry(e) +} diff --git a/models/persister.go b/models/persister.go index 3729c81..a639fc7 100644 --- a/models/persister.go +++ b/models/persister.go @@ -10,6 +10,11 @@ type Persister struct { db *badger.DB } +type PersisterResult struct { + Value []byte + Key []byte +} + func NewPersister(log *zap.SugaredLogger, db *badger.DB) *Persister { aso := &Persister{log: log, db: db} return aso @@ -36,8 +41,8 @@ func (p *Persister) Count(model model) (int, error) { return count, err } -func (p *Persister) Collect(model model) ([]string, error) { - var result []string +func (p *Persister) Collect(model model) ([]PersisterResult, error) { + var result []PersisterResult err := p.db.View(func(txn *badger.Txn) error { opts := badger.DefaultIteratorOptions opts.PrefetchValues = false // TODO Maybe we want true here @@ -47,7 +52,8 @@ func (p *Persister) Collect(model model) ([]string, error) { for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { item := it.Item() item.Value(func(v []byte) error { - result = append(result, string(v)) + pr := PersisterResult{Key: it.Item().Key(), Value: v} + result = append(result, pr) return nil }) } diff --git a/registry/registry.go b/registry/registry.go index 8372d40..aac21d4 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -40,7 +40,7 @@ func (r *Registry) Outbox(name string) (map[string]interface{}, error) { if handler == nil { return nil, nil } - aso := models.NewActivityStreamsObject(nil, handler.handlerCfg) + aso := models.NewOutboxItem(handler.handlerCfg) totalItems, err := r.persister.Count(aso) if err != nil { return nil, err @@ -48,15 +48,15 @@ func (r *Registry) Outbox(name string) (map[string]interface{}, error) { return views.RenderOutbox(handler.handlerCfg.Name, r.cfg.Domain, totalItems) } -func (r *Registry) OutboxCollection(name string) (string, error) { +func (r *Registry) OutboxCollection(name string) (map[string]interface{}, error) { handler := r.findByName(name) if handler == nil { - return "", nil + return nil, nil } - aso := models.NewActivityStreamsObject(nil, handler.handlerCfg) + aso := models.NewOutboxItem(handler.handlerCfg) page, err := r.persister.Collect(aso) if err != nil { - return "", err + return nil, err } return views.RenderOutboxCollection(handler.handlerCfg.Name, r.cfg.Domain, page) } diff --git a/sample-cgi-handler.sh b/sample-cgi-handler.sh index 5cafc84..b9b195d 100755 --- a/sample-cgi-handler.sh +++ b/sample-cgi-handler.sh @@ -1,14 +1,9 @@ #!/bin/bash -echo "Content-type: application/activity+json" +echo "Content-type: application/atom+xml" echo "" -echo '{ - "@context": "https://www.w3.org/ns/activitystreams", - "type": "OrderedCollection", - "totalItems": 1, - "orderedItems": [ - { - "type": "Note", - "content": "hi" - } - ]}' +echo ' + + asd + +' exit 0 diff --git a/views/outbox.go b/views/outbox.go index 97dd939..88f88c2 100644 --- a/views/outbox.go +++ b/views/outbox.go @@ -1,81 +1,56 @@ package views import ( - "bytes" - + "git.capotej.com/capotej/communique/models" "git.capotej.com/capotej/communique/urls" "github.com/go-fed/activity/streams" ) // RenderOutboxCollection takes a page of ActivityStream objects as JSON strings and concatenates them together to return an // ActivtyStreamsOrderedCollection -func RenderOutboxCollection(name, domain string, page []string) (string, error) { +func RenderOutboxCollection(name, domain string, page []models.PersisterResult) (map[string]interface{}, error) { id, err := urls.UrlOutboxPage(name, domain) if err != nil { - return "", err + return nil, err + } + + partOf, err := urls.UrlOutbox(name, domain) + if err != nil { + return nil, err } - buf := &bytes.Buffer{} - buf.WriteString("{") - buf.WriteString("\"id\":\"" + id.String() + "\"") - buf.WriteString("}") - - // partOf, err := urls.UrlOutbox(name, domain) - // if err != nil { - // return "", err - // } - - // result := make(map[string]interface{}) - // result["id"] = id.String() - // result["partOf"] = partOf.String() - // result["orderedItems"] = page - - // oc := streams.NewActivityStreamsOrderedCollectionPage() - - // idProp := streams.NewJSONLDIdProperty() - // idProp.Set(id) - // oc.SetJSONLDId(idProp) - - // partOfProp := streams.NewActivityStreamsPartOfProperty() - // partOfProp.SetIRI(partOf) - // oc.SetActivityStreamsPartOf(partOfProp) - - // itemsProp := streams.NewActivityStreamsOrderedItemsProperty() - - // // err = db.View(func(txn *badger.Txn) error { - // // opts := badger.DefaultIteratorOptions - // // opts.PrefetchValues = false - // // it := txn.NewIterator(opts) - // // defer it.Close() - // // prefix := []byte("outbox:sample") // TODO - // // for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { - // // item := it.Item() - // // err := item.Value(func(v []byte) error { - // // crea := streams.NewActivityStreamsCreate() - // // obj := streams.NewActivityStreamsObjectProperty() - // // crea.SetActivityStreamsObject(obj) - - // // note := streams.NewActivityStreamsNote() - // // contentProp := streams.NewActivityStreamsContentProperty() - // // contentProp.AppendXMLSchemaString(string(v)) - // // note.SetActivityStreamsContent(contentProp) - // // obj.AppendActivityStreamsNote(note) - - // // itemsProp.AppendActivityStreamsCreate(crea) - // // return nil - // // }) - // // if err != nil { - // // return err - // // } - // // } - // // return nil - // // }) - // // if err != nil { - // // return nil, err - // // } - - // oc.SetActivityStreamsOrderedItems(itemsProp) - // return streams.Serialize(oc) - return buf.String(), nil + + result := make(map[string]interface{}) + result["id"] = id.String() + result["partOf"] = partOf.String() + result["orderedItems"] = page + + oc := streams.NewActivityStreamsOrderedCollectionPage() + + idProp := streams.NewJSONLDIdProperty() + idProp.Set(id) + oc.SetJSONLDId(idProp) + + partOfProp := streams.NewActivityStreamsPartOfProperty() + partOfProp.SetIRI(partOf) + oc.SetActivityStreamsPartOf(partOfProp) + + itemsProp := streams.NewActivityStreamsOrderedItemsProperty() + + for _, v := range page { + crea := streams.NewActivityStreamsCreate() + obj := streams.NewActivityStreamsObjectProperty() + crea.SetActivityStreamsObject(obj) + + note := streams.NewActivityStreamsNote() + contentProp := streams.NewActivityStreamsContentProperty() + contentProp.AppendXMLSchemaString(string(v.Value)) + note.SetActivityStreamsContent(contentProp) + obj.AppendActivityStreamsNote(note) + + itemsProp.AppendActivityStreamsCreate(crea) + } + oc.SetActivityStreamsOrderedItems(itemsProp) + return streams.Serialize(oc) } func RenderOutbox(name, domain string, totalItems int) (map[string]interface{}, error) { -- cgit v1.2.3