experiment with write JSON to database and munge it for collections

8 files changed, 117 insertions, 69 deletions

diff --git a/cgi/servers.go b/cgi/servers.go
index 229bca2..f5652ed 100644
--- a/cgi/servers.go
+++ b/cgi/servers.go
@@ -104,10 +104,12 @@ func processTick(h config.Handler, output []byte, persister *models.Persister, l
 	orderedProp := coll.GetActivityStreamsOrderedItems()
 	for iter := orderedProp.Begin(); iter != orderedProp.End(); iter = iter.Next() {
 		contentType := iter.GetType().GetTypeName()
-		log.Debugf("found object type %s in activity stream", contentType)
-
 		contentNote := iter.GetActivityStreamsNote() // TODO make this configurable since it cant be dynamic
 
+		jsonMap, _ := streams.Serialize(contentNote)
+		jsonIter, _ := json.Marshal(jsonMap)
+		log.Debugf("found object %s of type %s in activity stream", jsonIter, contentType)
+
 		localNote := models.NewActivityStreamsObject(contentNote, h)
 		err = persister.Store(localNote)
 		if err != nil {
diff --git a/go.mod b/go.mod
index f0c839d..8c8e5b9 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.19
 require github.com/BurntSushi/toml v1.2.1
 
 require (
+	github.com/Jeffail/gabs/v2 v2.6.1 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dgraph-io/badger/v3 v3.2103.5 // indirect
diff --git a/go.sum b/go.sum
index 983bb88..9ff94e0 100644
--- a/go.sum
+++ b/go.sum
@@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/Jeffail/gabs/v2 v2.6.1 h1:wwbE6nTQTwIMsMxzi6XFQQYRZ6wDc1mSdxoAN+9U4Gk=
+github.com/Jeffail/gabs/v2 v2.6.1/go.mod h1:xCn81vdHKxFUuWWAaD5jCTQDNPBMh5pPs9IJ+NcziBI=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
diff --git a/http/server.go b/http/server.go
index b02e5c9..78d916c 100644
--- a/http/server.go
+++ b/http/server.go
@@ -62,15 +62,18 @@ func (s *Server) Start(zapWriter io.Writer) {
 	// Outbox
 	router.GET("/actors/:actor/outbox", func(c *gin.Context) {
 		actorParam := c.Param("actor")
-		var resource map[string]interface{}
+		var found bool
 		if c.Query("page") == "true" {
-			resource, _ = s.registry.OutboxPage(actorParam)
+			resource, _ := s.registry.OutboxCollection(actorParam)
+			c.String(http.StatusOK, resource)
+			found = true
 		} else {
-			resource, _ = s.registry.Outbox(actorParam)
+			resource, _ := s.registry.Outbox(actorParam)
+			c.JSON(http.StatusOK, resource)
+			found = true
 		}
-		if resource != nil {
+		if found {
 			c.Writer.Header().Set("Content-Type", "application/activity+json")
-			c.JSON(http.StatusOK, resource)
 		} else {
 			c.JSON(http.StatusNotFound, nil)
 		}
diff --git a/models/activity_streams_object.go b/models/activity_streams_object.go
index 85a7d09..98c69bb 100644
--- a/models/activity_streams_object.go
+++ b/models/activity_streams_object.go
@@ -1,6 +1,8 @@
 package models
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"time"
 
@@ -37,17 +39,16 @@ func (a *ActivityStreamsObject) Keybase() string {
 }
 
 func (a *ActivityStreamsObject) content() []byte {
-	var contentVal string
-	if a.wrappedObject.GetActivityStreamsContent() != nil {
-		content := a.wrappedObject.GetActivityStreamsContent()
-		for contentIter := content.Begin(); contentIter != content.End(); contentIter = contentIter.Next() {
-			contentVal = contentIter.GetXMLSchemaString()
-		}
-	}
-	return []byte(contentVal)
+	objMap, _ := a.wrappedObject.Serialize()
+	buffer := &bytes.Buffer{}
+	encoder := json.NewEncoder(buffer)
+	encoder.SetEscapeHTML(false)
+	encoder.Encode(objMap)
+	return bytes.TrimRight(buffer.Bytes(), "\n")
 }
 
 func (a *ActivityStreamsObject) Save(txn *badger.Txn) error {
-	e := badger.NewEntry(a.keyName(), a.content())
+	content := a.content()
+	e := badger.NewEntry(a.keyName(), content)
 	return txn.SetEntry(e)
 }
diff --git a/models/persister.go b/models/persister.go
index 71cddc8..3729c81 100644
--- a/models/persister.go
+++ b/models/persister.go
@@ -35,3 +35,23 @@ func (p *Persister) Count(model model) (int, error) {
 	})
 	return count, err
 }
+
+func (p *Persister) Collect(model model) ([]string, error) {
+	var result []string
+	err := p.db.View(func(txn *badger.Txn) error {
+		opts := badger.DefaultIteratorOptions
+		opts.PrefetchValues = false // TODO Maybe we want true here
+		it := txn.NewIterator(opts)
+		defer it.Close()
+		prefix := []byte(model.Keybase())
+		for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
+			item := it.Item()
+			item.Value(func(v []byte) error {
+				result = append(result, string(v))
+				return nil
+			})
+		}
+		return nil
+	})
+	return result, err
+}
diff --git a/registry/registry.go b/registry/registry.go
index f57aede..8372d40 100644
--- a/registry/registry.go
+++ b/registry/registry.go
@@ -48,12 +48,17 @@ func (r *Registry) Outbox(name string) (map[string]interface{}, error) {
 	return views.RenderOutbox(handler.handlerCfg.Name, r.cfg.Domain, totalItems)
 }
 
-func (r *Registry) OutboxPage(name string) (map[string]interface{}, error) {
+func (r *Registry) OutboxCollection(name string) (string, error) {
 	handler := r.findByName(name)
 	if handler == nil {
-		return nil, nil
+		return "", nil
+	}
+	aso := models.NewActivityStreamsObject(nil, handler.handlerCfg)
+	page, err := r.persister.Collect(aso)
+	if err != nil {
+		return "", err
 	}
-	return views.RenderOutboxPage(handler.handlerCfg.Name, r.cfg.Domain)
+	return views.RenderOutboxCollection(handler.handlerCfg.Name, r.cfg.Domain, page)
 }
 
 func (r *Registry) WebfingerResource(fqn string) (*views.WebfingerResource, error) {
diff --git a/views/outbox.go b/views/outbox.go
index fe87e24..97dd939 100644
--- a/views/outbox.go
+++ b/views/outbox.go
@@ -1,67 +1,81 @@
 package views
 
 import (
+	"bytes"
+
 	"git.capotej.com/capotej/communique/urls"
 	"github.com/go-fed/activity/streams"
 )
 
-func RenderOutboxPage(name, domain string) (map[string]interface{}, error) {
+// RenderOutboxCollection takes a page of ActivityStream objects as JSON strings and concatenates them together to return an
+// ActivtyStreamsOrderedCollection
+func RenderOutboxCollection(name, domain string, page []string) (string, error) {
 	id, err := urls.UrlOutboxPage(name, domain)
 	if err != nil {
-		return nil, err
-	}
-
-	partOf, err := urls.UrlOutbox(name, domain)
-	if err != nil {
-		return nil, err
+		return "", err
 	}
+	buf := &bytes.Buffer{}
+	buf.WriteString("{")
+	buf.WriteString("\"id\":\"" + id.String() + "\"")
+	buf.WriteString("}")
 
-	oc := streams.NewActivityStreamsOrderedCollectionPage()
-
-	idProp := streams.NewJSONLDIdProperty()
-	idProp.Set(id)
-	oc.SetJSONLDId(idProp)
-
-	partOfProp := streams.NewActivityStreamsPartOfProperty()
-	partOfProp.SetIRI(partOf)
-	oc.SetActivityStreamsPartOf(partOfProp)
-
-	itemsProp := streams.NewActivityStreamsOrderedItemsProperty()
-
-	// err = db.View(func(txn *badger.Txn) error {
-	// 	opts := badger.DefaultIteratorOptions
-	// 	opts.PrefetchValues = false
-	// 	it := txn.NewIterator(opts)
-	// 	defer it.Close()
-	// 	prefix := []byte("outbox:sample") // TODO
-	// 	for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
-	// 		item := it.Item()
-	// 		err := item.Value(func(v []byte) error {
-	// 			crea := streams.NewActivityStreamsCreate()
-	// 			obj := streams.NewActivityStreamsObjectProperty()
-	// 			crea.SetActivityStreamsObject(obj)
-
-	// 			note := streams.NewActivityStreamsNote()
-	// 			contentProp := streams.NewActivityStreamsContentProperty()
-	// 			contentProp.AppendXMLSchemaString(string(v))
-	// 			note.SetActivityStreamsContent(contentProp)
-	// 			obj.AppendActivityStreamsNote(note)
-
-	// 			itemsProp.AppendActivityStreamsCreate(crea)
-	// 			return nil
-	// 		})
-	// 		if err != nil {
-	// 			return err
-	// 		}
-	// 	}
-	// 	return nil
-	// })
+	// partOf, err := urls.UrlOutbox(name, domain)
 	// if err != nil {
-	// 	return nil, err
+	// 	return "", err
 	// }
 
-	oc.SetActivityStreamsOrderedItems(itemsProp)
-	return streams.Serialize(oc)
+	// result := make(map[string]interface{})
+	// result["id"] = id.String()
+	// result["partOf"] = partOf.String()
+	// result["orderedItems"] = page
+
+	// oc := streams.NewActivityStreamsOrderedCollectionPage()
+
+	// idProp := streams.NewJSONLDIdProperty()
+	// idProp.Set(id)
+	// oc.SetJSONLDId(idProp)
+
+	// partOfProp := streams.NewActivityStreamsPartOfProperty()
+	// partOfProp.SetIRI(partOf)
+	// oc.SetActivityStreamsPartOf(partOfProp)
+
+	// itemsProp := streams.NewActivityStreamsOrderedItemsProperty()
+
+	// // err = db.View(func(txn *badger.Txn) error {
+	// // 	opts := badger.DefaultIteratorOptions
+	// // 	opts.PrefetchValues = false
+	// // 	it := txn.NewIterator(opts)
+	// // 	defer it.Close()
+	// // 	prefix := []byte("outbox:sample") // TODO
+	// // 	for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
+	// // 		item := it.Item()
+	// // 		err := item.Value(func(v []byte) error {
+	// // 			crea := streams.NewActivityStreamsCreate()
+	// // 			obj := streams.NewActivityStreamsObjectProperty()
+	// // 			crea.SetActivityStreamsObject(obj)
+
+	// // 			note := streams.NewActivityStreamsNote()
+	// // 			contentProp := streams.NewActivityStreamsContentProperty()
+	// // 			contentProp.AppendXMLSchemaString(string(v))
+	// // 			note.SetActivityStreamsContent(contentProp)
+	// // 			obj.AppendActivityStreamsNote(note)
+
+	// // 			itemsProp.AppendActivityStreamsCreate(crea)
+	// // 			return nil
+	// // 		})
+	// // 		if err != nil {
+	// // 			return err
+	// // 		}
+	// // 	}
+	// // 	return nil
+	// // })
+	// // if err != nil {
+	// // 	return nil, err
+	// // }
+
+	// oc.SetActivityStreamsOrderedItems(itemsProp)
+	// return streams.Serialize(oc)
+	return buf.String(), nil
 }
 
 func RenderOutbox(name, domain string, totalItems int) (map[string]interface{}, error) {