working dedup

1 files changed, 29 insertions, 0 deletions

diff --git a/models/outbox_item.go b/models/outbox_item.go
index 9d9de97..34457d6 100644
--- a/models/outbox_item.go
+++ b/models/outbox_item.go
@@ -2,6 +2,7 @@ package models
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"encoding/gob"
 	"fmt"
 	"time"
@@ -15,6 +16,7 @@ type OutboxItem struct {
 	Handler   config.Handler
 	Content   []byte
 	Id        []byte
+	Sha256    string
 	CreatedAt time.Time
 }
 
@@ -26,25 +28,48 @@ func NewOutboxItem(h config.Handler) *OutboxItem {
 
 func CreateOutboxItem(h config.Handler, content []byte) *OutboxItem {
 	t := time.Now()
+	hash := sha256.New()
+	hash.Write(content)
+	binHash := hash.Sum(nil)
 	k, _ := ksuid.NewRandomWithTime(t)
 	aso := &OutboxItem{
 		Handler:   h,
 		CreatedAt: t,
 		Content:   content,
+		Sha256:    fmt.Sprintf("%x", binHash),
 		Id:        []byte(k.String()), // NOTE: we want the bytes of the string representation of a hash, NOT a binary hash
 	}
 	return aso
 }
 
+func (a *OutboxItem) Name() string {
+	return "OutboxItem"
+}
+
 func (a *OutboxItem) Key() string {
 	return fmt.Sprintf("%s:%s", a.Keybase(), a.Id)
 }
 
+func (a *OutboxItem) DedupKey() string {
+	return fmt.Sprintf("dd:%s:sha256:%s", a.Keybase(), a.Sha256)
+}
+
 func (a *OutboxItem) Keybase() string {
 	keyBase := fmt.Sprintf("outboxes:%s", a.Handler.Name)
 	return keyBase
 }
 
+func (a *OutboxItem) SaveDedup(txn *badger.Txn) error {
+	if len(a.Content) == 0 {
+		return fmt.Errorf("content not set")
+	}
+	if len(a.Sha256) == 0 {
+		return fmt.Errorf("sha256 not set")
+	}
+	e := badger.NewEntry([]byte(a.DedupKey()), nil).WithTTL(a.Handler.DedupWindow)
+	return txn.SetEntry(e)
+}
+
 func (a *OutboxItem) Save(txn *badger.Txn) error {
 	if len(a.Content) == 0 {
 		return fmt.Errorf("content not set")
@@ -52,6 +77,10 @@ func (a *OutboxItem) Save(txn *badger.Txn) error {
 	if len(a.Id) == 0 {
 		return fmt.Errorf("id not set")
 	}
+	if len(a.Sha256) == 0 {
+		return fmt.Errorf("sha256 not set")
+	}
+
 	var network bytes.Buffer
 	enc := gob.NewEncoder(&network)
 	err := enc.Encode(a)