Merge pull request #905 from dilshans2k/bugfix/drafty-should-correctly-parse-message-content-for-push-notification

fix: #904 drafty should correctly parse message content for push notification

go.mod

@@ -60,6 +60,7 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect

go.sum

@@ -214,6 +214,8 @@ github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSz
 github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=

server/drafty/drafty.go

@@ -6,6 +6,8 @@ import (
 	"errors"
 	"sort"
 	"strings"
+
+	"github.com/rivo/uniseg"
 )
 
 const (
@@ -34,7 +36,7 @@ type entity struct {
 
 type document struct {
 	Txt string `json:"txt,omitempty"`
-	txt []rune
+	txt *uniseg.Graphemes
 	Fmt []style  `json:"fmt,omitempty"`
 	Ent []entity `json:"ent,omitempty"`
 }
@@ -48,7 +50,7 @@ type span struct {
 }
 
 type node struct {
-	txt      []rune
+	txt      *uniseg.Graphemes
 	sp       *span
 	children []*node
 }
@@ -59,7 +61,7 @@ type previewState struct {
 	keymap    map[int]int
 }
 
-// Preview shortens Drafty to the specified length (in runes), removes quoted text, leading line breaks,
+// Preview shortens Drafty to the specified length (in graphemes), removes quoted text, leading line breaks,
 // and large content from entities making them suitable for a one-line preview,
 // for example for showing in push notifications.
 // The return value is a Drafty document encoded as JSON string.
@@ -93,7 +95,7 @@ func Preview(content interface{}, length int) (string, error) {
 		return "", err
 	}
 
-	state.drafty.Txt = string(state.drafty.txt)
+	state.drafty.Txt = graphemeToString(state.drafty.txt)
 	data, err := json.Marshal(state.drafty)
 	return string(data), err
 }
@@ -173,7 +175,7 @@ func toTree(drafty *document) (*node, error) {
 		return &node{txt: drafty.txt}, nil
 	}
 
-	textLen := len(drafty.txt)
+	textLen := getGraphemeLength(drafty.txt)
 
 	var spans []*span
 	for i := range drafty.Fmt {
@@ -231,8 +233,80 @@ func toTree(drafty *document) (*node, error) {
 	return &node{children: children}, nil
 }
 
+// Given a grapheme iterator, start and end pos, returns a new grapheme iterator
+// containing a slice of graphemes(start->end) from input interator.
+func sliceGraphemeClusters(g *uniseg.Graphemes, start, end int) *uniseg.Graphemes {
+	g.Reset()
+
+	output := ""
+
+	for i, j := 0, -1; g.Next(); {
+		if j > 0 {
+			if j < end {
+				output = output + g.Str()
+				j++
+			} else {
+				// end pos found, stop collecting string
+				break
+			}
+		} else if i < start {
+			i++
+		} else if i == start {
+			// starting pos found, start collecting string
+			output = output + g.Str()
+			j = i + 1
+		}
+
+	}
+
+	return uniseg.NewGraphemes(output)
+}
+
+// Given a grapheme iterator, returns the original string from which it was created from.
+func graphemeToString(g *uniseg.Graphemes) string {
+	g.Reset()
+
+	output := ""
+
+	for g.Next() {
+		output += g.Str()
+	}
+
+	return output
+}
+
+// Returns the number of grapheme cluster found in iterator.
+func getGraphemeLength(g *uniseg.Graphemes) int {
+	g.Reset()
+
+	output := 0
+
+	for g.Next() {
+		output++
+	}
+
+	return output
+}
+
+// Given two grapheme iterators g1 and g2,returns a grapheme iterator with string value equal to s1 + s2.
+func appendGraphemes(g1 *uniseg.Graphemes, g2 *uniseg.Graphemes) *uniseg.Graphemes {
+	g1.Reset()
+	g2.Reset()
+	output := ""
+
+	for g1.Next() {
+		output += g1.Str()
+	}
+
+	for g2.Next() {
+		output += g2.Str()
+	}
+
+	return uniseg.NewGraphemes(output)
+}
+
 // forEach recursively iterates nested spans to form a tree.
-func forEach(line []rune, start, end int, spans []*span) ([]*node, error) {
+func forEach(g *uniseg.Graphemes, start, end int, spans []*span) ([]*node, error) {
 	var result []*node
 
 	// Process ranges calling iterator for each range.
@@ -244,10 +318,9 @@ func forEach(line []rune, start, end int, spans []*span) ([]*node, error) {
 			result = append(result, &node{sp: sp})
 			continue
 		}
-
 		// Add un-styled range before the styled span starts.
 		if start < sp.at {
-			result = append(result, &node{txt: line[start:sp.at]})
+			result = append(result, &node{txt: sliceGraphemeClusters(g, start, sp.at)})
 			start = sp.at
 		}
 
@@ -261,7 +334,7 @@ func forEach(line []rune, start, end int, spans []*span) ([]*node, error) {
 		if tags[sp.tp].isVoid {
 			result = append(result, &node{sp: sp})
 		} else {
-			children, err := forEach(line, start, sp.end, subspans)
+			children, err := forEach(g, start, sp.end, subspans)
 			if err != nil {
 				return nil, err
 			}
@@ -272,7 +345,7 @@ func forEach(line []rune, start, end int, spans []*span) ([]*node, error) {
 
 	// Add the remaining unformatted range.
 	if start < end {
-		result = append(result, &node{txt: line[start:end]})
+		result = append(result, &node{txt: sliceGraphemeClusters(g, start, end)})
 	}
 
 	return result, nil
@@ -294,7 +367,7 @@ func plainTextFormatter(n *node, ctx interface{}) error {
 		}
 		text = string(state.txt)
 	} else {
-		text = string(n.txt)
+		text = graphemeToString(n.txt)
 	}
 
 	state := ctx.(*plainTextState)
@@ -338,7 +411,7 @@ func plainTextFormatter(n *node, ctx interface{}) error {
 func previewFormatter(n *node, ctx interface{}) error {
 
 	state := ctx.(*previewState)
-	at := len(state.drafty.txt)
+	at := getGraphemeLength(state.drafty.txt)
 	if at >= state.maxLength {
 		// Maximum doc length reached.
 		return nil
@@ -362,14 +435,17 @@ func previewFormatter(n *node, ctx interface{}) error {
 			}
 		}
 	} else {
-		increment := len(n.txt)
+		increment := getGraphemeLength(n.txt)
 		if at+increment > state.maxLength {
 			increment = state.maxLength - at
 		}
-		state.drafty.txt = append(state.drafty.txt, n.txt[:increment]...)
+		if state.drafty.txt == nil {
+			state.drafty.txt = uniseg.NewGraphemes("")
+		}
+		state.drafty.txt = appendGraphemes(state.drafty.txt, sliceGraphemeClusters(n.txt, 0, increment))
 	}
 
-	end := len(state.drafty.txt)
+	end := getGraphemeLength(state.drafty.txt)
 
 	if n.sp != nil {
 		fmt := style{}
@@ -421,13 +497,13 @@ func decodeAsDrafty(content interface{}) (*document, error) {
 
 	switch tmp := content.(type) {
 	case string:
-		drafty = &document{txt: []rune(tmp)}
+		drafty = &document{txt: uniseg.NewGraphemes(tmp)}
 	case map[string]interface{}:
 		drafty = &document{}
 		correct := 0
 		if txt, ok := tmp["txt"].(string); ok {
 			drafty.Txt = txt
-			drafty.txt = []rune(txt)
+			drafty.txt = uniseg.NewGraphemes(txt)
 			correct++
 		}
 		if ifmt, ok := tmp["fmt"].([]interface{}); ok {

server/drafty/drafty_test.go

@@ -52,6 +52,15 @@ var validInputs = []string{
 		"fmt":[{"at":13,"len":1,"tp":"BR"},{"at":15,"len":1},{"len":13,"key":1},{"len":16,"tp":"QQ"},{"at":16,"len":1,"tp":"BR"}],
 		"ent":[{"tp":"IM","data":{"mime":"image/jpeg","val":"<1292, bytes: /9j/4AAQSkZJ...rehH5o6D/9k=>","width":25,"height":14,"size":968}},{"tp":"MN","data":{"color":2}}]
 	}`,
+	`{
+		"txt": "Hello 😀, o😀k https://google.com",
+		"fmt":[{"at":9,"len":3,"tp":"ST"},{"at":13,"len":18}],
+		"ent":[{"tp":"LN","data":{"url":"https://google.com"}}]
+	}`,
+	`{
+		"txt": "Hi 🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿 🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿 🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿",
+		"fmt":[{"at":3,"len":4,"tp":"ST"},{"at":8,"len":4,"tp":"ST"}]
+	}`,
 }
 
 var invalidInputs = []string{
@@ -99,6 +108,8 @@ func TestPlainText(t *testing.T) {
 		"This *text* is _formatted_ and ~deleted *too*~",
 		"*мультибайтовый* _юникод_",
 		"This is a test",
+		"Hello 😀, *o😀k* https://google.com",
+		"Hi *🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿* *🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿* 🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿",
 	}
 
 	for i := range validInputs {
@@ -140,6 +151,8 @@ func TestPreview(t *testing.T) {
 		`{"txt":"This text is fo","fmt":[{"tp":"ST","at":5,"len":4},{"tp":"EM","at":13,"len":2}]}`,
 		`{"txt":"мультибайтовый ","fmt":[{"tp":"ST","len":14}]}`,
 		`{"txt":"This is a test"}`,
+		`{"txt":"Hello 😀, o😀k ht","fmt":[{"tp":"ST","at":9,"len":3},{"at":13,"len":2}],"ent":[{"tp":"LN","data":{"url":"https://google.com"}}]}`,
+		`{"txt":"Hi 🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿 🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿 🏴󠁧󠁢󠁳󠁣󠁴󠁿🏴󠁧󠁢󠁳󠁣󠁴󠁿","fmt":[{"tp":"ST","at":3,"len":4},{"tp":"ST","at":8,"len":4}]}`,
 	}
 	for i := range validInputs {
 		var val interface{}
@@ -164,7 +177,7 @@ func TestPreview(t *testing.T) {
 		}
 		res, err := Preview(val, 15)
 		if err == nil {
-			t.Errorf("invalid input %d did not cause an error '%s'", testsToFail[i], res)
+			t.Errorf("invalid input %d did not cause an error '%s'", i, res)
 		}
 	}
 }