fix: mentions for users with UTC-16 characters (#222)

* fix: mentions for users with UTC-16 characters

* fix: mention parsing & validation (indent)

Author: highesttt

pkg/connector/handle_message.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"html"
 	"sort"
-	"strconv"
 	"strings"
 	"time"
 
@@ -480,20 +479,16 @@ func (lc *LineClient) convertLineMessage(ctx context.Context, portal *bridgev2.P
 					lc.UserLogin.Bridge.Log.Debug().Str("mxid", string(mxid)).Msg("Formatted MXID from LINE MID")
 					mentions.UserIDs = append(mentions.UserIDs, mxid)
 					if canFormatMentions {
-						if s, errS := strconv.Atoi(ment.S); errS == nil && s >= 0 {
-							if e, errE := strconv.Atoi(ment.E); errE == nil && e <= len(unwrappedText) && e > s {
-								entries = append(entries, mentionEntry{start: s, end: e, mxid: string(mxid)})
-							}
+						if start, end, ok := resolveMentionRange(unwrappedText, ment.S, ment.E); ok {
+							entries = append(entries, mentionEntry{start: start, end: end, mxid: string(mxid)})
 						}
 					}
 				}
 				if ment.A == "1" {
 					mentions.Room = true
 					if canFormatMentions {
-						if s, errS := strconv.Atoi(ment.S); errS == nil && s >= 0 {
-							if e, errE := strconv.Atoi(ment.E); errE == nil && e <= len(unwrappedText) && e > s {
-								entries = append(entries, mentionEntry{start: s, end: e, mxid: "@room"})
-							}
+						if start, end, ok := resolveMentionRange(unwrappedText, ment.S, ment.E); ok {
+							entries = append(entries, mentionEntry{start: start, end: end, mxid: "@room"})
 						}
 					}
 				}

pkg/connector/handle_message_test.go

@@ -164,6 +164,78 @@ func TestConvertLineMessagePreservesMentionsForSticonFallback(t *testing.T) {
 	}
 }
 
+func TestConvertLineMessageUsesUTF16MentionOffsets(t *testing.T) {
+	text := "🙂 嗨 @กิ๊ก!"
+	userMXID := id.UserID("@user:example.com")
+	lc := &LineClient{
+		Mid: "self-mid",
+		UserLogin: &bridgev2.UserLogin{
+			UserLogin: &database.UserLogin{UserMXID: userMXID},
+			Bridge:    &bridgev2.Bridge{Log: zerolog.New(io.Discard)},
+		},
+	}
+	data := line.Message{
+		ContentType: int(ContentText),
+		ContentMetadata: map[string]string{
+			"MENTION": `{"MENTIONEES":[{"M":"self-mid","S":"5","E":"10"}]}`,
+		},
+	}
+
+	converted, err := lc.convertLineMessage(t.Context(), nil, nil, data, text, text, false)
+	if err != nil {
+		t.Fatalf("convertLineMessage returned error: %v", err)
+	}
+	if converted == nil || len(converted.Parts) != 1 || converted.Parts[0].Content == nil {
+		t.Fatalf("convertLineMessage returned %#v, want one message part", converted)
+	}
+	content := converted.Parts[0].Content
+	if content.Body != text {
+		t.Fatalf("Body = %q, want %q", content.Body, text)
+	}
+	if content.Mentions == nil || len(content.Mentions.UserIDs) != 1 || content.Mentions.UserIDs[0] != userMXID {
+		t.Fatalf("Mentions = %#v, want user %s", content.Mentions, userMXID)
+	}
+	expectedLink := `<a href="https://matrix.to/#/@user:example.com">@กิ๊ก</a>`
+	if !strings.Contains(content.FormattedBody, expectedLink) {
+		t.Fatalf("FormattedBody = %q, want link %q", content.FormattedBody, expectedLink)
+	}
+}
+
+func TestConvertLineMessageUsesUTF16RoomMentionOffsets(t *testing.T) {
+	text := "🙂 嗨 @everyone!"
+	lc := &LineClient{
+		UserLogin: &bridgev2.UserLogin{
+			UserLogin: &database.UserLogin{UserMXID: "@user:example.com"},
+			Bridge:    &bridgev2.Bridge{Log: zerolog.New(io.Discard)},
+		},
+	}
+	data := line.Message{
+		ContentType: int(ContentText),
+		ContentMetadata: map[string]string{
+			"MENTION": `{"MENTIONEES":[{"A":"1","S":"5","E":"14"}]}`,
+		},
+	}
+
+	converted, err := lc.convertLineMessage(t.Context(), nil, nil, data, text, text, false)
+	if err != nil {
+		t.Fatalf("convertLineMessage returned error: %v", err)
+	}
+	if converted == nil || len(converted.Parts) != 1 || converted.Parts[0].Content == nil {
+		t.Fatalf("convertLineMessage returned %#v, want one message part", converted)
+	}
+	content := converted.Parts[0].Content
+	if content.Body != "🙂 嗨 @room!" {
+		t.Fatalf("Body = %q, want room mention replacement", content.Body)
+	}
+	if content.Mentions == nil || !content.Mentions.Room {
+		t.Fatalf("Mentions = %#v, want room mention", content.Mentions)
+	}
+	expectedLink := `<a href="https://matrix.to/#/@room">@everyone</a>`
+	if !strings.Contains(content.FormattedBody, expectedLink) {
+		t.Fatalf("FormattedBody = %q, want link %q", content.FormattedBody, expectedLink)
+	}
+}
+
 func TestDecryptMessageBodySkipsGeneratedFallbackWhenDecryptUnavailable(t *testing.T) {
 	msg := &line.Message{
 		Text:   "",

pkg/connector/send_message.go

@@ -963,11 +963,15 @@ func (lc *LineClient) buildMentionMetadata(ctx context.Context, body, formattedB
 			}
 		}
 		if pos >= 0 {
-			mentionees = append(mentionees, mentionEntry{
-				S: strconv.Itoa(pos),
-				E: strconv.Itoa(pos + matchLen),
-				A: "1",
-			})
+			start, startOK := byteIndexToUTF16Offset(body, pos)
+			end, endOK := byteIndexToUTF16Offset(body, pos+matchLen)
+			if startOK && endOK {
+				mentionees = append(mentionees, mentionEntry{
+					S: strconv.Itoa(start),
+					E: strconv.Itoa(end),
+					A: "1",
+				})
+			}
 		}
 	}
 
@@ -1002,11 +1006,15 @@ func (lc *LineClient) buildMentionMetadata(ctx context.Context, body, formattedB
 		searchStr := "@" + displayName
 		pos := strings.Index(body, searchStr)
 		if pos >= 0 {
-			mentionees = append(mentionees, mentionEntry{
-				S: strconv.Itoa(pos),
-				E: strconv.Itoa(pos + len(searchStr)),
-				M: string(mid),
-			})
+			start, startOK := byteIndexToUTF16Offset(body, pos)
+			end, endOK := byteIndexToUTF16Offset(body, pos+len(searchStr))
+			if startOK && endOK {
+				mentionees = append(mentionees, mentionEntry{
+					S: strconv.Itoa(start),
+					E: strconv.Itoa(end),
+					M: string(mid),
+				})
+			}
 		}
 	}
 

pkg/connector/send_message_test.go

@@ -1,17 +1,83 @@
 package connector
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"testing"
 
 	"maunium.net/go/mautrix/bridgev2"
+	"maunium.net/go/mautrix/bridgev2/networkid"
 	"maunium.net/go/mautrix/event"
+	"maunium.net/go/mautrix/id"
 
 	"github.com/highesttt/matrix-line-messenger/pkg/e2ee"
 	"github.com/highesttt/matrix-line-messenger/pkg/line"
 )
 
+type mentionTestMatrix struct {
+	bridgev2.MatrixConnector
+	ghosts map[id.UserID]networkid.UserID
+}
+
+func (matrix *mentionTestMatrix) ParseGhostMXID(userID id.UserID) (networkid.UserID, bool) {
+	ghostID, ok := matrix.ghosts[userID]
+	return ghostID, ok
+}
+
+func TestBuildMentionMetadataUsesUTF16Offsets(t *testing.T) {
+	userIDs := []id.UserID{
+		"@line_zhang:example.com",
+		"@line_kik:example.com",
+		"@line_alice:example.com",
+	}
+	matrix := &mentionTestMatrix{ghosts: map[id.UserID]networkid.UserID{
+		userIDs[0]: "u-zhang",
+		userIDs[1]: "u-kik",
+		userIDs[2]: "u-alice",
+	}}
+	lc := &LineClient{UserLogin: &bridgev2.UserLogin{
+		Bridge: &bridgev2.Bridge{Matrix: matrix},
+	}}
+	body := "🙂 @张三 @กิ๊ก @Alice"
+	formattedBody := `🙂 <a href="https://matrix.to/#/@line_zhang:example.com">张三</a> ` +
+		`<a href="https://matrix.to/#/@line_kik:example.com">กิ๊ก</a> ` +
+		`<a href="https://matrix.to/#/@line_alice:example.com">Alice</a>`
+
+	metadata := lc.buildMentionMetadata(t.Context(), body, formattedBody, &event.Mentions{UserIDs: userIDs})
+	var payload struct {
+		Mentionees []mentionEntry `json:"MENTIONEES"`
+	}
+	if err := json.Unmarshal([]byte(metadata["MENTION"]), &payload); err != nil {
+		t.Fatalf("failed to unmarshal MENTION metadata: %v", err)
+	}
+
+	expected := []mentionEntry{
+		{S: "3", E: "6", M: "u-zhang"},
+		{S: "7", E: "12", M: "u-kik"},
+		{S: "13", E: "19", M: "u-alice"},
+	}
+	if fmt.Sprintf("%#v", payload.Mentionees) != fmt.Sprintf("%#v", expected) {
+		t.Fatalf("MENTIONEES = %#v, want %#v", payload.Mentionees, expected)
+	}
+}
+
+func TestBuildRoomMentionMetadataUsesUTF16Offsets(t *testing.T) {
+	lc := &LineClient{}
+	metadata := lc.buildMentionMetadata(t.Context(), "你好🙂 @room", "", &event.Mentions{Room: true})
+	var payload struct {
+		Mentionees []mentionEntry `json:"MENTIONEES"`
+	}
+	if err := json.Unmarshal([]byte(metadata["MENTION"]), &payload); err != nil {
+		t.Fatalf("failed to unmarshal MENTION metadata: %v", err)
+	}
+
+	expected := []mentionEntry{{S: "5", E: "10", A: "1"}}
+	if fmt.Sprintf("%#v", payload.Mentionees) != fmt.Sprintf("%#v", expected) {
+		t.Fatalf("MENTIONEES = %#v, want %#v", payload.Mentionees, expected)
+	}
+}
+
 func TestLineGroupE2EEReconnectRequiredError(t *testing.T) {
 	err := lineGroupE2EEReconnectRequiredError(fmt.Errorf("failed to unwrap group key: %w for 5625926", e2ee.ErrMissingOwnPrivateKey))
 	if err == nil {

pkg/connector/text_offsets.go

@@ -0,0 +1,77 @@
+package connector
+
+import (
+	"strconv"
+	"unicode/utf8"
+)
+
+func resolveMentionRange(text, startOffset, endOffset string) (int, int, bool) {
+	startUTF16, err := strconv.Atoi(startOffset)
+	if err != nil {
+		return 0, 0, false
+	}
+	endUTF16, err := strconv.Atoi(endOffset)
+	if err != nil || endUTF16 <= startUTF16 {
+		return 0, 0, false
+	}
+
+	start, startOK := utf16OffsetToByteIndex(text, startUTF16)
+	end, endOK := utf16OffsetToByteIndex(text, endUTF16)
+	return start, end, startOK && endOK && end > start
+}
+
+func utf16OffsetToByteIndex(text string, offset int) (int, bool) {
+	if offset < 0 {
+		return 0, false
+	}
+
+	codeUnits := 0
+	for byteIndex := 0; byteIndex < len(text); {
+		if codeUnits == offset {
+			return byteIndex, true
+		}
+
+		r, size := utf8.DecodeRuneInString(text[byteIndex:])
+		if r <= 0xFFFF {
+			codeUnits++
+		} else {
+			codeUnits += 2
+		}
+		byteIndex += size
+
+		if codeUnits > offset {
+			return 0, false
+		}
+		if codeUnits == offset {
+			return byteIndex, true
+		}
+	}
+
+	return len(text), codeUnits == offset
+}
+
+func byteIndexToUTF16Offset(text string, byteIndex int) (int, bool) {
+	if byteIndex < 0 || byteIndex > len(text) {
+		return 0, false
+	}
+
+	codeUnits := 0
+	for currentByteIndex := 0; currentByteIndex < len(text); {
+		if currentByteIndex == byteIndex {
+			return codeUnits, true
+		}
+		if currentByteIndex > byteIndex {
+			return 0, false
+		}
+
+		r, size := utf8.DecodeRuneInString(text[currentByteIndex:])
+		if r <= 0xFFFF {
+			codeUnits++
+		} else {
+			codeUnits += 2
+		}
+		currentByteIndex += size
+	}
+
+	return codeUnits, byteIndex == len(text)
+}