Properly escape null character in the consumer too

cmd/consumer/consumer.go

@@ -10,7 +10,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"regexp"
 	"strings"
 	"time"
 
@@ -28,6 +27,7 @@ import (
 	"github.com/uabluerail/indexer/models"
 	"github.com/uabluerail/indexer/pds"
 	"github.com/uabluerail/indexer/repo"
+	"github.com/uabluerail/indexer/util/fix"
 	"github.com/uabluerail/indexer/util/resolver"
 )
 
@@ -211,12 +211,6 @@ func (c *Consumer) updateCursor(ctx context.Context, seq int64) error {
 
 }
 
-var postgresFixRegexp = regexp.MustCompile(`[^\\](\\\\)*(\\u0000)`)
-
-func escapeNullCharForPostgres(b []byte) []byte {
-	return postgresFixRegexp.ReplaceAll(b, []byte(`$1<0x00>`))
-}
-
 func (c *Consumer) processMessage(ctx context.Context, typ string, r io.Reader, first bool) error {
 	log := zerolog.Ctx(ctx)
 
@@ -331,7 +325,7 @@ func (c *Consumer) processMessage(ctx context.Context, typ string, r io.Reader,
 				// XXX: proper replacement of \u0000 would require full parsing of JSON
 				// and recursive iteration over all string values, but this
 				// should work well enough for now.
-				Content: escapeNullCharForPostgres(v),
+				Content: fix.EscapeNullCharForPostgres(v),
 				AtRev:   payload.Rev,
 			})
 		}

cmd/record-indexer/workerpool.go

@@ -21,6 +21,7 @@ import (
 	"github.com/uabluerail/indexer/models"
 	"github.com/uabluerail/indexer/pds"
 	"github.com/uabluerail/indexer/repo"
+	"github.com/uabluerail/indexer/util/fix"
 	"github.com/uabluerail/indexer/util/resolver"
 )
 
@@ -132,14 +133,6 @@ func (p *WorkerPool) worker(ctx context.Context, signal chan struct{}) {
 	}
 }
 
-var postgresFixRegexp = regexp.MustCompile(`([^\\](\\\\)*)(\\u0000)+`)
-
-func escapeNullCharForPostgres(b []byte) []byte {
-	return postgresFixRegexp.ReplaceAllFunc(b, func(b []byte) []byte {
-		return bytes.ReplaceAll(b, []byte(`\u0000`), []byte(`<0x00>`))
-	})
-}
-
 func (p *WorkerPool) doWork(ctx context.Context, work WorkItem) error {
 	log := zerolog.Ctx(ctx)
 	defer close(work.signal)
@@ -244,7 +237,7 @@ retry:
 			// XXX: proper replacement of \u0000 would require full parsing of JSON
 			// and recursive iteration over all string values, but this
 			// should work well enough for now.
-			Content: escapeNullCharForPostgres(v),
+			Content: fix.EscapeNullCharForPostgres(v),
 			AtRev:   newRev,
 		})
 	}

util/fix/postgres_json.go

@@ -0,0 +1,14 @@
+package fix
+
+import (
+	"bytes"
+	"regexp"
+)
+
+var postgresFixRegexp = regexp.MustCompile(`([^\\](\\\\)*)(\\u0000)+`)
+
+func EscapeNullCharForPostgres(b []byte) []byte {
+	return postgresFixRegexp.ReplaceAllFunc(b, func(b []byte) []byte {
+		return bytes.ReplaceAll(b, []byte(`\u0000`), []byte(`<0x00>`))
+	})
+}

util/fix/postgres_json_test.go

@@ -1,4 +1,4 @@
-package main
+package fix
 
 import (
 	"testing"
@@ -18,7 +18,7 @@ func TestPostgresFix(t *testing.T) {
 	}
 
 	for _, tc := range cases {
-		got := escapeNullCharForPostgres([]byte(tc.input))
+		got := EscapeNullCharForPostgres([]byte(tc.input))
 		if string(got) != tc.want {
 			t.Errorf("escapeNullCharForPostgres(%s) = %s, want %s", tc.input, string(got), tc.want)
 		}