Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion internal/operations/operation_updater.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/hyperledger/firefly/internal/txcommon"
"github.com/hyperledger/firefly/pkg/core"
"github.com/hyperledger/firefly/pkg/database"
"github.com/hyperledger/firefly/pkg/utils"
)

type operationUpdaterBatch struct {
Expand Down Expand Up @@ -422,7 +423,7 @@ func (ou *operationUpdater) resolveOperation(ctx context.Context, ns string, id
update = update.Set("status", status)
}
if errorMsg != nil {
update = update.Set("error", *errorMsg)
update = update.Set("error", utils.DBSafeUTF8StringFromPtr(ctx, errorMsg))
}
if output != nil {
update = update.Set("output", output)
Expand Down
113 changes: 113 additions & 0 deletions internal/operations/operation_updater_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,119 @@ func TestDoUpdateVerifyBatchManifestFail(t *testing.T) {
mdi.AssertExpectations(t)
}

func TestResolveOperationValidUTF8ErrorPassesThrough(t *testing.T) {
ou := newTestOperationUpdaterNoConcurrency(t)
defer ou.close()

opID1 := fftypes.NewUUID()
mdi := ou.database.(*databasemocks.Plugin)
mdi.On("GetOperations", mock.Anything, mock.Anything, mock.Anything).Return([]*core.Operation{
{ID: opID1, Namespace: "ns1", Type: core.OpTypeBlockchainInvoke},
}, nil, nil)
mdi.On("UpdateOperation", mock.Anything, "ns1", opID1, mock.Anything, mock.MatchedBy(updateMatcher([][]string{
{"status", "Failed"},
{"error", "FF23021: EVM reverted: some normal error message"},
}))).Return(true, nil)

ou.initQueues()

err := ou.doBatchUpdate(ou.ctx, []*core.OperationUpdate{
{NamespacedOpID: "ns1:" + opID1.String(), Status: core.OpStatusFailed, ErrorMessage: "FF23021: EVM reverted: some normal error message"},
})
assert.NoError(t, err)

mdi.AssertExpectations(t)
}

func TestResolveOperationInvalidUTF8ErrorHexEncoded(t *testing.T) {
ou := newTestOperationUpdaterNoConcurrency(t)
defer ou.close()

opID1 := fftypes.NewUUID()

// Simulate the actual revert scenario: readable text with embedded ABI-encoded Error(string)
// selector bytes (0x08, 0xc3, 0x79, 0xa0) and null byte padding, which is invalid UTF-8
invalidMsg := "[OCPE]404/98 - \x08\xc3\x79\xa0\x00\x00\x00[TMM]404/16e"
expectedHex := "5b4f4350455d3430342f3938202d2008c379a00000005b544d4d5d3430342f313665"

mdi := ou.database.(*databasemocks.Plugin)
mdi.On("GetOperations", mock.Anything, mock.Anything, mock.Anything).Return([]*core.Operation{
{ID: opID1, Namespace: "ns1", Type: core.OpTypeBlockchainInvoke},
}, nil, nil)
mdi.On("UpdateOperation", mock.Anything, "ns1", opID1, mock.Anything, mock.MatchedBy(updateMatcher([][]string{
{"status", "Failed"},
{"error", expectedHex},
}))).Return(true, nil)

ou.initQueues()

err := ou.doBatchUpdate(ou.ctx, []*core.OperationUpdate{
{NamespacedOpID: "ns1:" + opID1.String(), Status: core.OpStatusFailed, ErrorMessage: invalidMsg},
})
assert.NoError(t, err)

mdi.AssertExpectations(t)
}

func TestResolveOperationNullBytesOnlyInvalidUTF8(t *testing.T) {
ou := newTestOperationUpdaterNoConcurrency(t)
defer ou.close()

opID1 := fftypes.NewUUID()

// Null bytes mixed with non-continuation bytes that break UTF-8 validity
invalidMsg := "error\x00with\x80null"
expectedHex := "6572726f720077697468806e756c6c"

mdi := ou.database.(*databasemocks.Plugin)
mdi.On("GetOperations", mock.Anything, mock.Anything, mock.Anything).Return([]*core.Operation{
{ID: opID1, Namespace: "ns1", Type: core.OpTypeBlockchainInvoke},
}, nil, nil)
mdi.On("UpdateOperation", mock.Anything, "ns1", opID1, mock.Anything, mock.MatchedBy(updateMatcher([][]string{
{"status", "Failed"},
{"error", expectedHex},
}))).Return(true, nil)

ou.initQueues()

err := ou.doBatchUpdate(ou.ctx, []*core.OperationUpdate{
{NamespacedOpID: "ns1:" + opID1.String(), Status: core.OpStatusFailed, ErrorMessage: invalidMsg},
})
assert.NoError(t, err)

mdi.AssertExpectations(t)
}

func TestResolveOperationNullBytesInValidUTF8HexEncoded(t *testing.T) {
ou := newTestOperationUpdaterNoConcurrency(t)
defer ou.close()

opID1 := fftypes.NewUUID()

// Pure null bytes embedded in otherwise valid UTF-8 text.
// utf8.ValidString returns true for this, but PostgreSQL rejects 0x00 in text columns.
invalidMsg := "hello\x00world"
expectedHex := "68656c6c6f00776f726c64"

mdi := ou.database.(*databasemocks.Plugin)
mdi.On("GetOperations", mock.Anything, mock.Anything, mock.Anything).Return([]*core.Operation{
{ID: opID1, Namespace: "ns1", Type: core.OpTypeBlockchainInvoke},
}, nil, nil)
mdi.On("UpdateOperation", mock.Anything, "ns1", opID1, mock.Anything, mock.MatchedBy(updateMatcher([][]string{
{"status", "Failed"},
{"error", expectedHex},
}))).Return(true, nil)

ou.initQueues()

err := ou.doBatchUpdate(ou.ctx, []*core.OperationUpdate{
{NamespacedOpID: "ns1:" + opID1.String(), Status: core.OpStatusFailed, ErrorMessage: invalidMsg},
})
assert.NoError(t, err)

mdi.AssertExpectations(t)
}

func TestDoUpdateVerifyBlobManifestFail(t *testing.T) {
ou := newTestOperationUpdaterNoConcurrency(t)
defer ou.close()
Expand Down
42 changes: 42 additions & 0 deletions pkg/utils/dbstring.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// Copyright © 2026 Kaleido, Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package utils

import (
"context"
"encoding/hex"
"strings"
"unicode/utf8"

"github.com/hyperledger/firefly-common/pkg/log"
)

// DBSafeUTF8StringFromPtr returns a DB-safe UTF-8 string from a pointer.
// Nil pointers return "". Strings containing invalid UTF-8 sequences or null
// bytes (which PostgreSQL rejects in text columns even though null bytes are
// technically valid UTF-8) are hex-encoded instead, with a warning logged.
func DBSafeUTF8StringFromPtr(ctx context.Context, s *string) string {
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Eventually this should be moved to FF Common

if s == nil {
return ""
}
if !utf8.ValidString(*s) || strings.ContainsRune(*s, 0) {
hexString := hex.EncodeToString([]byte(*s))
log.L(ctx).Warnf("String contains invalid UTF-8 or null bytes - encoding as hex: %s", hexString)
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So this is an one way and then on the API surface we don't decode this right?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Correct if an error is hex encoded as it is inserted into the DB it remain hex encoded on the API surface.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

More broadly the query is whether we should decode hex bytes coming out the other side of the database in order to rehydrate the initial context.

I assert that the answer is no since the entire call surface expects this field to be a human readable string, strings containing null bytes or non-UTF8 characters likely already pose a risk to consumers that are expecting strings. These would appears as \u0000 in serialized json which at best will present a garbled error message and at worst may be treated as string terminators. It's safer to present a consistently encoded string to the user to deal with these abnormal cases.

Additionally due to hyperledger/firefly-evmconnect#184 and hyperledger/firefly-signer#98 a known source of these malformed error strings will now be decoded in a more friendly way preserving the context of nested ABI.

Therefore this particular change should be viewed as a belt-and-braces protection for preventing unforseen types of malformed error from completely blocking transaction processing (and blocks submission of transaction with following nonces etc) due to:

[2026-02-27T12:37:44.884Z] ERROR SQL update failed: pq: invalid byte sequence for encoding "UTF8": 0x00 sql=[ UPDATE operations SET opstatus = $1, error = $2, updated = $3 WHERE (id = $4 AND namespace = $5) ] dbtx=Qirvs5Un ns=ff1 opupdater=opu_004 pid=....

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for detailed explanation here

return hexString
}
return *s
}
43 changes: 43 additions & 0 deletions pkg/utils/dbstring_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// Copyright © 2026 Kaleido, Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package utils

import (
"context"
"testing"

"github.com/stretchr/testify/assert"
)

func TestDBSafeUTF8StringFromPtrNil(t *testing.T) {
assert.Equal(t, "", DBSafeUTF8StringFromPtr(context.Background(), nil))
}

func TestDBSafeUTF8StringFromPtrValid(t *testing.T) {
msg := "FF23021: EVM reverted: some normal error message"
assert.Equal(t, msg, DBSafeUTF8StringFromPtr(context.Background(), &msg))
}

func TestDBSafeUTF8StringFromPtrInvalidUTF8(t *testing.T) {
msg := "[OCPE]404/98 - \x08\xc3\x79\xa0\x00\x00\x00[TMM]404/16e"
assert.Equal(t, "5b4f4350455d3430342f3938202d2008c379a00000005b544d4d5d3430342f313665", DBSafeUTF8StringFromPtr(context.Background(), &msg))
}

func TestDBSafeUTF8StringFromPtrNullBytesInValidUTF8(t *testing.T) {
msg := "hello\x00world"
assert.Equal(t, "68656c6c6f00776f726c64", DBSafeUTF8StringFromPtr(context.Background(), &msg))
}
Loading