-
Notifications
You must be signed in to change notification settings - Fork 242
Protect operations database from non-string data in error messages fr… #1716
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
23f87c2
fe2e153
0b1d192
06960e0
0baf6f8
3607be6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| // Copyright © 2026 Kaleido, Inc. | ||
| // | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package utils | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/hex" | ||
| "strings" | ||
| "unicode/utf8" | ||
|
|
||
| "github.com/hyperledger/firefly-common/pkg/log" | ||
| ) | ||
|
|
||
| // DBSafeUTF8StringFromPtr returns a DB-safe UTF-8 string from a pointer. | ||
| // Nil pointers return "". Strings containing invalid UTF-8 sequences or null | ||
| // bytes (which PostgreSQL rejects in text columns even though null bytes are | ||
| // technically valid UTF-8) are hex-encoded instead, with a warning logged. | ||
| func DBSafeUTF8StringFromPtr(ctx context.Context, s *string) string { | ||
| if s == nil { | ||
| return "" | ||
| } | ||
| if !utf8.ValidString(*s) || strings.ContainsRune(*s, 0) { | ||
| hexString := hex.EncodeToString([]byte(*s)) | ||
| log.L(ctx).Warnf("String contains invalid UTF-8 or null bytes - encoding as hex: %s", hexString) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. So this is an one way and then on the API surface we don't decode this right?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Correct if an error is hex encoded as it is inserted into the DB it remain hex encoded on the API surface.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. More broadly the query is whether we should decode hex bytes coming out the other side of the database in order to rehydrate the initial context. I assert that the answer is no since the entire call surface expects this field to be a human readable string, strings containing null bytes or non-UTF8 characters likely already pose a risk to consumers that are expecting strings. These would appears as \u0000 in serialized json which at best will present a garbled error message and at worst may be treated as string terminators. It's safer to present a consistently encoded string to the user to deal with these abnormal cases. Additionally due to hyperledger/firefly-evmconnect#184 and hyperledger/firefly-signer#98 a known source of these malformed error strings will now be decoded in a more friendly way preserving the context of nested ABI. Therefore this particular change should be viewed as a belt-and-braces protection for preventing unforseen types of malformed error from completely blocking transaction processing (and blocks submission of transaction with following nonces etc) due to:
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for detailed explanation here |
||
| return hexString | ||
| } | ||
| return *s | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| // Copyright © 2026 Kaleido, Inc. | ||
| // | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package utils | ||
|
|
||
| import ( | ||
| "context" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestDBSafeUTF8StringFromPtrNil(t *testing.T) { | ||
| assert.Equal(t, "", DBSafeUTF8StringFromPtr(context.Background(), nil)) | ||
| } | ||
|
|
||
| func TestDBSafeUTF8StringFromPtrValid(t *testing.T) { | ||
| msg := "FF23021: EVM reverted: some normal error message" | ||
| assert.Equal(t, msg, DBSafeUTF8StringFromPtr(context.Background(), &msg)) | ||
| } | ||
|
|
||
| func TestDBSafeUTF8StringFromPtrInvalidUTF8(t *testing.T) { | ||
| msg := "[OCPE]404/98 - \x08\xc3\x79\xa0\x00\x00\x00[TMM]404/16e" | ||
| assert.Equal(t, "5b4f4350455d3430342f3938202d2008c379a00000005b544d4d5d3430342f313665", DBSafeUTF8StringFromPtr(context.Background(), &msg)) | ||
| } | ||
|
|
||
| func TestDBSafeUTF8StringFromPtrNullBytesInValidUTF8(t *testing.T) { | ||
| msg := "hello\x00world" | ||
| assert.Equal(t, "68656c6c6f00776f726c64", DBSafeUTF8StringFromPtr(context.Background(), &msg)) | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Eventually this should be moved to FF Common