Skip to content

fix: classify WAL restore errors with precise gRPC statuses #927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
leonardoce wants to merge 1 commit into
base: main
Choose a base branch
from
+75 −16
Open

fix: classify WAL restore errors with precise gRPC statuses #927

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/cloudnative-pg/cloudnative-pg v1.29.1
github.com/cloudnative-pg/cnpg-i v0.5.0
github.com/cloudnative-pg/cnpg-i-machinery v0.4.2
github.com/cloudnative-pg/machinery v0.4.0
github.com/cloudnative-pg/machinery v0.5.0
github.com/onsi/ginkgo/v2 v2.29.0
github.com/onsi/gomega v1.41.0
github.com/spf13/cobra v1.10.2
Expand Down Expand Up @@ -114,7 +114,7 @@ require (
golang.org/x/net v0.53.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.43.0 // indirect
golang.org/x/sys v0.45.0 // indirect
golang.org/x/term v0.42.0 // indirect
golang.org/x/text v0.36.0 // indirect
golang.org/x/time v0.15.0 // indirect
Expand All @@ -137,3 +137,5 @@ require (
sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect
)

replace github.com/cloudnative-pg/barman-cloud => github.com/leonardoce/barman-cloud v0.0.0-20260527084420-d2ef1c114ae8
12 changes: 6 additions & 6 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,14 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudnative-pg/api v1.29.1 h1:FWr8S7EQeOfdhYXyr2cof8wUXLARZiiQt5Qa6ltED7w=
github.com/cloudnative-pg/api v1.29.1/go.mod h1:QtWF3yzSvIfORMHaSkPAk/o3bhCJwEHJgN3riyRiz3o=
github.com/cloudnative-pg/barman-cloud v0.5.1 h1:vjkXrrxo2DQXHT9u9usqhtaHiPZ/lTfDVs/pIWYTepQ=
github.com/cloudnative-pg/barman-cloud v0.5.1/go.mod h1:XPc5IUFP1y4cZX1sg+Pd8j9V4tmUEVnv3BGCpfQOOg8=
github.com/cloudnative-pg/cloudnative-pg v1.29.1 h1:ZNEt1TMlnQKXI1kho2UqQuqdfvIvjGln4kN7C1lsmGA=
github.com/cloudnative-pg/cloudnative-pg v1.29.1/go.mod h1:Sbgx9jVmkle4/gR2U5JHrzDd74sRPOBHDtPkvncg5v8=
github.com/cloudnative-pg/cnpg-i v0.5.0 h1:/TOzpNT6cwNgrpftTtrnLKdoHgMwd+88vZgXjlVgXeE=
github.com/cloudnative-pg/cnpg-i v0.5.0/go.mod h1:7Gh4+UzhBpGhr4DreB1GN9wGYfvxwXCXZUyVt3zE/3I=
github.com/cloudnative-pg/cnpg-i-machinery v0.4.2 h1:0reS9MtyLYINHXQ/MfxJ9jp39hhBf8e3Qdj+T5Nsq6I=
github.com/cloudnative-pg/cnpg-i-machinery v0.4.2/go.mod h1:gvrKabgxXq0zGthXGucemDdsxakLEQDMxn43M4HLW30=
github.com/cloudnative-pg/machinery v0.4.0 h1:3sfqrBptH4QQSVB4g10Z+7aiQRnh4g+6AsqsK0ibKaQ=
github.com/cloudnative-pg/machinery v0.4.0/go.mod h1:OIwaTYAnLv8PmBmBvEf0BvMK2JBX6J+naTMg9UgV1FQ=
github.com/cloudnative-pg/machinery v0.5.0 h1:hhTnkzn+AiN3NmbjCQ6RXj5rfqV3K6arzq6kdXAzcnQ=
github.com/cloudnative-pg/machinery v0.5.0/go.mod h1:uuFjqBUjWn0a9uvAk1ixTSzPM0PrjaS+QiKLOIBqLm4=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down Expand Up @@ -143,6 +141,8 @@ github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0 h1:bMqrb3UHgHbP+
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.4.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/leonardoce/barman-cloud v0.0.0-20260527084420-d2ef1c114ae8 h1:svNVHqtp2a7FfxuLPTm7bs9esj/tAp4arYY/8hvK3no=
github.com/leonardoce/barman-cloud v0.0.0-20260527084420-d2ef1c114ae8/go.mod h1:DACs5rdlMIQ7mUOpJVBRTrbf+d7OGyMOLsFUEMJ2JV8=
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
Expand Down Expand Up @@ -275,8 +275,8 @@ golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
Expand Down
48 changes: 44 additions & 4 deletions internal/cnpgi/common/errors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,50 @@ var ErrEndOfWALStreamReached = status.Errorf(codes.OutOfRange, "end of WAL reach
var ErrMissingPermissions = status.Errorf(codes.FailedPrecondition,
"no permission to download the backup credentials, retrying")

// newWALNotFoundError returns a error that states that a
// certain WAL file has not been found. This error is
// compatible with GRPC status codes, resulting in a 404
// being used as a response code.
// newWALNotFoundError returns an error indicating that the
// requested WAL file is not present in the object store.
// It carries a gRPC NotFound status so the operator can
// treat it as a terminal condition.
func newWALNotFoundError(walName string) error {
return status.Errorf(codes.NotFound, "wal %q not found", walName)
}

// newUnavailableError returns an error indicating that
// downloading the given WAL file failed for a transient
// reason (e.g. a connectivity blip). It carries a gRPC
// Unavailable status so the operator will retry the
// request.
func newUnavailableError(walName string, err error) error {
return status.Errorf(
codes.Unavailable,
"transient error while downloading %q: %s",
walName,
err.Error(),
)
}

// newInvalidWALNameError returns an error indicating that
// the requested WAL name is not valid. It carries a gRPC
// InvalidArgument status so the operator treats it as a
// terminal condition.
func newInvalidWALNameError(walName string, err error) error {
return status.Errorf(
codes.InvalidArgument,
"invalid WAL name %q: %s",
walName,
err.Error(),
)
}

// newInternalWALRestoreError returns an error indicating
// that downloading the given WAL file failed for an
// unclassified reason. It carries a gRPC Internal status
// so the operator treats it as a terminal condition.
func newInternalWALRestoreError(walName string, err error) error {
return status.Errorf(
codes.Internal,
"internal error while downloading %q: %s",
walName,
err.Error(),
)
}
25 changes: 21 additions & 4 deletions internal/cnpgi/common/wal.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,11 +365,28 @@ func (w WALServiceImplementation) restoreFromBarmanObjectStore(
// is the one that PostgreSQL has requested to restore.
// The failure has already been logged in walRestorer.RestoreList method
if walStatus[0].Err != nil {
if errors.Is(walStatus[0].Err, barmanRestorer.ErrWALNotFound) {
return newWALNotFoundError(walStatus[0].WalName)
walName := walStatus[0].WalName
walErr := walStatus[0].Err
switch {
case errors.Is(walErr, barmanRestorer.ErrWALNotFound):
// A missing WAL is a terminal condition: surface it as a
// gRPC NotFound so the operator stops retrying.
return newWALNotFoundError(walName)
case errors.Is(walErr, barmanRestorer.ErrInvalidWalName):
// A malformed WAL name will never succeed on retry.
return newInvalidWALNameError(walName, walErr)
case errors.Is(walErr, barmanRestorer.ErrConnectivity),
errors.Is(walErr, barmanRestorer.ErrGeneric):
// Connectivity and generic barman-cloud failures are
// considered transient: surface them as gRPC Unavailable
// so the operator retries the download.
return newUnavailableError(walName, walErr)
default:
// Unrecognized exit codes and unexpected failures (e.g. the
// barman-cloud command could not be executed) are surfaced
// as gRPC Internal so the operator treats them as terminal.
return newInternalWALRestoreError(walName, walErr)
}

return walStatus[0].Err
}

// We skip this step if streaming connection is not available
Expand Down
Loading