Use ListOffchainData endpoint instead of GetOffchainData

dataavailability/datacommittee/datacommittee.go

@@ -89,58 +89,136 @@ func (d *DataCommitteeBackend) Init() error {
 
 // GetSequence gets backend data one hash at a time. This should be optimized on the DAC side to get them all at once.
 func (d *DataCommitteeBackend) GetSequence(ctx context.Context, hashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) {
-	// TODO: optimize this on the DAC side by implementing a multi batch retrieve api
 	var batchData [][]byte
-	for _, h := range hashes {
-		data, err := d.GetBatchL2Data(h)
+
+	const batchSize = 5
+	for i := 0; i < len(hashes); i += batchSize {
+		end := i + batchSize
+		if end > len(hashes) {
+			end = len(hashes)
+		}
+
+		data, err := d.ListBatchL2Data(ctx, hashes[i:end])
 		if err != nil {
 			return nil, err
 		}
-		batchData = append(batchData, data)
+
+		batchData = append(batchData, data...)
 	}
+
 	return batchData, nil
 }
 
-// GetBatchL2Data returns the data from the DAC. It checks that it matches with the expected hash
-func (d *DataCommitteeBackend) GetBatchL2Data(hash common.Hash) ([]byte, error) {
+// ListBatchL2Data returns the offchain data from the DAC by the given list of hashes.
+// It checks that the returned data matches with the expected hashes.
+func (d *DataCommitteeBackend) ListBatchL2Data(ctx context.Context, hashes []common.Hash) ([][]byte, error) {
+	var batchData [][]byte
+
 	intialMember := d.selectedCommitteeMember
-	found := false
-	for !found && intialMember != -1 {
+	for intialMember != -1 {
 		member := d.committeeMembers[d.selectedCommitteeMember]
-		log.Infof("trying to get data from %s at %s", member.Addr.Hex(), member.URL)
 		c := d.dataCommitteeClientFactory.New(member.URL)
-		data, err := c.GetOffChainData(d.ctx, hash)
-		if err != nil {
-			log.Warnf(
-				"error getting data from DAC node %s at %s: %s",
-				member.Addr.Hex(), member.URL, err,
-			)
-			d.selectedCommitteeMember = (d.selectedCommitteeMember + 1) % len(d.committeeMembers)
-			if d.selectedCommitteeMember == intialMember {
+		dataMap := make(map[common.Hash][]byte)
+
+		log.Infof("trying to list off-chain data from %s at %s", member.Addr.Hex(), member.URL)
+
+		// The offchain data is going to be fetched depending on the DAC version
+		if _, err := c.GetStatus(ctx); err == nil {
+			// The endpoint to get DAC status was implemented after the one to list offchain data.
+			// Meaning the endpoint is there is the status endpoint does not return error.
+			// No needed to check the version here.
+			if dataMap, err = c.ListOffChainData(ctx, hashes); err != nil {
+				log.Warnf(
+					"error getting data from DAC node %s at %s: %s",
+					member.Addr.Hex(), member.URL, err,
+				)
+				d.selectedCommitteeMember = (d.selectedCommitteeMember + 1) % len(d.committeeMembers)
+				if d.selectedCommitteeMember == intialMember {
+					break
+				}
+				continue
+			}
+		} else {
+			valid := true
+			outOfCommittee := false
+			for _, hash := range hashes {
+				data, err := c.GetOffChainData(ctx, hash)
+				if err != nil {
+					log.Warnf(
+						"error getting data from DAC node %s at %s: %s",
+						member.Addr.Hex(), member.URL, err,
+					)
+					d.selectedCommitteeMember = (d.selectedCommitteeMember + 1) % len(d.committeeMembers)
+					if d.selectedCommitteeMember == intialMember {
+						outOfCommittee = true
+						break
+					}
+
+					valid = false
+					break
+				}
+
+				dataMap[hash] = data
+			}
+
+			if outOfCommittee {
 				break
 			}
-			continue
+
+			if !valid {
+				continue
+			}
 		}
-		actualTransactionsHash := crypto.Keccak256Hash(data)
-		if actualTransactionsHash != hash {
-			unexpectedHash := fmt.Errorf(
-				unexpectedHashTemplate, hash, actualTransactionsHash,
-			)
-			log.Warnf(
-				"error getting data from DAC node %s at %s: %s",
-				member.Addr.Hex(), member.URL, unexpectedHash,
-			)
-			d.selectedCommitteeMember = (d.selectedCommitteeMember + 1) % len(d.committeeMembers)
-			if d.selectedCommitteeMember == intialMember {
+
+		valid := true
+		outOfCommittee := false
+		for _, hash := range hashes {
+			data, ok := dataMap[hash]
+			if !ok {
+				log.Warnf(
+					"error getting data from DAC node %s at %s: hash %s not found",
+					member.Addr.Hex(), member.URL, hash,
+				)
+
+				valid = false
+				break
+			}
+
+			actualTransactionsHash := crypto.Keccak256Hash(data)
+			if actualTransactionsHash != hash {
+				unexpectedHash := fmt.Errorf(unexpectedHashTemplate, hash, actualTransactionsHash)
+				log.Warnf(
+					"error getting data from DAC node %s at %s: %s",
+					member.Addr.Hex(), member.URL, unexpectedHash,
+				)
+				d.selectedCommitteeMember = (d.selectedCommitteeMember + 1) % len(d.committeeMembers)
+				if d.selectedCommitteeMember == intialMember {
+					outOfCommittee = true
+					break
+				}
+
+				valid = false
 				break
 			}
+
+			batchData = append(batchData, data)
+		}
+
+		if outOfCommittee {
+			break
+		}
+
+		if !valid {
 			continue
 		}
-		return data, nil
+
+		return batchData, nil
 	}
+
 	if err := d.Init(); err != nil {
 		return nil, fmt.Errorf("error loading data committee: %s", err)
 	}
+
 	return nil, fmt.Errorf("couldn't get the data from any committee member")
 }
 
@@ -201,14 +279,14 @@ func (s *DataCommitteeBackend) PostSequence(ctx context.Context, batchesData [][
 	// Stop requesting as soon as we have N valid signatures
 	cancelSignatureCollection()
 
-	return buildSignaturesAndAddrs(signatureMsgs(msgs), committee.Members), nil
+	return buildSignaturesAndAddrs(msgs, committee.Members), nil
 }
 
 func requestSignatureFromMember(ctx context.Context, signedSequence daTypes.SignedSequence, member DataCommitteeMember, ch chan signatureMsg) {
 	// request
 	c := client.New(member.URL)
 	log.Infof("sending request to sign the sequence to %s at %s", member.Addr.Hex(), member.URL)
-	signature, err := c.SignSequence(signedSequence)
+	signature, err := c.SignSequence(ctx, signedSequence)
 	if err != nil {
 		ch <- signatureMsg{
 			addr: member.Addr,

dataavailability/datacommittee/datacommittee_test.go

@@ -8,7 +8,7 @@ import (
 	"github.com/0xPolygonHermez/zkevm-node/log"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethclient/simulated"
 	"github.com/stretchr/testify/assert"
@@ -99,7 +99,7 @@ func newSimulatedDacman(t *testing.T, auth *bind.TransactOpts) (
 	// 10000000 ETH in wei
 	balance, _ := new(big.Int).SetString("10000000000000000000000000", 10) //nolint:gomnd
 	address := auth.From
-	genesisAlloc := map[common.Address]core.GenesisAccount{
+	genesisAlloc := map[common.Address]types.Account{
 		address: {
 			Balance: balance,
 		},

etherman/simulated.go

@@ -17,7 +17,7 @@ import (
 	"github.com/0xPolygonHermez/zkevm-node/log"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethclient/simulated"
 )
@@ -32,7 +32,7 @@ func NewSimulatedEtherman(cfg Config, auth *bind.TransactOpts, daBackend dataAva
 	// 10000000 ETH in wei
 	balance, _ := new(big.Int).SetString("10000000000000000000000000", 10) //nolint:gomnd
 	address := auth.From
-	genesisAlloc := map[common.Address]core.GenesisAccount{
+	genesisAlloc := map[common.Address]types.Account{
 		address: {
 			Balance: balance,
 		},

go.mod

@@ -1,36 +1,36 @@
 module github.com/0xPolygonHermez/zkevm-node
 
-go 1.21
+go 1.21.3
 
 require (
-	github.com/0xPolygonHermez/zkevm-data-streamer v0.1.18
+	github.com/0xPolygonHermez/zkevm-data-streamer v0.1.19
 	github.com/didip/tollbooth/v6 v6.1.2
 	github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127
-	github.com/ethereum/go-ethereum v1.13.11
+	github.com/ethereum/go-ethereum v1.13.14
 	github.com/go-git/go-billy/v5 v5.5.0
 	github.com/go-git/go-git/v5 v5.11.0
 	github.com/gobuffalo/packr/v2 v2.8.3
-	github.com/google/uuid v1.5.0
+	github.com/google/uuid v1.6.0
 	github.com/habx/pg-commands v0.6.1
 	github.com/hermeznetwork/tracerr v0.3.2
-	github.com/iden3/go-iden3-crypto v0.0.15
+	github.com/iden3/go-iden3-crypto v0.0.16
 	github.com/invopop/jsonschema v0.12.0
 	github.com/jackc/pgconn v1.14.1
 	github.com/jackc/pgx/v4 v4.18.1
 	github.com/mitchellh/mapstructure v1.5.0
-	github.com/prometheus/client_model v0.5.0
-	github.com/prometheus/common v0.45.0
+	github.com/prometheus/client_model v0.6.0
+	github.com/prometheus/common v0.49.0
 	github.com/rubenv/sql-migrate v1.6.1
 	github.com/spf13/afero v1.11.0
 	github.com/spf13/viper v1.18.2
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/umbracle/ethgo v0.1.4-0.20230712173909-df37dddf16f0
-	github.com/urfave/cli/v2 v2.26.0
-	go.uber.org/zap v1.26.0
-	golang.org/x/crypto v0.18.0
-	golang.org/x/net v0.20.0
-	golang.org/x/sync v0.5.0
-	google.golang.org/grpc v1.60.1
+	github.com/urfave/cli/v2 v2.27.1
+	go.uber.org/zap v1.27.0
+	golang.org/x/crypto v0.19.0
+	golang.org/x/net v0.21.0
+	golang.org/x/sync v0.6.0
+	google.golang.org/grpc v1.62.0
 	google.golang.org/protobuf v1.32.0
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -67,7 +67,7 @@ require (
 	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
-	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
+	github.com/fjl/memsize v0.0.2 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
 	github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect
@@ -89,7 +89,7 @@ require (
 	github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
 	github.com/hashicorp/go-bexpr v0.1.10 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 // indirect
+	github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
 	github.com/huin/goupnp v1.3.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
@@ -116,7 +116,6 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
-	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/miguelmota/go-solidity-sha3 v0.1.1 // indirect
 	github.com/mitchellh/pointerstructure v1.2.0 // indirect
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
@@ -140,7 +139,7 @@ require (
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/status-im/keycard-go v0.2.0 // indirect
-	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/supranational/blst v0.3.11 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
@@ -153,12 +152,12 @@ require (
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
-	golang.org/x/term v0.16.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/term v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.15.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
@@ -172,10 +171,9 @@ require (
 )
 
 require (
-	github.com/0xPolygon/agglayer v0.0.1
-	github.com/0xPolygon/cdk-data-availability v0.0.5
+	github.com/0xPolygon/agglayer v0.1.0
+	github.com/0xPolygon/cdk-data-availability v0.0.0-20240326192934-3ab9d9a79b72
 	github.com/fatih/color v1.16.0
-	github.com/joho/godotenv v1.5.1
-	github.com/prometheus/client_golang v1.18.0
+	github.com/prometheus/client_golang v1.19.0
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
 )