Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
// See the :ref:`load balancing architecture
// overview<arch_overview_load_balancing_types>` for more information.
//
// [#next-free-field: 9]
// [#next-free-field: 10]
message ClientSideWeightedRoundRobin {
// Whether to enable out-of-band utilization reporting collection from
// the endpoints. By default, per-request utilization reporting is used.
Expand Down Expand Up @@ -88,4 +88,9 @@ message ClientSideWeightedRoundRobin {
// Configuration for slow start mode.
// If this configuration is not set, slow start will not be not enabled.
common.v3.SlowStartConfig slow_start_config = 8;

// Optional overrides for the OOB reporting connection (alternative port,
// ``:authority``, transport socket selection). Honored only when
// ``enable_oob_load_report`` is true.
common.v3.OrcaOobReportingConfig oob_reporting_config = 9;
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import "envoy/config/route/v3/route_components.proto";
import "envoy/type/v3/percent.proto";

import "google/protobuf/duration.proto";
import "google/protobuf/struct.proto";
import "google/protobuf/wrappers.proto";

import "envoy/annotations/deprecation.proto";
Expand Down Expand Up @@ -159,3 +160,29 @@ message ConsistentHashingLbConfig {
// will be ignored.
repeated config.route.v3.RouteAction.HashPolicy hash_policy = 3;
}

// Connection overrides for the ORCA out-of-band (OOB) reporting stream, used by
// load balancing policies that consume ORCA load reports (e.g.
// :ref:`client_side_weighted_round_robin
// <envoy_v3_api_msg_extensions.load_balancing_policies.client_side_weighted_round_robin.v3.ClientSideWeightedRoundRobin>`).
// Whether and when OOB reporting runs is controlled by the embedding policy.
message OrcaOobReportingConfig {
// Optional alternative port for the OOB reporting connection, for example an
// ORCA reporting sidecar listening on a dedicated port. If 0 or unset, the
// port of the host's ORCA reporting address is used. Ignored for non-IP
// (pipe/UDS) host addresses.
uint32 port_value = 1 [(validate.rules).uint32 = {lte: 65535}];

// Value of the ``:authority`` header on the OOB gRPC stream. If empty, the
// endpoint hostname is used, then the host address, then the cluster name.
string authority = 2
[(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE strict: false}];

// Optional key/value pairs used to select a transport socket from the
// cluster's :ref:`transport_socket_matches
// <envoy_v3_api_field_config.cluster.v3.Cluster.transport_socket_matches>`
// for the OOB connection. If unset, or if no match is found, the cluster's
// default transport socket is used. ALPN ``h2`` is always forced on the OOB
// connection regardless of this setting.
google.protobuf.Struct transport_socket_match_criteria = 3;
}
2 changes: 2 additions & 0 deletions changelogs/changelogs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ areas:
title: http_inspector
jwt_authn:
title: jwt_authn
load_balancing:
title: load_balancing
load_report:
title: load_report
logging:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Added :ref:`oob_reporting_config
<envoy_v3_api_field_extensions.load_balancing_policies.client_side_weighted_round_robin.v3.ClientSideWeightedRoundRobin.oob_reporting_config>`
to the ``client_side_weighted_round_robin`` load balancing policy. It supplies optional
overrides for the ORCA out-of-band reporting connection: an alternative port (e.g. a
reporting sidecar), the ``:authority`` header, and transport socket selection via
``transport_socket_match_criteria``. Honored only when ``enable_oob_load_report`` is true.
4 changes: 3 additions & 1 deletion envoy/upstream/upstream.h
Original file line number Diff line number Diff line change
Expand Up @@ -224,12 +224,14 @@ class Host : virtual public HostDescription {
* @param transport_socket_options supplies the transport options that will be set on the new
* connection.
* @param metadata when non-null drives transport socket factory resolution.
* @param address_override when non-null, dials this address instead of orcaReportingAddress().
* @return the connection data.
*/
virtual CreateConnectionData createOrcaReportingConnection(
Event::Dispatcher& dispatcher,
Network::TransportSocketOptionsConstSharedPtr transport_socket_options,
const envoy::config::core::v3::Metadata* metadata) const PURE;
const envoy::config::core::v3::Metadata* metadata,
Network::Address::InstanceConstSharedPtr address_override = nullptr) const PURE;

/**
* @return host specific gauges.
Expand Down
9 changes: 6 additions & 3 deletions source/common/upstream/upstream_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -606,13 +606,16 @@ Host::CreateConnectionData HostImplBase::createHealthCheckConnection(
Host::CreateConnectionData HostImplBase::createOrcaReportingConnection(
Event::Dispatcher& dispatcher,
Network::TransportSocketOptionsConstSharedPtr transport_socket_options,
const envoy::config::core::v3::Metadata* metadata) const {
const auto orca_address = orcaReportingAddress();
const envoy::config::core::v3::Metadata* metadata,
Network::Address::InstanceConstSharedPtr address_override) const {
const auto orca_address = address_override != nullptr ? address_override : orcaReportingAddress();
Network::UpstreamTransportSocketFactory& factory =
(metadata != nullptr)
? resolveTransportSocketFactory(orca_address, metadata, transport_socket_options)
: transportSocketFactory();
return createConnection(dispatcher, cluster(), orca_address, addressListOrNull(), factory,
// The OOB stream dials a single address; the happy-eyeballs address list is
// intentionally not used.
return createConnection(dispatcher, cluster(), orca_address, /*address_list_or_null=*/{}, factory,
/*options=*/nullptr, transport_socket_options, shared_from_this());
}

Expand Down
3 changes: 2 additions & 1 deletion source/common/upstream/upstream_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,8 @@ class HostImplBase : public Host,
CreateConnectionData createOrcaReportingConnection(
Event::Dispatcher& dispatcher,
Network::TransportSocketOptionsConstSharedPtr transport_socket_options,
const envoy::config::core::v3::Metadata* metadata) const override;
const envoy::config::core::v3::Metadata* metadata,
Network::Address::InstanceConstSharedPtr address_override = nullptr) const override;

std::vector<std::pair<absl::string_view, Stats::PrimitiveGaugeReference>>
gauges() const override {
Expand Down
25 changes: 14 additions & 11 deletions source/extensions/clusters/common/logical_host.cc
Original file line number Diff line number Diff line change
Expand Up @@ -116,19 +116,22 @@ Upstream::Host::CreateConnectionData LogicalHost::createConnection(
Upstream::Host::CreateConnectionData LogicalHost::createOrcaReportingConnection(
Event::Dispatcher& dispatcher,
Network::TransportSocketOptionsConstSharedPtr transport_socket_options,
const envoy::config::core::v3::Metadata* metadata) const {
const Network::Address::InstanceConstSharedPtr address = orcaReportingAddress();
const SharedConstAddressVector address_list_or_null = addressListOrNull();
// Use override_transport_socket_options if set, otherwise use the passed options.
const auto& effective_options = override_transport_socket_options_ != nullptr
? override_transport_socket_options_
: transport_socket_options;
const envoy::config::core::v3::Metadata* metadata,
Network::Address::InstanceConstSharedPtr address_override) const {
const Network::Address::InstanceConstSharedPtr address =
address_override != nullptr ? address_override : orcaReportingAddress();
// OrcaOobManager passes forced HTTP/2 ALPN; override_transport_socket_options_
// would clobber it.
ASSERT(override_transport_socket_options_ == nullptr);
Network::UpstreamTransportSocketFactory& factory =
(metadata != nullptr) ? resolveTransportSocketFactory(address, metadata, effective_options)
: transportSocketFactory();
(metadata != nullptr)
? resolveTransportSocketFactory(address, metadata, transport_socket_options)
: transportSocketFactory();
// The OOB stream dials a single address; the happy-eyeballs address list is
// intentionally not used.
return HostImplBase::createConnection(
dispatcher, cluster(), address, address_list_or_null, factory,
/*options=*/nullptr, effective_options,
dispatcher, cluster(), address, /*address_list_or_null=*/{}, factory,
/*options=*/nullptr, transport_socket_options,
std::make_shared<RealHostDescription>(address, shared_from_this()));
}

Expand Down
3 changes: 2 additions & 1 deletion source/extensions/clusters/common/logical_host.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ class LogicalHost : public HostImplBase, public HostDescriptionImplBase {
Upstream::Host::CreateConnectionData createOrcaReportingConnection(
Event::Dispatcher& dispatcher,
Network::TransportSocketOptionsConstSharedPtr transport_socket_options,
const envoy::config::core::v3::Metadata* metadata) const override;
const envoy::config::core::v3::Metadata* metadata,
Network::Address::InstanceConstSharedPtr address_override = nullptr) const override;

// Upstream::HostDescription
SharedConstAddressVector addressListOrNull() const override;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,18 @@ ClientSideWeightedRoundRobinLbConfig::ClientSideWeightedRoundRobinLbConfig(
weight_update_period =
std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(lb_proto, weight_update_period, 1000));

enable_oob_load_report = lb_proto.enable_oob_load_report().value();
oob_reporting_period =
std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(lb_proto, oob_reporting_period, 10000));
oob_enabled = lb_proto.enable_oob_load_report().value();
// oob_reporting_period has no proto validation; clamp non-positive to default.
const int64_t period_ms = PROTOBUF_GET_MS_OR_DEFAULT(
lb_proto, oob_reporting_period,
Extensions::LoadBalancingPolicies::Common::kDefaultOobReportingPeriodMs);
oob_manager_config.reporting_period = std::chrono::milliseconds(
period_ms > 0 ? period_ms
: Extensions::LoadBalancingPolicies::Common::kDefaultOobReportingPeriodMs);
if (lb_proto.has_oob_reporting_config()) {
Extensions::LoadBalancingPolicies::Common::mergeOrcaOobConnectionOverrides(
lb_proto.oob_reporting_config(), oob_manager_config);
}

if (lb_proto.has_slow_start_config()) {
*round_robin_overrides_.mutable_slow_start_config() = lb_proto.slow_start_config();
Expand Down Expand Up @@ -123,10 +132,10 @@ ClientSideWeightedRoundRobinLoadBalancer::ClientSideWeightedRoundRobinLoadBalanc
// Init order relies on PrioritySetImpl::updateHosts() firing priority callbacks
// (OrcaWeightManager attaches OrcaHostLbPolicyData) before member callbacks (OrcaOobManager
// opens the session), so the data is in place before the first OOB report.
if (typed_lb_config->enable_oob_load_report) {
if (typed_lb_config->oob_enabled) {
orca_oob_manager_ =
std::make_unique<Extensions::LoadBalancingPolicies::Common::ProdOrcaOobManager>(
typed_lb_config->oob_reporting_period, priority_set,
typed_lb_config->oob_manager_config, priority_set,
typed_lb_config->main_thread_dispatcher_, random, cluster_info.statsScope(),
orca_weight_manager_->reportHandler());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ class ClientSideWeightedRoundRobinLbConfig : public Upstream::LoadBalancerConfig
std::chrono::milliseconds weight_expiration_period;
std::chrono::milliseconds weight_update_period;

bool enable_oob_load_report;
std::chrono::milliseconds oob_reporting_period;
bool oob_enabled;
Extensions::LoadBalancingPolicies::Common::OrcaOobManagerConfig oob_manager_config;

// Round robin proto overrides that we want to propagate to the worker RR LB (e.g., slow start).
RoundRobinConfig round_robin_overrides_;
Expand Down Expand Up @@ -131,7 +131,7 @@ class ClientSideWeightedRoundRobinLoadBalancer : public Upstream::ThreadAwareLoa
std::unique_ptr<Extensions::LoadBalancingPolicies::Common::OrcaWeightManager>
orca_weight_manager_;

// ORCA out-of-band manager. Constructed only when enable_oob_load_report is true; null
// ORCA out-of-band manager. Constructed only when oob_enabled; null
// otherwise. Shares the OrcaWeightManager's report handler so OOB reports feed the same
// per-host atomics as in-band reports.
std::unique_ptr<Extensions::LoadBalancingPolicies::Common::OrcaOobManager> orca_oob_manager_;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
#include "source/extensions/load_balancing_policies/client_side_weighted_round_robin/client_side_weighted_round_robin_lb.h"
#include "source/extensions/load_balancing_policies/common/factory_base.h"

#include "absl/status/status.h"

namespace Envoy {
namespace Extensions {
namespace LoadBalancingPolicies {
Expand Down
6 changes: 6 additions & 0 deletions source/extensions/load_balancing_policies/common/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -81,14 +81,20 @@ envoy_cc_library(
"//source/common/common:backoff_lib",
"//source/common/common:callback_impl_lib",
"//source/common/common:minimal_logger_lib",
"//source/common/config:well_known_names",
"//source/common/grpc:codec_lib",
"//source/common/grpc:common_lib",
"//source/common/grpc:status_lib",
"//source/common/http:codec_client_lib",
"//source/common/http:headers_lib",
"//source/common/http:utility_lib",
"//source/common/network:transport_socket_options_lib",
"//source/common/network:utility_lib",
"//source/common/protobuf:utility_lib",
"@abseil-cpp//absl/container:node_hash_map",
"@abseil-cpp//absl/status",
"@envoy_api//envoy/config/core/v3:pkg_cc_proto",
"@envoy_api//envoy/extensions/load_balancing_policies/common/v3:pkg_cc_proto",
"@xds//xds/data/orca/v3:pkg_cc_proto",
"@xds//xds/service/orca/v3:pkg_cc_proto",
],
Expand Down
Loading
Loading