[improve][broker] Simplify listener config and support proxyless Admin API (complete PIP-61/PIP-95)

conf/broker.conf

@@ -38,16 +38,22 @@ configurationMetadataSyncEventTopic=
 # The metadata store URL for the configuration data. If empty, we fall back to use metadataStoreUrl
 configurationMetadataStoreUrl=
 
-# Broker data port
+# Port for the Pulsar binary protocol of the internal listener.
+# The same port number is used both for the local socket binding (with bindAddress) and for the
+# advertised URL (with advertisedAddress), so no entry in advertisedListeners or bindAddresses
+# is required for the internal listener.
 brokerServicePort=6650
 
-# Broker data port for TLS - By default TLS is disabled
+# Port for the Pulsar binary protocol TLS endpoint of the internal listener.
+# Used both for the local socket binding and the advertised URL. By default TLS is disabled.
 brokerServicePortTls=
 
-# Port to use to server HTTP request
+# Port for the HTTP admin/REST endpoint of the internal listener.
+# Used both for the local socket binding and the advertised URL.
 webServicePort=8080
 
-# Port to use to server HTTPS request - By default TLS is disabled
+# Port for the HTTPS admin/REST endpoint of the internal listener.
+# Used both for the local socket binding and the advertised URL. By default TLS is disabled.
 webServicePortTls=
 
 # Specify the tls protocols the broker's web service will use to negotiate during TLS handshake
@@ -62,27 +68,63 @@ webServiceTlsProtocols=
 # webServiceTlsCiphers=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
 webServiceTlsCiphers=
 
-# Hostname or IP address the service binds on, default is 0.0.0.0.
+# Local network interface IP for the internal listener's port bindings.
+# Use a specific local IP to bind to a single interface, or 0.0.0.0 to bind on all interfaces.
 bindAddress=0.0.0.0
 
-# Extra bind addresses for the service: <listener_name>:<scheme>://<host>:<port>,[...]
+# Per-listener socket bindings.
+# The internal listener's bindings are derived automatically from bindAddress plus the port
+# properties (brokerServicePort / brokerServicePortTls / webServicePort / webServicePortTls)
+# and do not need to be repeated here.
+# PIP-95 smart listener selection routes a connection to the listener whose port it arrived on.
+# For the Pulsar binary protocol (pulsar / pulsar+ssl) this is optional — clients can also pass
+# an explicit listenerName. For the HTTP/HTTPS Admin API (http / https) it is the only routing
+# mechanism, so every HTTP/HTTPS advertised listener reachable from outside the cluster needs a
+# dedicated entry here on a unique port. This lets a layer-4 TCP load balancer serve the Admin
+# API directly per listener, without an HTTP reverse proxy.
+# Format: comma-separated <listener_name>:<scheme>://<ip>:<port> entries.
+# Supported schemes: pulsar, pulsar+ssl, http, https.
+# The <ip> part selects which local network interface the port binds to: use a specific local
+# IP, or 0.0.0.0 to bind on all interfaces. A local hostname is also accepted but not
+# recommended.
+# Each ip:port may be bound by exactly one (listener, scheme) pair. An entry that exactly
+# matches the auto-derived internal-listener binding is tolerated; assigning the same ip:port
+# to a different listener or scheme fails validation.
+# Port 0 (OS-assigned ephemeral port) is supported only for the internal listener.
 bindAddresses=
 
-# Hostname or IP address the service advertises to the outside world.
-# If not set, the value of InetAddress.getLocalHost().getCanonicalHostName() is used.
+# Hostname or IP advertised to clients for the internal listener.
+# Combined with the *Port properties it forms the internal listener's advertised URLs
+# (e.g. pulsar://<advertisedAddress>:<brokerServicePort>).
+# If not set, defaults to InetAddress.getLocalHost().getCanonicalHostName().
 advertisedAddress=
 
-# Used to specify multiple advertised listeners for the broker.
-# The value must format as <listener_name>:pulsar://<host>:<port>,
-# multiple listeners should separate with commas.
-# Do not use this configuration with advertisedAddress and brokerServicePort.
-# The Default value is absent means use advertisedAddress and brokerServicePort.
+# Declares additional advertised listeners — typically external listeners that complement the
+# internal one.
+# Format: comma-separated <listener_name>:<scheme>://<host>:<port> entries.
+# Supported schemes: pulsar, pulsar+ssl, http, https.
+# A listener name may be repeated to declare multiple schemes for the same listener.
+# The internal listener is auto-configured from advertisedAddress plus the *Port properties
+# (brokerServicePort / brokerServicePortTls / webServicePort / webServicePortTls) so it does
+# not need an entry here. URLs declared here under internalListenerName do override that
+# auto-configured listener, but this is not recommended because it can route cluster-internal
+# traffic through an external endpoint (for example, an external load balancer).
+# Legacy fallback: when internalListenerName is left blank, the first listener parsed from this
+# property is used as the internal listener (so in that case its entry here is required).
 # advertisedListeners=
 
-# Used to specify the internal listener name for the broker.
-# The listener name must contain in the advertisedListeners.
-# The Default value is absent, the broker uses the first listener as the internal listener.
-# internalListenerName=
+# Name of the listener used for cluster-internal broker-to-broker communication (lookup
+# redirects, admin forwarding to owner or leader broker). Defaults to "internal".
+# When set (the default), the internal listener is auto-configured from advertisedAddress plus
+# the *Port properties. The internal listener must advertise addresses reachable from other
+# brokers in the same cluster; avoid overriding its URLs in advertisedListeners to point at an
+# external load balancer because that would route cluster-internal traffic outside the cluster.
+# For external clients, declare a separate non-internal listener in advertisedListeners
+# instead. The default name "internal" also keeps the port-derived internal listener distinct
+# from any user-declared listener names.
+# Setting this to an empty string restores the legacy fallback: the first listener parsed from
+# advertisedListeners is used as the internal listener.
+internalListenerName=internal
 
 # Enable or disable the HAProxy protocol.
 # If true, the real IP addresses of consumers and producers can be obtained when getting topic statistics data.

conf/standalone.conf

@@ -32,21 +32,60 @@ metadataStoreConfigPath=
 # The metadata store URL for the configuration data. If empty, we fall back to use metadataStoreUrl
 configurationMetadataStoreUrl=
 
+# Port for the Pulsar binary protocol of the internal listener.
+# The same port number is used both for the local socket binding (with bindAddress) and for the
+# advertised URL (with advertisedAddress), so no entry in advertisedListeners or bindAddresses
+# is required for the internal listener.
 brokerServicePort=6650
 
-# Port to use to server HTTP request
+# Port for the HTTP admin/REST endpoint of the internal listener.
+# Used both for the local socket binding and the advertised URL.
 webServicePort=8080
 
-# Hostname or IP address the service binds on, default is 0.0.0.0.
+# Local network interface IP for the internal listener's port bindings.
+# Use a specific local IP to bind to a single interface, or 0.0.0.0 to bind on all interfaces.
 bindAddress=0.0.0.0
 
-# Extra bind addresses for the service: <listener_name>:<scheme>://<host>:<port>,[...]
+# Per-listener socket bindings.
+# The internal listener's bindings are derived automatically from bindAddress plus the port
+# properties (brokerServicePort / brokerServicePortTls / webServicePort / webServicePortTls)
+# and do not need to be repeated here.
+# PIP-95 smart listener selection routes a connection to the listener whose port it arrived on.
+# For the Pulsar binary protocol (pulsar / pulsar+ssl) this is optional — clients can also pass
+# an explicit listenerName. For the HTTP/HTTPS Admin API (http / https) it is the only routing
+# mechanism, so every HTTP/HTTPS advertised listener reachable from outside the cluster needs a
+# dedicated entry here on a unique port. This lets a layer-4 TCP load balancer serve the Admin
+# API directly per listener, without an HTTP reverse proxy.
+# Format: comma-separated <listener_name>:<scheme>://<ip>:<port> entries.
+# Supported schemes: pulsar, pulsar+ssl, http, https.
+# The <ip> part selects which local network interface the port binds to: use a specific local
+# IP, or 0.0.0.0 to bind on all interfaces. A local hostname is also accepted but not
+# recommended.
+# Each ip:port may be bound by exactly one (listener, scheme) pair. An entry that exactly
+# matches the auto-derived internal-listener binding is tolerated; assigning the same ip:port
+# to a different listener or scheme fails validation.
+# Port 0 (OS-assigned ephemeral port) is supported only for the internal listener.
 bindAddresses=
 
-# Hostname or IP address the service advertises to the outside world.
-# If not set, the value of InetAddress.getLocalHost().getCanonicalHostName() is used.
+# Hostname or IP advertised to clients for the internal listener.
+# Combined with the *Port properties it forms the internal listener's advertised URLs
+# (e.g. pulsar://<advertisedAddress>:<brokerServicePort>).
+# If not set, defaults to InetAddress.getLocalHost().getCanonicalHostName().
 advertisedAddress=
 
+# Name of the listener used for cluster-internal broker-to-broker communication (lookup
+# redirects, admin forwarding to owner or leader broker). Defaults to "internal".
+# When set (the default), the internal listener is auto-configured from advertisedAddress plus
+# the *Port properties. The internal listener must advertise addresses reachable from other
+# brokers in the same cluster; avoid overriding its URLs in advertisedListeners to point at an
+# external load balancer because that would route cluster-internal traffic outside the cluster.
+# For external clients, declare a separate non-internal listener in advertisedListeners
+# instead. The default name "internal" also keeps the port-derived internal listener distinct
+# from any user-declared listener names.
+# Setting this to an empty string restores the legacy fallback: the first listener parsed from
+# advertisedListeners is used as the internal listener.
+internalListenerName=internal
+
 # Enable or disable the HAProxy protocol.
 # If true, the real IP addresses of consumers and producers can be obtained when getting topic statistics data.
 haProxyProtocolEnabled=false

pulsar-broker-common/build.gradle.kts

@@ -27,6 +27,7 @@ dependencies {
     implementation(libs.slog)
     implementation(libs.guava)
     implementation(libs.commons.lang3)
+    implementation(libs.netty.common)
     implementation(libs.bookkeeper.server)
     implementation(libs.opentelemetry.api)
     implementation(libs.simpleclient)