Add UseDetectedLocalDC balancing policy

include/ydb-cpp-sdk/client/types/ydb.h

@@ -54,6 +54,10 @@ class TBalancingPolicy {
     //! location is a name of datacenter (VLA, MAN), if location is nullopt local datacenter is used
     static TBalancingPolicy UsePreferableLocation(const std::optional<std::string>& location = {});
 
+    //! Use detected local DC
+    //! prefer datacenter with fastest tcp ping
+    static TBalancingPolicy UseDetectedLocalDC();
+
     //! Use all available cluster nodes regardless datacenter locality
     static TBalancingPolicy UseAllNodes();
 

library/cpp/CMakeLists.txt

@@ -17,6 +17,7 @@ add_subdirectory(diff)
 add_subdirectory(digest/lower_case)
 add_subdirectory(digest/md5)
 add_subdirectory(digest/murmur)
+add_subdirectory(dns)
 add_subdirectory(getopt)
 add_subdirectory(http/fetch)
 add_subdirectory(http/io)
@@ -33,6 +34,7 @@ add_subdirectory(monlib/encode)
 add_subdirectory(monlib/exception)
 add_subdirectory(monlib/metrics)
 add_subdirectory(monlib/service)
+add_subdirectory(neh/asio)
 add_subdirectory(openssl/holders)
 add_subdirectory(openssl/init)
 add_subdirectory(openssl/io)

library/cpp/dns/CMakeLists.txt

@@ -0,0 +1,19 @@
+if (YDB_SDK_TESTS)
+  add_subdirectory(ut)
+endif()
+
+_ydb_sdk_add_library(dns)
+
+target_sources(dns
+  PRIVATE
+    cache.cpp
+    thread.cpp
+    magic.cpp
+)
+
+target_link_libraries(dns
+  PUBLIC
+    yutil
+)
+
+_ydb_sdk_install_targets(TARGETS dns)

library/cpp/dns/README.md

@@ -0,0 +1,9 @@
+Overview
+===
+Библиотека кеширующего resolving-а - изначально писалась для имплементации neh http протокола, использующей корутины.
+Для предотвращения пробоя короткого стека корутин есть метод, предусматривающий вынос в отдельный тред собственно вызов функции резолвинга.
+Для предотвращения обращения к DNS серверам (использования вместо этого заранее заданных ip-адресов),
+предусмотрена ручка добавления alias-ов hosname -> ip-address (требование от метапоискового движка).
+
+Из-за того, что библиотека разрабатывалась под задачу максимально быстрого резолвинга добавлены слои кеширования результатов
+resoving-а, - возможности сбросить кеш для того, чтобы получить более свежие адреса для указанного host-а _нет_.

library/cpp/dns/cache.cpp

@@ -0,0 +1,198 @@
+#include "cache.h"
+
+#include "thread.h"
+
+#include <util/system/tls.h>
+#include <util/system/info.h>
+#include <util/system/rwlock.h>
+#include <util/thread/singleton.h>
+#include <util/generic/singleton.h>
+#include <util/generic/hash.h>
+
+using namespace NDns;
+
+namespace {
+    struct TResolveTask {
+        enum EMethod {
+            Normal,
+            Threaded
+        };
+
+        inline TResolveTask(const TResolveInfo& info, EMethod method)
+            : Info(info)
+            , Method(method)
+        {
+        }
+
+        const TResolveInfo& Info;
+        const EMethod Method;
+    };
+
+    class IDns {
+    public:
+        virtual ~IDns() = default;
+        virtual const TResolvedHost* Resolve(const TResolveTask&) = 0;
+    };
+
+    typedef TAtomicSharedPtr<TResolvedHost> TResolvedHostPtr;
+
+    struct THashResolveInfo {
+        inline size_t operator()(const TResolveInfo& ri) const {
+            return ComputeHash(ri.Host) ^ ri.Port;
+        }
+    };
+
+    struct TCompareResolveInfo {
+        inline bool operator()(const NDns::TResolveInfo& x, const NDns::TResolveInfo& y) const {
+            return x.Host == y.Host && x.Port == y.Port;
+        }
+    };
+
+    class TGlobalCachedDns: public IDns, public TNonCopyable {
+    public:
+        const TResolvedHost* Resolve(const TResolveTask& rt) override {
+            //2. search host in cache
+            {
+                TReadGuard guard(L_);
+
+                TCache::const_iterator it = C_.find(rt.Info);
+
+                if (it != C_.end()) {
+                    return it->second.Get();
+                }
+            }
+
+            TResolvedHostPtr res = ResolveA(rt);
+
+            //update cache
+            {
+                TWriteGuard guard(L_);
+
+                std::pair<TCache::iterator, bool> updateResult = C_.insert(std::make_pair(TResolveInfo(res->Host, rt.Info.Port), res));
+                TResolvedHost* rh = updateResult.first->second.Get();
+
+                if (updateResult.second) {
+                    //fresh resolved host, set cache record id for it
+                    rh->Id = C_.size() - 1;
+                }
+
+                return rh;
+            }
+        }
+
+        void AddAlias(const TString& host, const TString& alias) noexcept {
+            TWriteGuard guard(LA_);
+
+            A_[host] = alias;
+        }
+
+        static inline TGlobalCachedDns* Instance() {
+            return SingletonWithPriority<TGlobalCachedDns, 65530>();
+        }
+
+    private:
+        inline TResolvedHostPtr ResolveA(const TResolveTask& rt) {
+            TString originalHost(rt.Info.Host);
+            TString host(originalHost);
+
+            //3. replace host to alias, if exist
+            if (A_.size()) {
+                TReadGuard guard(LA_);
+                TString names[] = {"*", host};
+
+                for (const auto& name : names) {
+                    TAliases::const_iterator it = A_.find(name);
+
+                    if (it != A_.end()) {
+                        host = it->second;
+                    }
+                }
+            }
+
+            if (host.length() > 2 && host[0] == '[') {
+                TString unbracedIpV6(host.data() + 1, host.size() - 2);
+                host.swap(unbracedIpV6);
+            }
+
+            TAutoPtr<TNetworkAddress> na;
+
+            //4. getaddrinfo (direct or in separate thread)
+            if (rt.Method == TResolveTask::Normal) {
+                na.Reset(new TNetworkAddress(host, rt.Info.Port));
+            } else if (rt.Method == TResolveTask::Threaded) {
+                na = ThreadedResolve(host, rt.Info.Port);
+            } else {
+                Y_ASSERT(0);
+                throw yexception() << TStringBuf("invalid resolve method");
+            }
+
+            return new TResolvedHost(originalHost, *na);
+        }
+
+        typedef THashMap<TResolveInfo, TResolvedHostPtr, THashResolveInfo, TCompareResolveInfo> TCache;
+        TCache C_;
+        TRWMutex L_;
+        typedef THashMap<TString, TString> TAliases;
+        TAliases A_;
+        TRWMutex LA_;
+    };
+
+    class TCachedDns: public IDns {
+    public:
+        inline TCachedDns(IDns* slave)
+            : S_(slave)
+        {
+        }
+
+        const TResolvedHost* Resolve(const TResolveTask& rt) override {
+            //1. search in local thread cache
+            {
+                TCache::const_iterator it = C_.find(rt.Info);
+
+                if (it != C_.end()) {
+                    return it->second;
+                }
+            }
+
+            const TResolvedHost* res = S_->Resolve(rt);
+
+            C_[TResolveInfo(res->Host, rt.Info.Port)] = res;
+
+            return res;
+        }
+
+    private:
+        typedef THashMap<TResolveInfo, const TResolvedHost*, THashResolveInfo, TCompareResolveInfo> TCache;
+        TCache C_;
+        IDns* S_;
+    };
+
+    struct TThreadedDns: public TCachedDns {
+        inline TThreadedDns()
+            : TCachedDns(TGlobalCachedDns::Instance())
+        {
+        }
+    };
+
+    inline IDns* ThrDns() {
+        return FastTlsSingleton<TThreadedDns>();
+    }
+}
+
+namespace NDns {
+    const TResolvedHost* CachedResolve(const TResolveInfo& ri) {
+        TResolveTask rt(ri, TResolveTask::Normal);
+
+        return ThrDns()->Resolve(rt);
+    }
+
+    const TResolvedHost* CachedThrResolve(const TResolveInfo& ri) {
+        TResolveTask rt(ri, TResolveTask::Threaded);
+
+        return ThrDns()->Resolve(rt);
+    }
+
+    void AddHostAlias(const TString& host, const TString& alias) {
+        TGlobalCachedDns::Instance()->AddAlias(host, alias);
+    }
+}

library/cpp/dns/cache.h

@@ -0,0 +1,45 @@
+#pragma once
+
+#include <util/network/socket.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/string.h>
+
+namespace NDns {
+    struct TResolveInfo {
+        inline TResolveInfo(const TStringBuf& host, ui16 port)
+            : Host(host)
+            , Port(port)
+        {
+        }
+
+        TStringBuf Host;
+        ui16 Port;
+    };
+
+    struct TResolvedHost {
+        inline TResolvedHost(const TString& host, const TNetworkAddress& addr) noexcept
+            : Host(host)
+            , Addr(addr)
+            , Id(0)
+        {
+        }
+
+        TString Host; //resolved hostname (from TResolveInfo, - before aliasing)
+        TNetworkAddress Addr;
+        size_t Id; //cache record id
+    };
+
+    // Resolving order:
+    //   1. check local thread cache, return if found
+    //   2. check global cache, return if found
+    //   3. search alias for hostname, if found, continue resolving alias
+    //   4. normal resolver
+    const TResolvedHost* CachedResolve(const TResolveInfo& ri);
+
+    //like previous, but at stage 4 use separate thread for resolving (created on first usage)
+    //useful in green-threads with tiny stack
+    const TResolvedHost* CachedThrResolve(const TResolveInfo& ri);
+
+    //create alias for host, which can be used for static resolving (when alias is ip address)
+    void AddHostAlias(const TString& host, const TString& alias);
+}

library/cpp/dns/magic.cpp

@@ -0,0 +1,28 @@
+#include "magic.h"
+
+#include <util/generic/yexception.h>
+
+using namespace NDns;
+
+namespace {
+    namespace NX {
+        struct TError: public IError {
+            inline TError()
+                : E_(std::current_exception())
+            {
+            }
+
+            void Raise() override {
+                std::rethrow_exception(E_);
+            }
+
+            std::exception_ptr E_;
+        };
+    }
+}
+
+IErrorRef NDns::SaveError() {
+    using namespace NX;
+
+    return new NX::TError();
+}

library/cpp/dns/magic.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <util/generic/yexception.h>
+#include <util/generic/ptr.h>
+
+namespace NDns {
+    class IError {
+    public:
+        virtual ~IError() = default;
+
+        virtual void Raise() = 0;
+    };
+
+    typedef TAutoPtr<IError> IErrorRef;
+
+    IErrorRef SaveError();
+}