Data race on OpenSSL internals in ConnectionPoolWithFailover

https://s3.amazonaws.com/clickhouse-test-reports/62738/39deb3ee7f74d6249f1e57238d14489fde6cb019/stress_test__tsan_.html

```
WARNING: ThreadSanitizer: data race (pid=75215)
  Write of size 4 at 0x7b08002e1670 by thread T614 (mutexes: write M0, write M1, write M2, write M3):
    #0 OPENSSL_sk_insert build_docker/./contrib/openssl/crypto/stack/stack.c:280:16 (clickhouse+0x203a4ea4) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #1 OPENSSL_sk_push build_docker/./contrib/openssl/crypto/stack/stack.c:401:12 (clickhouse+0x203a4ea4)
    #2 x509_store_add build_docker/./contrib/openssl/crypto/x509/x509_lu.c:419:17 (clickhouse+0x203cc112) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #3 X509_STORE_add_cert build_docker/./contrib/openssl/crypto/x509/x509_lu.c:432:10 (clickhouse+0x203cbf62) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #4 X509_load_cert_file_ex build_docker/./contrib/openssl/crypto/x509/by_file.c:127:18 (clickhouse+0x203aeba6) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #5 get_cert_by_subject_ex build_docker/./contrib/openssl/crypto/x509/by_dir.c:333:22 (clickhouse+0x203adf0c) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #6 X509_LOOKUP_by_subject_ex build_docker/./contrib/openssl/crypto/x509/x509_lu.c:105:16 (clickhouse+0x203cbdac) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #7 ossl_x509_store_ctx_get_by_subject build_docker/./contrib/openssl/crypto/x509/x509_lu.c:360:17 (clickhouse+0x203cbdac)
    #8 X509_STORE_CTX_get1_issuer build_docker/./contrib/openssl/crypto/x509/x509_lu.c:782:10 (clickhouse+0x203ccd8b) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #9 get1_trusted_issuer build_docker/./contrib/openssl/crypto/x509/x509_vfy.c:3194:10 (clickhouse+0x203d2b69) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #10 build_chain build_docker/./contrib/openssl/crypto/x509/x509_vfy.c:3324:40 (clickhouse+0x203d2b69)
    #11 verify_chain build_docker/./contrib/openssl/crypto/x509/x509_vfy.c:240:15 (clickhouse+0x203d34e7) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #12 x509_verify_x509 build_docker/./contrib/openssl/crypto/x509/x509_vfy.c:358 (clickhouse+0x203cf698) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #13 X509_verify_cert build_docker/./contrib/openssl/crypto/x509/x509_vfy.c:293:56 (clickhouse+0x203cf8d5) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #14 ssl_verify_internal build_docker/./contrib/openssl/ssl/ssl_cert.c:496:13 (clickhouse+0x20191964) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #15 ssl_verify_cert_chain build_docker/./contrib/openssl/ssl/ssl_cert.c:543:12 (clickhouse+0x20191ac2) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #16 tls_post_process_server_certificate build_docker/./contrib/openssl/ssl/statem/statem_clnt.c:2072:9 (clickhouse+0x2021ed18) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #17 ossl_statem_client_post_process_message build_docker/./contrib/openssl/ssl/statem/statem_clnt.c:1159:16 (clickhouse+0x2021e9ae) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #18 read_state_machine build_docker/./contrib/openssl/ssl/statem/statem.c:712:35 (clickhouse+0x2021602d) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #19 state_machine build_docker/./contrib/openssl/ssl/statem/statem.c:478:21 (clickhouse+0x2021602d)
    #20 ossl_statem_connect build_docker/./contrib/openssl/ssl/statem/statem.c:297:12 (clickhouse+0x2021548e) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #21 SSL_do_handshake build_docker/./contrib/openssl/ssl/ssl_lib.c:4746:19 (clickhouse+0x2019ce41) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #22 SSL_connect build_docker/./contrib/openssl/ssl/ssl_lib.c:2208:12 (clickhouse+0x2019cf53) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #23 Poco::Net::SecureSocketImpl::connectSSL(bool) build_docker/./base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp:206:11 (clickhouse+0x1d170c27) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #24 Poco::Net::SecureSocketImpl::connect(Poco::Net::SocketAddress const&, Poco::Timespan const&, bool) build_docker/./base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp:149:2 (clickhouse+0x1d1710ab) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #25 Poco::Net::SecureStreamSocketImpl::connect(Poco::Net::SocketAddress const&, Poco::Timespan const&) build_docker/./base/poco/NetSSL_OpenSSL/src/SecureStreamSocketImpl.cpp:87:8 (clickhouse+0x1d176e66) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #26 Poco::Net::StreamSocket::connect(Poco::Net::SocketAddress const&, Poco::Timespan const&) build_docker/./base/poco/Net/src/StreamSocket.cpp:84:10 (clickhouse+0x1d1d05bf) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #27 DB::Connection::connect(DB::ConnectionTimeouts const&) build_docker/./src/Client/Connection.cpp:152:29 (clickhouse+0x194c4f53) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #28 DB::Connection::getServerRevision(DB::ConnectionTimeouts const&) build_docker/./src/Client/Connection.cpp:515:9 (clickhouse+0x194c9455) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #29 DB::ConnectionEstablisher::run(PoolWithFailoverBase<DB::IConnectionPool>::TryResult&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) build_docker/./src/Client/ConnectionEstablisher.cpp:44:45 (clickhouse+0x194dbc2a) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #30 DB::ConnectionPoolWithFailover::tryGetEntry(std::__1::shared_ptr<DB::IConnectionPool> const&, DB::ConnectionTimeouts const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&, DB::Settings const&, DB::QualifiedTableName const*, std::__1::function<void (int, Poco::Timespan, DB::AsyncEventTimeoutType, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, unsigned int)>) build_docker/./src/Client/ConnectionPoolWithFailover.cpp:265:28 (clickhouse+0x194e95ba) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #31 DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0::operator()(std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) const build_docker/./src/Client/ConnectionPoolWithFailover.cpp:168:14 (clickhouse+0x194ea3c2) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #32 decltype(std::declval<DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0&>()(std::declval<std::__1::shared_ptr<DB::IConnectionPool> const&>(), std::declval<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&>())) std::__1::__invoke[abi:v15000]<DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0&, std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&>(DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0&, std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0x194ea3c2)
    #33 PoolWithFailoverBase<DB::IConnectionPool>::TryResult std::__1::__invoke_void_return_wrapper<PoolWithFailoverBase<DB::IConnectionPool>::TryResult, false>::__call<DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0&, std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&>(DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0&, std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:470:16 (clickhouse+0x194ea3c2)
    #34 std::__1::__function::__default_alloc_func<DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0, PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)>::operator()[abi:v15000](std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:235:12 (clickhouse+0x194ea3c2)
    #35 PoolWithFailoverBase<DB::IConnectionPool>::TryResult std::__1::__function::__policy_invoker<PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)>::__call_impl<std::__1::__function::__default_alloc_func<DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0, PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)>>(std::__1::__function::__policy_storage const*, std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:716:16 (clickhouse+0x194ea3c2)
    #36 std::__1::__function::__policy_func<PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)>::operator()[abi:v15000](std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:848:16 (clickhouse+0x194eb5e7) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #37 std::__1::function<PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)>::operator()(std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:1187:12 (clickhouse+0x194eb5e7)
    #38 PoolWithFailoverBase<DB::IConnectionPool>::getMany(unsigned long, unsigned long, unsigned long, unsigned long, bool, bool, std::__1::function<PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)> const&, std::__1::function<Priority (unsigned long)> const&) build_docker/./src/Common/PoolWithFailoverBase.h:266:22 (clickhouse+0x194eb5e7)
    #39 DB::ConnectionPoolWithFailover::getManyImpl(DB::Settings const&, DB::PoolMode, std::__1::function<PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)> const&, std::__1::optional<bool>, std::__1::function<Priority (unsigned long)>, bool) build_docker/./src/Client/ConnectionPoolWithFailover.cpp:228:18 (clickhouse+0x194e8c51) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #40 DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&) build_docker/./src/Client/ConnectionPoolWithFailover.cpp:170:12 (clickhouse+0x194e92a1) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #41 DB::DistributedAsyncInsertDirectoryQueue::processFile(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&, DB::SettingsChanges const&) build_docker/./src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp:415:29 (clickhouse+0x18cf5be3) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #42 DB::DistributedAsyncInsertDirectoryQueue::processFiles(DB::SettingsChanges const&) build_docker/./src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp:377:13 (clickhouse+0x18cef7d9) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #43 DB::DistributedAsyncInsertDirectoryQueue::run() build_docker/./src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp:200:17 (clickhouse+0x18cefd53) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #44 DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0::operator()() const build_docker/./src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp:135:71 (clickhouse+0x18cf98c1) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #45 decltype(std::declval<DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0&>()()) std::__1::__invoke[abi:v15000]<DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0&>(DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0x18cf98c1)
    #46 void std::__1::__invoke_void_return_wrapper<void, true>::__call<DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0&>(DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:479:9 (clickhouse+0x18cf98c1)
    #47 std::__1::__function::__default_alloc_func<DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0, void ()>::operator()[abi:v15000]() build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:235:12 (clickhouse+0x18cf98c1)
    #48 void std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0, void ()>>(std::__1::__function::__policy_storage const*) build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:716:16 (clickhouse+0x18cf98c1)
    #49 std::__1::__function::__policy_func<void ()>::operator()[abi:v15000]() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:848:16 (clickhouse+0x15e3aad5) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #50 std::__1::function<void ()>::operator()() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:1187:12 (clickhouse+0x15e3aad5)
    #51 DB::BackgroundSchedulePoolTaskInfo::execute() build_docker/./src/Core/BackgroundSchedulePool.cpp:106:9 (clickhouse+0x15e3aad5)
    #52 DB::BackgroundSchedulePool::threadFunction() build_docker/./src/Core/BackgroundSchedulePool.cpp:304:19 (clickhouse+0x15e3ed4d) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #53 DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0::operator()() const build_docker/./src/Core/BackgroundSchedulePool.cpp:170:79 (clickhouse+0x15e3f68e) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #54 decltype(std::declval<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&>()()) std::__1::__invoke[abi:v15000]<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0x15e3f68e)
    #55 decltype(auto) std::__1::__apply_tuple_impl[abi:v15000]<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&, std::__1::tuple<>&>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&, std::__1::tuple<>&, std::__1::__tuple_indices<>) build_docker/./contrib/llvm-project/libcxx/include/tuple:1789:1 (clickhouse+0x15e3f68e)
    #56 decltype(auto) std::__1::apply[abi:v15000]<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&, std::__1::tuple<>&>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&, std::__1::tuple<>&) build_docker/./contrib/llvm-project/libcxx/include/tuple:1798:1 (clickhouse+0x15e3f68e)
    #57 ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'()::operator()() build_docker/./src/Common/ThreadPool.h:246:13 (clickhouse+0x15e3f68e)
    #58 decltype(std::declval<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>()()) std::__1::__invoke[abi:v15000]<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'()&>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0x15e3f68e)
    #59 void std::__1::__invoke_void_return_wrapper<void, true>::__call<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'()&>(ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'()&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:479:9 (clickhouse+0x15e3f68e)
    #60 std::__1::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>::operator()[abi:v15000]() build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:235:12 (clickhouse+0x15e3f68e)
    #61 void std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__1::__function::__policy_storage const*) build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:716:16 (clickhouse+0x15e3f68e)
    #62 std::__1::__function::__policy_func<void ()>::operator()[abi:v15000]() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:848:16 (clickhouse+0xf0445ae) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #63 std::__1::function<void ()>::operator()() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:1187:12 (clickhouse+0xf0445ae)
    #64 ThreadPoolImpl<std::__1::thread>::worker(std::__1::__list_iterator<std::__1::thread, void*>) build_docker/./src/Common/ThreadPool.cpp:458:13 (clickhouse+0xf0445ae)
    #65 void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()::operator()() const build_docker/./src/Common/ThreadPool.cpp:220:73 (clickhouse+0xf04ad91) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #66 decltype(std::declval<void>()()) std::__1::__invoke[abi:v15000]<void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0xf04ad91)
    #67 void std::__1::__thread_execute[abi:v15000]<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()>(std::__1::tuple<void, void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()>&, std::__1::__tuple_indices<>) build_docker/./contrib/llvm-project/libcxx/include/thread:284:5 (clickhouse+0xf04ad91)
    #68 void* std::__1::__thread_proxy[abi:v15000]<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()>>(void*) build_docker/./contrib/llvm-project/libcxx/include/thread:295:5 (clickhouse+0xf04ad91)

  Previous read of size 4 at 0x7b08002e1670 by thread T616 (mutexes: write M4, write M5, write M6):
    #0 OPENSSL_sk_is_sorted build_docker/./contrib/openssl/crypto/stack/stack.c:490:33 (clickhouse+0x203a53bf) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #1 get_cert_by_subject_ex build_docker/./contrib/openssl/crypto/x509/by_dir.c:423:10 (clickhouse+0x203ae44f) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #2 X509_LOOKUP_by_subject_ex build_docker/./contrib/openssl/crypto/x509/x509_lu.c:105:16 (clickhouse+0x203cbdac) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #3 ossl_x509_store_ctx_get_by_subject build_docker/./contrib/openssl/crypto/x509/x509_lu.c:360:17 (clickhouse+0x203cbdac)
    #4 X509_STORE_CTX_get1_issuer build_docker/./contrib/openssl/crypto/x509/x509_lu.c:782:10 (clickhouse+0x203ccd8b) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #5 get1_trusted_issuer build_docker/./contrib/openssl/crypto/x509/x509_vfy.c:3194:10 (clickhouse+0x203d2b69) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #6 build_chain build_docker/./contrib/openssl/crypto/x509/x509_vfy.c:3324:40 (clickhouse+0x203d2b69)
    #7 verify_chain build_docker/./contrib/openssl/crypto/x509/x509_vfy.c:240:15 (clickhouse+0x203d34e7) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #8 x509_verify_x509 build_docker/./contrib/openssl/crypto/x509/x509_vfy.c:358 (clickhouse+0x203cf698) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #9 X509_verify_cert build_docker/./contrib/openssl/crypto/x509/x509_vfy.c:293:56 (clickhouse+0x203cf8d5) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #10 ssl_verify_internal build_docker/./contrib/openssl/ssl/ssl_cert.c:496:13 (clickhouse+0x20191964) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #11 ssl_verify_cert_chain build_docker/./contrib/openssl/ssl/ssl_cert.c:543:12 (clickhouse+0x20191ac2) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #12 tls_post_process_server_certificate build_docker/./contrib/openssl/ssl/statem/statem_clnt.c:2072:9 (clickhouse+0x2021ed18) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #13 ossl_statem_client_post_process_message build_docker/./contrib/openssl/ssl/statem/statem_clnt.c:1159:16 (clickhouse+0x2021e9ae) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #14 read_state_machine build_docker/./contrib/openssl/ssl/statem/statem.c:712:35 (clickhouse+0x2021602d) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #15 state_machine build_docker/./contrib/openssl/ssl/statem/statem.c:478:21 (clickhouse+0x2021602d)
    #16 ossl_statem_connect build_docker/./contrib/openssl/ssl/statem/statem.c:297:12 (clickhouse+0x2021548e) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #17 SSL_do_handshake build_docker/./contrib/openssl/ssl/ssl_lib.c:4746:19 (clickhouse+0x2019ce41) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #18 SSL_connect build_docker/./contrib/openssl/ssl/ssl_lib.c:2208:12 (clickhouse+0x2019cf53) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #19 Poco::Net::SecureSocketImpl::connectSSL(bool) build_docker/./base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp:206:11 (clickhouse+0x1d170c27) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #20 Poco::Net::SecureSocketImpl::connect(Poco::Net::SocketAddress const&, Poco::Timespan const&, bool) build_docker/./base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp:149:2 (clickhouse+0x1d1710ab) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #21 Poco::Net::SecureStreamSocketImpl::connect(Poco::Net::SocketAddress const&, Poco::Timespan const&) build_docker/./base/poco/NetSSL_OpenSSL/src/SecureStreamSocketImpl.cpp:87:8 (clickhouse+0x1d176e66) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #22 Poco::Net::StreamSocket::connect(Poco::Net::SocketAddress const&, Poco::Timespan const&) build_docker/./base/poco/Net/src/StreamSocket.cpp:84:10 (clickhouse+0x1d1d05bf) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #23 DB::Connection::connect(DB::ConnectionTimeouts const&) build_docker/./src/Client/Connection.cpp:152:29 (clickhouse+0x194c4f53) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #24 DB::Connection::getServerRevision(DB::ConnectionTimeouts const&) build_docker/./src/Client/Connection.cpp:515:9 (clickhouse+0x194c9455) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #25 DB::ConnectionEstablisher::run(PoolWithFailoverBase<DB::IConnectionPool>::TryResult&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) build_docker/./src/Client/ConnectionEstablisher.cpp:44:45 (clickhouse+0x194dbc2a) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #26 DB::ConnectionPoolWithFailover::tryGetEntry(std::__1::shared_ptr<DB::IConnectionPool> const&, DB::ConnectionTimeouts const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&, DB::Settings const&, DB::QualifiedTableName const*, std::__1::function<void (int, Poco::Timespan, DB::AsyncEventTimeoutType, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, unsigned int)>) build_docker/./src/Client/ConnectionPoolWithFailover.cpp:265:28 (clickhouse+0x194e95ba) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #27 DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0::operator()(std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) const build_docker/./src/Client/ConnectionPoolWithFailover.cpp:168:14 (clickhouse+0x194ea3c2) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #28 decltype(std::declval<DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0&>()(std::declval<std::__1::shared_ptr<DB::IConnectionPool> const&>(), std::declval<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&>())) std::__1::__invoke[abi:v15000]<DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0&, std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&>(DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0&, std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0x194ea3c2)
    #29 PoolWithFailoverBase<DB::IConnectionPool>::TryResult std::__1::__invoke_void_return_wrapper<PoolWithFailoverBase<DB::IConnectionPool>::TryResult, false>::__call<DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0&, std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&>(DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0&, std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:470:16 (clickhouse+0x194ea3c2)
    #30 std::__1::__function::__default_alloc_func<DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0, PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)>::operator()[abi:v15000](std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:235:12 (clickhouse+0x194ea3c2)
    #31 PoolWithFailoverBase<DB::IConnectionPool>::TryResult std::__1::__function::__policy_invoker<PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)>::__call_impl<std::__1::__function::__default_alloc_func<DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&)::$_0, PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)>>(std::__1::__function::__policy_storage const*, std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:716:16 (clickhouse+0x194ea3c2)
    #32 std::__1::__function::__policy_func<PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)>::operator()[abi:v15000](std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:848:16 (clickhouse+0x194eb5e7) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #33 std::__1::function<PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)>::operator()(std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&) const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:1187:12 (clickhouse+0x194eb5e7)
    #34 PoolWithFailoverBase<DB::IConnectionPool>::getMany(unsigned long, unsigned long, unsigned long, unsigned long, bool, bool, std::__1::function<PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)> const&, std::__1::function<Priority (unsigned long)> const&) build_docker/./src/Common/PoolWithFailoverBase.h:266:22 (clickhouse+0x194eb5e7)
    #35 DB::ConnectionPoolWithFailover::getManyImpl(DB::Settings const&, DB::PoolMode, std::__1::function<PoolWithFailoverBase<DB::IConnectionPool>::TryResult (std::__1::shared_ptr<DB::IConnectionPool> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&)> const&, std::__1::optional<bool>, std::__1::function<Priority (unsigned long)>, bool) build_docker/./src/Client/ConnectionPoolWithFailover.cpp:228:18 (clickhouse+0x194e8c51) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #36 DB::ConnectionPoolWithFailover::getManyCheckedForInsert(DB::ConnectionTimeouts const&, DB::Settings const&, DB::PoolMode, DB::QualifiedTableName const&) build_docker/./src/Client/ConnectionPoolWithFailover.cpp:170:12 (clickhouse+0x194e92a1) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #37 DB::DistributedAsyncInsertDirectoryQueue::processFile(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>&, DB::SettingsChanges const&) build_docker/./src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp:415:29 (clickhouse+0x18cf5be3) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #38 DB::DistributedAsyncInsertDirectoryQueue::processFiles(DB::SettingsChanges const&) build_docker/./src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp:377:13 (clickhouse+0x18cef7d9) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #39 DB::DistributedAsyncInsertDirectoryQueue::run() build_docker/./src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp:200:17 (clickhouse+0x18cefd53) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #40 DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0::operator()() const build_docker/./src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp:135:71 (clickhouse+0x18cf98c1) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #41 decltype(std::declval<DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0&>()()) std::__1::__invoke[abi:v15000]<DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0&>(DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0x18cf98c1)
    #42 void std::__1::__invoke_void_return_wrapper<void, true>::__call<DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0&>(DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:479:9 (clickhouse+0x18cf98c1)
    #43 std::__1::__function::__default_alloc_func<DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0, void ()>::operator()[abi:v15000]() build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:235:12 (clickhouse+0x18cf98c1)
    #44 void std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<DB::DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue(DB::StorageDistributed&, std::__1::shared_ptr<DB::IDisk> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::ConnectionPoolWithFailover>, DB::ActionBlocker&, DB::BackgroundSchedulePool&)::$_0, void ()>>(std::__1::__function::__policy_storage const*) build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:716:16 (clickhouse+0x18cf98c1)
    #45 std::__1::__function::__policy_func<void ()>::operator()[abi:v15000]() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:848:16 (clickhouse+0x15e3aad5) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #46 std::__1::function<void ()>::operator()() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:1187:12 (clickhouse+0x15e3aad5)
    #47 DB::BackgroundSchedulePoolTaskInfo::execute() build_docker/./src/Core/BackgroundSchedulePool.cpp:106:9 (clickhouse+0x15e3aad5)
    #48 DB::BackgroundSchedulePool::threadFunction() build_docker/./src/Core/BackgroundSchedulePool.cpp:304:19 (clickhouse+0x15e3ed4d) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #49 DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0::operator()() const build_docker/./src/Core/BackgroundSchedulePool.cpp:170:79 (clickhouse+0x15e3f68e) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #50 decltype(std::declval<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&>()()) std::__1::__invoke[abi:v15000]<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0x15e3f68e)
    #51 decltype(auto) std::__1::__apply_tuple_impl[abi:v15000]<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&, std::__1::tuple<>&>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&, std::__1::tuple<>&, std::__1::__tuple_indices<>) build_docker/./contrib/llvm-project/libcxx/include/tuple:1789:1 (clickhouse+0x15e3f68e)
    #52 decltype(auto) std::__1::apply[abi:v15000]<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&, std::__1::tuple<>&>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&, std::__1::tuple<>&) build_docker/./contrib/llvm-project/libcxx/include/tuple:1798:1 (clickhouse+0x15e3f68e)
    #53 ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'()::operator()() build_docker/./src/Common/ThreadPool.h:246:13 (clickhouse+0x15e3f68e)
    #54 decltype(std::declval<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>()()) std::__1::__invoke[abi:v15000]<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'()&>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0x15e3f68e)
    #55 void std::__1::__invoke_void_return_wrapper<void, true>::__call<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'()&>(ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'()&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:479:9 (clickhouse+0x15e3f68e)
    #56 std::__1::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>::operator()[abi:v15000]() build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:235:12 (clickhouse+0x15e3f68e)
    #57 void std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__1::__function::__policy_storage const*) build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:716:16 (clickhouse+0x15e3f68e)
    #58 std::__1::__function::__policy_func<void ()>::operator()[abi:v15000]() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:848:16 (clickhouse+0xf0445ae) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #59 std::__1::function<void ()>::operator()() const build_docker/./contrib/llvm-project/libcxx/include/__functional/function.h:1187:12 (clickhouse+0xf0445ae)
    #60 ThreadPoolImpl<std::__1::thread>::worker(std::__1::__list_iterator<std::__1::thread, void*>) build_docker/./src/Common/ThreadPool.cpp:458:13 (clickhouse+0xf0445ae)
    #61 void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()::operator()() const build_docker/./src/Common/ThreadPool.cpp:220:73 (clickhouse+0xf04ad91) (BuildId: a943b6e495ff2fe12e8b59d59bc8e6efcdb42c34)
    #62 decltype(std::declval<void>()()) std::__1::__invoke[abi:v15000]<void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&) build_docker/./contrib/llvm-project/libcxx/include/__functional/invoke.h:394:23 (clickhouse+0xf04ad91)
    #63 void std::__1::__thread_execute[abi:v15000]<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()>(std::__1::tuple<void, void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()>&, std::__1::__tuple_indices<>) build_docker/./contrib/llvm-project/libcxx/include/thread:284:5 (clickhouse+0xf04ad91)
    #64 void* std::__1::__thread_proxy[abi:v15000]<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, void ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::'lambda0'()>>(void*) build_docker/./contrib/llvm-project/libcxx/include/thread:295:5 (clickhouse+0xf04ad91)
```

I don't think it's specific to `ConnectionPoolWithFailover`, looks like OpenSSL has some cache for certificates, and it's not thread-safe

cc: @rschu1ze 

Navigation Menu

Uh oh!

Data race on OpenSSL internals in ConnectionPoolWithFailover #63049

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Uh oh!

Data race on OpenSSL internals in ConnectionPoolWithFailover #63049

Description

Activity

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions