Build AWS SDK as part of FDB, and use it to acquire blob credentials. This supports IAM role-based authentication, in addition to all existing credential types.

cmake/FDBComponents.cmake

@@ -212,6 +212,17 @@ endif()
 
 set(COROUTINE_IMPL ${DEFAULT_COROUTINE_IMPL} CACHE STRING "Which coroutine implementation to use. Options are boost and libcoro")
 
+################################################################################
+# AWS SDK
+################################################################################
+
+set(BUILD_AWS_BACKUP OFF CACHE BOOL "Build AWS S3 SDK backup client")
+if (BUILD_AWS_BACKUP)
+  set(WITH_AWS_BACKUP ON)
+else()
+  set(WITH_AWS_BACKUP OFF)
+endif()
+
 ################################################################################
 
 file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/packages)
@@ -232,6 +243,7 @@ function(print_components)
   message(STATUS "Build Python sdist (make package):    ${WITH_PYTHON_BINDING}")
   message(STATUS "Configure CTest (depends on Python):  ${WITH_PYTHON}")
   message(STATUS "Build with RocksDB:                   ${WITH_ROCKSDB_EXPERIMENTAL}")
+  message(STATUS "Build with AWS SDK:                   ${WITH_AWS_BACKUP}")
   message(STATUS "=========================================")
 endfunction()
 

cmake/awssdk.cmake

@@ -0,0 +1,98 @@
+project(awssdk-download NONE)
+
+# Compile the sdk with clang and libc++, since otherwise we get libc++ vs libstdc++ link errors when compiling fdb with clang
+set(AWSSDK_COMPILER_FLAGS "")
+set(AWSSDK_LINK_FLAGS "")
+if(APPLE OR CLANG OR USE_LIBCXX)
+  set(AWSSDK_COMPILER_FLAGS -stdlib=libc++ -nostdlib++)
+  set(AWSSDK_LINK_FLAGS -stdlib=libc++ -lc++abi)
+endif()
+
+include(ExternalProject)
+ExternalProject_Add(awssdk_project
+  GIT_REPOSITORY    https://github.com/aws/aws-sdk-cpp.git
+  GIT_TAG           2af3ce543c322cb259471b3b090829464f825972 # v1.9.200
+  SOURCE_DIR        "${CMAKE_CURRENT_BINARY_DIR}/awssdk-src"
+  BINARY_DIR        "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build"
+  GIT_CONFIG        advice.detachedHead=false
+  CMAKE_ARGS        -DBUILD_SHARED_LIBS=OFF        # SDK builds shared libs by default, we want static libs
+                    -DENABLE_TESTING=OFF
+                    -DBUILD_ONLY=core              # git repo contains SDK for every AWS product, we only want the core auth libraries
+                    -DSIMPLE_INSTALL=ON
+                    -DCMAKE_INSTALL_PREFIX=install # need to specify an install prefix so it doesn't install in /usr/lib - FIXME: use absolute path
+                    -DBYO_CRYPTO=ON                # we have our own crypto libraries that conflict if we let aws sdk build and link its own
+
+                    
+                    -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+                    -DCMAKE_EXE_LINKER_FLAGS=${AWSSDK_COMPILER_FLAGS}
+                    -DCMAKE_CXX_FLAGS=${AWSSDK_LINK_FLAGS}
+  TEST_COMMAND      ""
+  BUILD_ALWAYS      TRUE
+  # the sdk build produces a ton of artifacts, with their own dependency tree, so there is a very specific dependency order they must be linked in
+  BUILD_BYPRODUCTS  "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-core.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-crt-cpp.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-s3.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-auth.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-event-stream.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-http.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-mqtt.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-io.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-checksums.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-compression.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-cal.a"
+                    "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-common.a"
+)
+
+add_library(awssdk_core STATIC IMPORTED)
+add_dependencies(awssdk_core awssdk_project)
+set_target_properties(awssdk_core PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-core.a")
+
+add_library(awssdk_crt STATIC IMPORTED)
+add_dependencies(awssdk_crt awssdk_project)
+set_target_properties(awssdk_crt PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-crt-cpp.a")
+
+# TODO: can we remove c_s3? It seems to be a dependency of libaws-crt
+add_library(awssdk_c_s3 STATIC IMPORTED)
+add_dependencies(awssdk_c_s3 awssdk_project)
+set_target_properties(awssdk_c_s3 PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-s3.a")
+
+add_library(awssdk_c_auth STATIC IMPORTED)
+add_dependencies(awssdk_c_auth awssdk_project)
+set_target_properties(awssdk_c_auth PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-auth.a")
+
+add_library(awssdk_c_eventstream STATIC IMPORTED)
+add_dependencies(awssdk_c_eventstream awssdk_project)
+set_target_properties(awssdk_c_eventstream PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-event-stream.a")
+
+add_library(awssdk_c_http STATIC IMPORTED)
+add_dependencies(awssdk_c_http awssdk_project)
+set_target_properties(awssdk_c_http PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-http.a")
+
+add_library(awssdk_c_mqtt STATIC IMPORTED)
+add_dependencies(awssdk_c_mqtt awssdk_project)
+set_target_properties(awssdk_c_mqtt PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-mqtt.a")
+
+add_library(awssdk_c_io STATIC IMPORTED)
+add_dependencies(awssdk_c_io awssdk_project)
+set_target_properties(awssdk_c_io PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-io.a")
+
+add_library(awssdk_checksums STATIC IMPORTED)
+add_dependencies(awssdk_checksums awssdk_project)
+set_target_properties(awssdk_checksums PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-checksums.a")
+
+add_library(awssdk_c_compression STATIC IMPORTED)
+add_dependencies(awssdk_c_compression awssdk_project)
+set_target_properties(awssdk_c_compression PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-compression.a")
+
+add_library(awssdk_c_cal STATIC IMPORTED)
+add_dependencies(awssdk_c_cal awssdk_project)
+set_target_properties(awssdk_c_cal PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-cal.a")
+
+add_library(awssdk_c_common STATIC IMPORTED)
+add_dependencies(awssdk_c_common awssdk_project)
+set_target_properties(awssdk_c_common PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-common.a")
+
+# link them all together in one interface target
+add_library(awssdk_target INTERFACE)
+target_include_directories(awssdk_target SYSTEM INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/include)
+target_link_libraries(awssdk_target INTERFACE awssdk_core awssdk_crt awssdk_c_s3 awssdk_c_auth awssdk_c_eventstream awssdk_c_http awssdk_c_mqtt awssdk_c_io awssdk_checksums awssdk_c_compression awssdk_c_cal awssdk_c_common curl)

fdbclient/CMakeLists.txt

@@ -205,6 +205,17 @@ if(BUILD_AZURE_BACKUP)
   )
 endif()
 
+
+if(WITH_AWS_BACKUP)
+  add_compile_definitions(BUILD_AWS_BACKUP)
+
+  set(FDBCLIENT_SRCS
+    ${FDBCLIENT_SRCS}
+    FDBAWSCredentialsProvider.h)
+
+  include(awssdk)
+endif()
+
 add_flow_target(STATIC_LIBRARY NAME fdbclient SRCS ${FDBCLIENT_SRCS} ADDL_SRCS ${options_srcs})
 add_dependencies(fdbclient fdboptions)
 target_link_libraries(fdbclient PUBLIC fdbrpc msgpack)
@@ -224,3 +235,8 @@ if(BUILD_AZURE_BACKUP)
   target_link_libraries(fdbclient PRIVATE curl uuid azure-storage-lite)
   target_link_libraries(fdbclient_sampling PRIVATE curl uuid azure-storage-lite)
 endif()
+
+if(BUILD_AWS_BACKUP)
+  target_link_libraries(fdbclient PUBLIC awssdk_target)
+  target_link_libraries(fdbclient_sampling PUBLIC awssdk_target)
+endif()

fdbclient/FDBAWSCredentialsProvider.h

@@ -0,0 +1,47 @@
+/*
+ * FDBAWSCredentialsProvider.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#if (!defined FDB_AWS_CREDENTIALS_PROVIDER_H) && (defined BUILD_AWS_BACKUP)
+#define FDB_AWS_CREDENTIALS_PROVIDER_H
+#pragma once
+
+#include "aws/core/Aws.h"
+#include "aws/core/auth/AWSCredentialsProviderChain.h"
+
+// Singleton
+namespace FDBAWSCredentialsProvider {
+bool doneInit = false;
+
+// You're supposed to call AWS::ShutdownAPI(options); once done
+// But we want this to live for the lifetime of the process, so we don't do that
+static Aws::Auth::AWSCredentials getAwsCredentials() {
+	if (!doneInit) {
+		doneInit = true;
+		Aws::SDKOptions options;
+		Aws::InitAPI(options);
+		TraceEvent("AWSSDKInitSuccessful");
+	}
+	Aws::Auth::DefaultAWSCredentialsProviderChain credProvider;
+	Aws::Auth::AWSCredentials creds = credProvider.GetAWSCredentials();
+	return creds;
+}
+} // namespace FDBAWSCredentialsProvider
+
+#endif

fdbclient/S3BlobStore.actor.cpp

@@ -34,6 +34,8 @@
 #include "fdbrpc/IAsyncFile.h"
 #include "flow/UnitTest.h"
 #include "fdbclient/rapidxml/rapidxml.hpp"
+#include "fdbclient/FDBAWSCredentialsProvider.h"
+
 #include "flow/actorcompiler.h" // has to be last include
 
 using namespace rapidxml;
@@ -82,6 +84,7 @@ S3BlobStoreEndpoint::BlobKnobs::BlobKnobs() {
 	read_cache_blocks_per_file = CLIENT_KNOBS->BLOBSTORE_READ_CACHE_BLOCKS_PER_FILE;
 	max_send_bytes_per_second = CLIENT_KNOBS->BLOBSTORE_MAX_SEND_BYTES_PER_SECOND;
 	max_recv_bytes_per_second = CLIENT_KNOBS->BLOBSTORE_MAX_RECV_BYTES_PER_SECOND;
+	sdk_auth = false;
 }
 
 bool S3BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) {
@@ -118,6 +121,7 @@ bool S3BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) {
 	TRY_PARAM(read_cache_blocks_per_file, rcb);
 	TRY_PARAM(max_send_bytes_per_second, sbps);
 	TRY_PARAM(max_recv_bytes_per_second, rbps);
+	TRY_PARAM(sdk_auth, sa);
 #undef TRY_PARAM
 	return false;
 }
@@ -506,7 +510,38 @@ ACTOR Future<Optional<json_spirit::mObject>> tryReadJSONFile(std::string path) {
 	return Optional<json_spirit::mObject>();
 }
 
+// If the credentials expire, the connection will eventually fail and be discarded from the pool, and then a new
+// connection will be constructed, which will call this again to get updated credentials
+static S3BlobStoreEndpoint::Credentials getSecretSdk() {
+#ifdef BUILD_AWS_BACKUP
+	double elapsed = -timer_monotonic();
+	Aws::Auth::AWSCredentials awsCreds = FDBAWSCredentialsProvider::getAwsCredentials();
+	elapsed += timer_monotonic();
+
+	if (awsCreds.IsEmpty()) {
+		TraceEvent(SevWarn, "S3BlobStoreAWSCredsEmpty");
+		throw backup_auth_missing();
+	}
+
+	S3BlobStoreEndpoint::Credentials fdbCreds;
+	fdbCreds.key = awsCreds.GetAWSAccessKeyId();
+	fdbCreds.secret = awsCreds.GetAWSSecretKey();
+	fdbCreds.securityToken = awsCreds.GetSessionToken();
+
+	TraceEvent("S3BlobStoreGotSdkCredentials").suppressFor(60).detail("Duration", elapsed);
+
+	return fdbCreds;
+#else
+	TraceEvent(SevError, "S3BlobStoreNoSDK");
+	throw backup_auth_missing();
+#endif
+}
+
 ACTOR Future<Void> updateSecret_impl(Reference<S3BlobStoreEndpoint> b) {
+	if (b->knobs.sdk_auth) {
+		b->credentials = getSecretSdk();
+		return Void();
+	}
 	std::vector<std::string>* pFiles = (std::vector<std::string>*)g_network->global(INetwork::enBlobCredentialFiles);
 	if (pFiles == nullptr)
 		return Void();
@@ -601,7 +636,7 @@ ACTOR Future<S3BlobStoreEndpoint::ReusableConnection> connect_impl(Reference<S3B
 	    .detail("RemoteEndpoint", conn->getPeerAddress())
 	    .detail("ExpiresIn", b->knobs.max_connection_life);
 
-	if (b->lookupKey || b->lookupSecret)
+	if (b->lookupKey || b->lookupSecret || b->knobs.sdk_auth)
 		wait(b->updateSecret());
 
 	return S3BlobStoreEndpoint::ReusableConnection({ conn, now() + b->knobs.max_connection_life });

fdbclient/S3BlobStore.h

@@ -59,7 +59,7 @@ public:
 		    delete_requests_per_second, multipart_max_part_size, multipart_min_part_size, concurrent_requests,
 		    concurrent_uploads, concurrent_lists, concurrent_reads_per_file, concurrent_writes_per_file,
 		    read_block_size, read_ahead_blocks, read_cache_blocks_per_file, max_send_bytes_per_second,
-		    max_recv_bytes_per_second;
+		    max_recv_bytes_per_second, sdk_auth;
 		bool set(StringRef name, int value);
 		std::string getURLParameters() const;
 		static std::vector<std::string> getKnobDescriptions() {
@@ -91,7 +91,9 @@ public:
 				"read_cache_blocks_per_file (or rcb)   Size of the read cache for a file in blocks.",
 				"max_send_bytes_per_second (or sbps)   Max send bytes per second for all requests combined.",
 				"max_recv_bytes_per_second (or rbps)   Max receive bytes per second for all requests combined (NOT YET "
-				"USED)."
+				"USED).",
+				"sdk_auth (or sa)                      Use AWS SDK to resolve credentials. Only valid if "
+				"BUILD_AWS_BACKUP is enabled."
 			};
 		}
 	};