GDAL
cpl_aws.h
1 /**********************************************************************
2  * $Id$
3  *
4  * Name: cpl_aws.h
5  * Project: CPL - Common Portability Library
6  * Purpose: Amazon Web Services routines
7  * Author: Even Rouault <even.rouault at spatialys.com>
8  *
9  **********************************************************************
10  * Copyright (c) 2015, Even Rouault <even.rouault at spatialys.com>
11  *
12  * Permission is hereby granted, free of charge, to any person obtaining a
13  * copy of this software and associated documentation files (the "Software"),
14  * to deal in the Software without restriction, including without limitation
15  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16  * and/or sell copies of the Software, and to permit persons to whom the
17  * Software is furnished to do so, subject to the following conditions:
18  *
19  * The above copyright notice and this permission notice shall be included
20  * in all copies or substantial portions of the Software.
21  *
22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28  * DEALINGS IN THE SOFTWARE.
29  ****************************************************************************/
30 
31 #ifndef CPL_AWS_INCLUDED_H
32 #define CPL_AWS_INCLUDED_H
33 
34 #ifndef DOXYGEN_SKIP
35 
36 #ifdef HAVE_CURL
37 
38 #include <cstddef>
39 #include <mutex>
40 
41 #include "cpl_string.h"
42 
43 #include <curl/curl.h>
44 #include <map>
45 
46 CPLString CPLGetLowerCaseHexSHA256( const void *pabyData, size_t nBytes );
47 CPLString CPLGetLowerCaseHexSHA256( const CPLString& osStr );
48 
49 CPLString CPLGetAWS_SIGN4_Timestamp(GIntBig timestamp);
50 
51 CPLString CPLAWSURLEncode(const CPLString& osURL, bool bEncodeSlash = true);
52 
53 CPLString CPLAWSGetHeaderVal(const struct curl_slist* psExistingHeaders,
54  const char* pszKey);
55 
57 CPLGetAWS_SIGN4_Signature( const CPLString& osSecretAccessKey,
58  const CPLString& osAccessToken,
59  const CPLString& osRegion,
60  const CPLString& osRequestPayer,
61  const CPLString& osService,
62  const CPLString& osVerb,
63  const struct curl_slist* psExistingHeaders,
64  const CPLString& osHost,
65  const CPLString& osCanonicalURI,
66  const CPLString& osCanonicalQueryString,
67  const CPLString& osXAMZContentSHA256,
68  bool bAddHeaderAMZContentSHA256,
69  const CPLString& osTimestamp,
70  CPLString& osSignedHeaders );
71 
72 CPLString CPLGetAWS_SIGN4_Authorization(const CPLString& osSecretAccessKey,
73  const CPLString& osAccessKeyId,
74  const CPLString& osAccessToken,
75  const CPLString& osRegion,
76  const CPLString& osRequestPayer,
77  const CPLString& osService,
78  const CPLString& osVerb,
79  const struct curl_slist* psExistingHeaders,
80  const CPLString& osHost,
81  const CPLString& osCanonicalURI,
82  const CPLString& osCanonicalQueryString,
83  const CPLString& osXAMZContentSHA256,
84  bool bAddHeaderAMZContentSHA256,
85  const CPLString& osTimestamp);
86 
87 class IVSIS3LikeHandleHelper
88 {
89  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandleHelper)
90 
91 protected:
92  std::map<CPLString, CPLString> m_oMapQueryParameters{};
93 
94  virtual void RebuildURL() = 0;
95  CPLString GetQueryString(bool bAddEmptyValueAfterEqual) const;
96 
97 public:
98  IVSIS3LikeHandleHelper() = default;
99  virtual ~IVSIS3LikeHandleHelper() = default;
100 
101  void ResetQueryParameters();
102  void AddQueryParameter(const CPLString& osKey, const CPLString& osValue);
103 
104  virtual struct curl_slist* GetCurlHeaders(const CPLString& osVerb,
105  const struct curl_slist* psExistingHeaders,
106  const void *pabyDataContent = nullptr,
107  size_t nBytesContent = 0) const = 0;
108 
109  virtual bool AllowAutomaticRedirection() { return true; }
110  virtual bool CanRestartOnError(const char*, const char* /* pszHeaders*/,
111  bool /*bSetError*/, bool* /*pbUpdateMap*/ = nullptr) { return false;}
112 
113  virtual const CPLString& GetURL() const = 0;
114  CPLString GetURLNoKVP() const;
115 
116  virtual CPLString GetCopySourceHeader() const { return std::string(); }
117  virtual const char* GetMetadataDirectiveREPLACE() const { return ""; }
118 
119  static bool GetBucketAndObjectKey(const char* pszURI,
120  const char* pszFSPrefix,
121  bool bAllowNoObject,
122  CPLString &osBucketOut,
123  CPLString &osObjectKeyOut);
124 
125  static CPLString BuildCanonicalizedHeaders(
126  std::map<CPLString, CPLString>& oSortedMapHeaders,
127  const struct curl_slist* psExistingHeaders,
128  const char* pszHeaderPrefix);
129 
130  static CPLString GetRFC822DateTime();
131 };
132 
133 enum class AWSCredentialsSource
134 {
135  REGULAR, // credentials from env variables or ~/.aws/crediential
136  EC2, // credentials from EC2 private networking
137  WEB_IDENTITY, // credentials from Web Identity Token
138  // See https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
139  ASSUMED_ROLE // credentials from an STS assumed role
140  // See https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-cli.html
141  // and https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html
142 };
143 
144 class VSIS3HandleHelper final: public IVSIS3LikeHandleHelper
145 {
146  CPL_DISALLOW_COPY_ASSIGN(VSIS3HandleHelper)
147 
148  CPLString m_osURL{};
149  mutable CPLString m_osSecretAccessKey{};
150  mutable CPLString m_osAccessKeyId{};
151  mutable CPLString m_osSessionToken{};
152  CPLString m_osEndpoint{};
153  CPLString m_osRegion{};
154  CPLString m_osRequestPayer{};
155  CPLString m_osBucket{};
156  CPLString m_osObjectKey{};
157  bool m_bUseHTTPS = false;
158  bool m_bUseVirtualHosting = false;
159  AWSCredentialsSource m_eCredentialsSource = AWSCredentialsSource::REGULAR;
160 
161  void RebuildURL() override;
162 
163  static bool GetOrRefreshTemporaryCredentialsForRole(bool bForceRefresh,
164  CPLString& osSecretAccessKey,
165  CPLString& osAccessKeyId,
166  CPLString& osSessionToken,
167  CPLString& osRegion);
168 
169  static bool GetConfigurationFromAssumeRoleWithWebIdentity(bool bForceRefresh,
170  const std::string& osPathForOption,
171  const std::string& osRoleArnIn,
172  const std::string& osWebIdentityTokenFileIn,
173  CPLString& osSecretAccessKey,
174  CPLString& osAccessKeyId,
175  CPLString& osSessionToken);
176 
177  static bool GetConfigurationFromEC2(bool bForceRefresh,
178  const std::string& osPathForOption,
179  CPLString& osSecretAccessKey,
180  CPLString& osAccessKeyId,
181  CPLString& osSessionToken);
182 
183  static bool GetConfigurationFromAWSConfigFiles(
184  const std::string& osPathForOption,
185  const char* pszProfile,
186  CPLString& osSecretAccessKey,
187  CPLString& osAccessKeyId,
188  CPLString& osSessionToken,
189  CPLString& osRegion,
190  CPLString& osCredentials,
191  CPLString& osRoleArn,
192  CPLString& osSourceProfile,
193  CPLString& osExternalId,
194  CPLString& osMFASerial,
195  CPLString& osRoleSessionName,
196  CPLString& osWebIdentityTokenFile);
197 
198  static bool GetConfiguration(const std::string& osPathForOption,
199  CSLConstList papszOptions,
200  CPLString& osSecretAccessKey,
201  CPLString& osAccessKeyId,
202  CPLString& osSessionToken,
203  CPLString& osRegion,
204  AWSCredentialsSource& eCredentialsSource);
205 
206  void RefreshCredentials(const std::string& osPathForOption,
207  bool bForceRefresh) const;
208 
209  protected:
210 
211  public:
212  VSIS3HandleHelper(const CPLString& osSecretAccessKey,
213  const CPLString& osAccessKeyId,
214  const CPLString& osSessionToken,
215  const CPLString& osEndpoint,
216  const CPLString& osRegion,
217  const CPLString& osRequestPayer,
218  const CPLString& osBucket,
219  const CPLString& osObjectKey,
220  bool bUseHTTPS, bool bUseVirtualHosting,
221  AWSCredentialsSource eCredentialsSource);
222  ~VSIS3HandleHelper();
223 
224  static VSIS3HandleHelper* BuildFromURI(const char* pszURI,
225  const char* pszFSPrefix,
226  bool bAllowNoObject,
227  CSLConstList papszOptions = nullptr);
228  static CPLString BuildURL(const CPLString& osEndpoint,
229  const CPLString& osBucket,
230  const CPLString& osObjectKey,
231  bool bUseHTTPS, bool bUseVirtualHosting);
232 
233  struct curl_slist* GetCurlHeaders(
234  const CPLString& osVerb,
235  const struct curl_slist* psExistingHeaders,
236  const void *pabyDataContent = nullptr,
237  size_t nBytesContent = 0) const override;
238 
239  bool AllowAutomaticRedirection() override { return false; }
240  bool CanRestartOnError(const char*, const char* pszHeaders,
241  bool bSetError,
242  bool* pbUpdateMap = nullptr) override;
243 
244  const CPLString& GetURL() const override { return m_osURL; }
245  const CPLString& GetBucket() const { return m_osBucket; }
246  const CPLString& GetObjectKey() const { return m_osObjectKey; }
247  const CPLString& GetEndpoint()const { return m_osEndpoint; }
248  const CPLString& GetRegion() const { return m_osRegion; }
249  const CPLString& GetRequestPayer() const { return m_osRequestPayer; }
250  bool GetVirtualHosting() const { return m_bUseVirtualHosting; }
251  void SetEndpoint(const CPLString &osStr);
252  void SetRegion(const CPLString &osStr);
253  void SetRequestPayer(const CPLString &osStr);
254  void SetVirtualHosting(bool b);
255 
256  CPLString GetCopySourceHeader() const override { return "x-amz-copy-source"; }
257  const char* GetMetadataDirectiveREPLACE() const override { return "x-amz-metadata-directive: REPLACE"; }
258 
259  CPLString GetSignedURL(CSLConstList papszOptions);
260 
261  static void CleanMutex();
262  static void ClearCache();
263 };
264 
265 class VSIS3UpdateParams
266 {
267  public:
268  CPLString m_osRegion{};
269  CPLString m_osEndpoint{};
270  CPLString m_osRequestPayer{};
271  bool m_bUseVirtualHosting = false;
272 
273  VSIS3UpdateParams() = default;
274 
275  explicit VSIS3UpdateParams(const VSIS3HandleHelper* poHelper) :
276  m_osRegion(poHelper->GetRegion()),
277  m_osEndpoint(poHelper->GetEndpoint()),
278  m_osRequestPayer(poHelper->GetRequestPayer()),
279  m_bUseVirtualHosting(poHelper->GetVirtualHosting()) {}
280 
281  void UpdateHandlerHelper(VSIS3HandleHelper* poHelper) {
282  poHelper->SetRegion(m_osRegion);
283  poHelper->SetEndpoint(m_osEndpoint);
284  poHelper->SetRequestPayer(m_osRequestPayer);
285  poHelper->SetVirtualHosting(m_bUseVirtualHosting);
286  }
287 
288  static std::mutex gsMutex;
289  static std::map< CPLString, VSIS3UpdateParams > goMapBucketsToS3Params;
290  static void UpdateMapFromHandle( IVSIS3LikeHandleHelper* poHandleHelper );
291  static void UpdateHandleFromMap( IVSIS3LikeHandleHelper* poHandleHelper );
292  static void ClearCache();
293 };
294 
295 #endif /* HAVE_CURL */
296 
297 #endif /* #ifndef DOXYGEN_SKIP */
298 
299 #endif /* CPL_AWS_INCLUDED_H */
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:320
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1056
cpl_string.h
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:230
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:930