GDAL
cpl_aws.h
1 /**********************************************************************
2  * $Id$
3  *
4  * Name: cpl_aws.h
5  * Project: CPL - Common Portability Library
6  * Purpose: Amazon Web Services routines
7  * Author: Even Rouault <even.rouault at spatialys.com>
8  *
9  **********************************************************************
10  * Copyright (c) 2015, Even Rouault <even.rouault at spatialys.com>
11  *
12  * Permission is hereby granted, free of charge, to any person obtaining a
13  * copy of this software and associated documentation files (the "Software"),
14  * to deal in the Software without restriction, including without limitation
15  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16  * and/or sell copies of the Software, and to permit persons to whom the
17  * Software is furnished to do so, subject to the following conditions:
18  *
19  * The above copyright notice and this permission notice shall be included
20  * in all copies or substantial portions of the Software.
21  *
22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28  * DEALINGS IN THE SOFTWARE.
29  ****************************************************************************/
30 
31 #ifndef CPL_AWS_INCLUDED_H
32 #define CPL_AWS_INCLUDED_H
33 
34 #ifndef DOXYGEN_SKIP
35 
36 #ifdef HAVE_CURL
37 
38 #include <cstddef>
39 #include <mutex>
40 
41 #include "cpl_string.h"
42 
43 #include <curl/curl.h>
44 #include <map>
45 
46 CPLString CPLGetLowerCaseHexSHA256(const void *pabyData, size_t nBytes);
47 CPLString CPLGetLowerCaseHexSHA256(const CPLString &osStr);
48 
49 CPLString CPLGetAWS_SIGN4_Timestamp(GIntBig timestamp);
50 
51 CPLString CPLAWSURLEncode(const CPLString &osURL, bool bEncodeSlash = true);
52 
53 CPLString CPLAWSGetHeaderVal(const struct curl_slist *psExistingHeaders,
54  const char *pszKey);
55 
56 CPLString CPLGetAWS_SIGN4_Signature(
57  const CPLString &osSecretAccessKey, const CPLString &osAccessToken,
58  const CPLString &osRegion, const CPLString &osRequestPayer,
59  const CPLString &osService, const CPLString &osVerb,
60  const struct curl_slist *psExistingHeaders, const CPLString &osHost,
61  const CPLString &osCanonicalURI, const CPLString &osCanonicalQueryString,
62  const CPLString &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
63  const CPLString &osTimestamp, CPLString &osSignedHeaders);
64 
65 CPLString CPLGetAWS_SIGN4_Authorization(
66  const CPLString &osSecretAccessKey, const CPLString &osAccessKeyId,
67  const CPLString &osAccessToken, const CPLString &osRegion,
68  const CPLString &osRequestPayer, const CPLString &osService,
69  const CPLString &osVerb, const struct curl_slist *psExistingHeaders,
70  const CPLString &osHost, const CPLString &osCanonicalURI,
71  const CPLString &osCanonicalQueryString,
72  const CPLString &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
73  const CPLString &osTimestamp);
74 
75 class IVSIS3LikeHandleHelper
76 {
77  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandleHelper)
78 
79  protected:
80  std::map<CPLString, CPLString> m_oMapQueryParameters{};
81 
82  virtual void RebuildURL() = 0;
83  CPLString GetQueryString(bool bAddEmptyValueAfterEqual) const;
84 
85  public:
86  IVSIS3LikeHandleHelper() = default;
87  virtual ~IVSIS3LikeHandleHelper() = default;
88 
89  void ResetQueryParameters();
90  void AddQueryParameter(const CPLString &osKey, const CPLString &osValue);
91 
92  virtual struct curl_slist *
93  GetCurlHeaders(const CPLString &osVerb,
94  const struct curl_slist *psExistingHeaders,
95  const void *pabyDataContent = nullptr,
96  size_t nBytesContent = 0) const = 0;
97 
98  virtual bool AllowAutomaticRedirection()
99  {
100  return true;
101  }
102  virtual bool CanRestartOnError(const char *, const char * /* pszHeaders*/,
103  bool /*bSetError*/,
104  bool * /*pbUpdateMap*/ = nullptr)
105  {
106  return false;
107  }
108 
109  virtual const CPLString &GetURL() const = 0;
110  CPLString GetURLNoKVP() const;
111 
112  virtual CPLString GetCopySourceHeader() const
113  {
114  return std::string();
115  }
116  virtual const char *GetMetadataDirectiveREPLACE() const
117  {
118  return "";
119  }
120 
121  static bool GetBucketAndObjectKey(const char *pszURI,
122  const char *pszFSPrefix,
123  bool bAllowNoObject,
124  CPLString &osBucketOut,
125  CPLString &osObjectKeyOut);
126 
127  static CPLString
128  BuildCanonicalizedHeaders(std::map<CPLString, CPLString> &oSortedMapHeaders,
129  const struct curl_slist *psExistingHeaders,
130  const char *pszHeaderPrefix);
131 
132  static CPLString GetRFC822DateTime();
133 };
134 
135 enum class AWSCredentialsSource
136 {
137  REGULAR, // credentials from env variables or ~/.aws/crediential
138  EC2, // credentials from EC2 private networking
139  WEB_IDENTITY, // credentials from Web Identity Token
140  // See
141  // https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
142  ASSUMED_ROLE // credentials from an STS assumed role
143  // See
144  // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-cli.html
145  // and
146  // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html
147 };
148 
149 class VSIS3HandleHelper final : public IVSIS3LikeHandleHelper
150 {
151  CPL_DISALLOW_COPY_ASSIGN(VSIS3HandleHelper)
152 
153  CPLString m_osURL{};
154  mutable CPLString m_osSecretAccessKey{};
155  mutable CPLString m_osAccessKeyId{};
156  mutable CPLString m_osSessionToken{};
157  CPLString m_osEndpoint{};
158  CPLString m_osRegion{};
159  CPLString m_osRequestPayer{};
160  CPLString m_osBucket{};
161  CPLString m_osObjectKey{};
162  bool m_bUseHTTPS = false;
163  bool m_bUseVirtualHosting = false;
164  AWSCredentialsSource m_eCredentialsSource = AWSCredentialsSource::REGULAR;
165 
166  void RebuildURL() override;
167 
168  static bool GetOrRefreshTemporaryCredentialsForRole(
169  bool bForceRefresh, CPLString &osSecretAccessKey,
170  CPLString &osAccessKeyId, CPLString &osSessionToken,
171  CPLString &osRegion);
172 
173  static bool GetConfigurationFromAssumeRoleWithWebIdentity(
174  bool bForceRefresh, const std::string &osPathForOption,
175  const std::string &osRoleArnIn,
176  const std::string &osWebIdentityTokenFileIn,
177  CPLString &osSecretAccessKey, CPLString &osAccessKeyId,
178  CPLString &osSessionToken);
179 
180  static bool GetConfigurationFromEC2(bool bForceRefresh,
181  const std::string &osPathForOption,
182  CPLString &osSecretAccessKey,
183  CPLString &osAccessKeyId,
184  CPLString &osSessionToken);
185 
186  static bool GetConfigurationFromAWSConfigFiles(
187  const std::string &osPathForOption, const char *pszProfile,
188  CPLString &osSecretAccessKey, CPLString &osAccessKeyId,
189  CPLString &osSessionToken, CPLString &osRegion,
190  CPLString &osCredentials, CPLString &osRoleArn,
191  CPLString &osSourceProfile, CPLString &osExternalId,
192  CPLString &osMFASerial, CPLString &osRoleSessionName,
193  CPLString &osWebIdentityTokenFile);
194 
195  static bool GetConfiguration(const std::string &osPathForOption,
196  CSLConstList papszOptions,
197  CPLString &osSecretAccessKey,
198  CPLString &osAccessKeyId,
199  CPLString &osSessionToken, CPLString &osRegion,
200  AWSCredentialsSource &eCredentialsSource);
201 
202  void RefreshCredentials(const std::string &osPathForOption,
203  bool bForceRefresh) const;
204 
205  protected:
206  public:
207  VSIS3HandleHelper(const CPLString &osSecretAccessKey,
208  const CPLString &osAccessKeyId,
209  const CPLString &osSessionToken,
210  const CPLString &osEndpoint, const CPLString &osRegion,
211  const CPLString &osRequestPayer,
212  const CPLString &osBucket, const CPLString &osObjectKey,
213  bool bUseHTTPS, bool bUseVirtualHosting,
214  AWSCredentialsSource eCredentialsSource);
215  ~VSIS3HandleHelper();
216 
217  static VSIS3HandleHelper *BuildFromURI(const char *pszURI,
218  const char *pszFSPrefix,
219  bool bAllowNoObject,
220  CSLConstList papszOptions = nullptr);
221  static CPLString BuildURL(const CPLString &osEndpoint,
222  const CPLString &osBucket,
223  const CPLString &osObjectKey, bool bUseHTTPS,
224  bool bUseVirtualHosting);
225 
226  struct curl_slist *
227  GetCurlHeaders(const CPLString &osVerb,
228  const struct curl_slist *psExistingHeaders,
229  const void *pabyDataContent = nullptr,
230  size_t nBytesContent = 0) const override;
231 
232  bool AllowAutomaticRedirection() override
233  {
234  return false;
235  }
236  bool CanRestartOnError(const char *, const char *pszHeaders, bool bSetError,
237  bool *pbUpdateMap = nullptr) override;
238 
239  const CPLString &GetURL() const override
240  {
241  return m_osURL;
242  }
243  const CPLString &GetBucket() const
244  {
245  return m_osBucket;
246  }
247  const CPLString &GetObjectKey() const
248  {
249  return m_osObjectKey;
250  }
251  const CPLString &GetEndpoint() const
252  {
253  return m_osEndpoint;
254  }
255  const CPLString &GetRegion() const
256  {
257  return m_osRegion;
258  }
259  const CPLString &GetRequestPayer() const
260  {
261  return m_osRequestPayer;
262  }
263  bool GetVirtualHosting() const
264  {
265  return m_bUseVirtualHosting;
266  }
267  void SetEndpoint(const CPLString &osStr);
268  void SetRegion(const CPLString &osStr);
269  void SetRequestPayer(const CPLString &osStr);
270  void SetVirtualHosting(bool b);
271 
272  CPLString GetCopySourceHeader() const override
273  {
274  return "x-amz-copy-source";
275  }
276  const char *GetMetadataDirectiveREPLACE() const override
277  {
278  return "x-amz-metadata-directive: REPLACE";
279  }
280 
281  CPLString GetSignedURL(CSLConstList papszOptions);
282 
283  static void CleanMutex();
284  static void ClearCache();
285 };
286 
287 class VSIS3UpdateParams
288 {
289  public:
290  CPLString m_osRegion{};
291  CPLString m_osEndpoint{};
292  CPLString m_osRequestPayer{};
293  bool m_bUseVirtualHosting = false;
294 
295  VSIS3UpdateParams() = default;
296 
297  explicit VSIS3UpdateParams(const VSIS3HandleHelper *poHelper)
298  : m_osRegion(poHelper->GetRegion()),
299  m_osEndpoint(poHelper->GetEndpoint()),
300  m_osRequestPayer(poHelper->GetRequestPayer()),
301  m_bUseVirtualHosting(poHelper->GetVirtualHosting())
302  {
303  }
304 
305  void UpdateHandlerHelper(VSIS3HandleHelper *poHelper)
306  {
307  poHelper->SetRegion(m_osRegion);
308  poHelper->SetEndpoint(m_osEndpoint);
309  poHelper->SetRequestPayer(m_osRequestPayer);
310  poHelper->SetVirtualHosting(m_bUseVirtualHosting);
311  }
312 
313  static std::mutex gsMutex;
314  static std::map<CPLString, VSIS3UpdateParams> goMapBucketsToS3Params;
315  static void UpdateMapFromHandle(IVSIS3LikeHandleHelper *poHandleHelper);
316  static void UpdateHandleFromMap(IVSIS3LikeHandleHelper *poHandleHelper);
317  static void ClearCache();
318 };
319 
320 #endif /* HAVE_CURL */
321 
322 #endif /* #ifndef DOXYGEN_SKIP */
323 
324 #endif /* CPL_AWS_INCLUDED_H */
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:311
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1195
cpl_string.h
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:235
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1051