GDAL
cpl_aws.h
1 /**********************************************************************
2  * $Id$
3  *
4  * Name: cpl_aws.h
5  * Project: CPL - Common Portability Library
6  * Purpose: Amazon Web Services routines
7  * Author: Even Rouault <even.rouault at spatialys.com>
8  *
9  **********************************************************************
10  * Copyright (c) 2015, Even Rouault <even.rouault at spatialys.com>
11  *
12  * Permission is hereby granted, free of charge, to any person obtaining a
13  * copy of this software and associated documentation files (the "Software"),
14  * to deal in the Software without restriction, including without limitation
15  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16  * and/or sell copies of the Software, and to permit persons to whom the
17  * Software is furnished to do so, subject to the following conditions:
18  *
19  * The above copyright notice and this permission notice shall be included
20  * in all copies or substantial portions of the Software.
21  *
22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28  * DEALINGS IN THE SOFTWARE.
29  ****************************************************************************/
30 
31 #ifndef CPL_AWS_INCLUDED_H
32 #define CPL_AWS_INCLUDED_H
33 
34 #ifndef DOXYGEN_SKIP
35 
36 #ifdef HAVE_CURL
37 
38 #include <cstddef>
39 #include <mutex>
40 
41 #include "cpl_string.h"
42 
43 #include <curl/curl.h>
44 #include <map>
45 
46 CPLString CPLGetLowerCaseHexSHA256(const void *pabyData, size_t nBytes);
47 CPLString CPLGetLowerCaseHexSHA256(const CPLString &osStr);
48 
49 CPLString CPLGetAWS_SIGN4_Timestamp(GIntBig timestamp);
50 
51 CPLString CPLAWSURLEncode(const CPLString &osURL, bool bEncodeSlash = true);
52 
53 CPLString CPLAWSGetHeaderVal(const struct curl_slist *psExistingHeaders,
54  const char *pszKey);
55 
56 CPLString CPLGetAWS_SIGN4_Signature(
57  const CPLString &osSecretAccessKey, const CPLString &osAccessToken,
58  const CPLString &osRegion, const CPLString &osRequestPayer,
59  const CPLString &osService, const CPLString &osVerb,
60  const struct curl_slist *psExistingHeaders, const CPLString &osHost,
61  const CPLString &osCanonicalURI, const CPLString &osCanonicalQueryString,
62  const CPLString &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
63  const CPLString &osTimestamp, CPLString &osSignedHeaders);
64 
65 CPLString CPLGetAWS_SIGN4_Authorization(
66  const CPLString &osSecretAccessKey, const CPLString &osAccessKeyId,
67  const CPLString &osAccessToken, const CPLString &osRegion,
68  const CPLString &osRequestPayer, const CPLString &osService,
69  const CPLString &osVerb, const struct curl_slist *psExistingHeaders,
70  const CPLString &osHost, const CPLString &osCanonicalURI,
71  const CPLString &osCanonicalQueryString,
72  const CPLString &osXAMZContentSHA256, bool bAddHeaderAMZContentSHA256,
73  const CPLString &osTimestamp);
74 
75 class IVSIS3LikeHandleHelper
76 {
77  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandleHelper)
78 
79  protected:
80  std::map<CPLString, CPLString> m_oMapQueryParameters{};
81 
82  virtual void RebuildURL() = 0;
83  CPLString GetQueryString(bool bAddEmptyValueAfterEqual) const;
84 
85  public:
86  IVSIS3LikeHandleHelper() = default;
87  virtual ~IVSIS3LikeHandleHelper() = default;
88 
89  void ResetQueryParameters();
90  void AddQueryParameter(const CPLString &osKey, const CPLString &osValue);
91 
92  virtual struct curl_slist *
93  GetCurlHeaders(const CPLString &osVerb,
94  const struct curl_slist *psExistingHeaders,
95  const void *pabyDataContent = nullptr,
96  size_t nBytesContent = 0) const = 0;
97 
98  virtual bool AllowAutomaticRedirection()
99  {
100  return true;
101  }
102  virtual bool CanRestartOnError(const char *, const char * /* pszHeaders*/,
103  bool /*bSetError*/)
104  {
105  return false;
106  }
107 
108  virtual const CPLString &GetURL() const = 0;
109  CPLString GetURLNoKVP() const;
110 
111  virtual CPLString GetCopySourceHeader() const
112  {
113  return std::string();
114  }
115  virtual const char *GetMetadataDirectiveREPLACE() const
116  {
117  return "";
118  }
119 
120  static bool GetBucketAndObjectKey(const char *pszURI,
121  const char *pszFSPrefix,
122  bool bAllowNoObject,
123  CPLString &osBucketOut,
124  CPLString &osObjectKeyOut);
125 
126  static CPLString
127  BuildCanonicalizedHeaders(std::map<CPLString, CPLString> &oSortedMapHeaders,
128  const struct curl_slist *psExistingHeaders,
129  const char *pszHeaderPrefix);
130 
131  static CPLString GetRFC822DateTime();
132 };
133 
134 enum class AWSCredentialsSource
135 {
136  REGULAR, // credentials from env variables or ~/.aws/crediential
137  EC2, // credentials from EC2 private networking
138  WEB_IDENTITY, // credentials from Web Identity Token
139  // See
140  // https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html
141  ASSUMED_ROLE // credentials from an STS assumed role
142  // See
143  // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-cli.html
144  // and
145  // https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html
146 };
147 
148 class VSIS3HandleHelper final : public IVSIS3LikeHandleHelper
149 {
150  CPL_DISALLOW_COPY_ASSIGN(VSIS3HandleHelper)
151 
152  CPLString m_osURL{};
153  mutable CPLString m_osSecretAccessKey{};
154  mutable CPLString m_osAccessKeyId{};
155  mutable CPLString m_osSessionToken{};
156  CPLString m_osEndpoint{};
157  CPLString m_osRegion{};
158  CPLString m_osRequestPayer{};
159  CPLString m_osBucket{};
160  CPLString m_osObjectKey{};
161  bool m_bUseHTTPS = false;
162  bool m_bUseVirtualHosting = false;
163  AWSCredentialsSource m_eCredentialsSource = AWSCredentialsSource::REGULAR;
164 
165  void RebuildURL() override;
166 
167  static bool GetOrRefreshTemporaryCredentialsForRole(
168  bool bForceRefresh, CPLString &osSecretAccessKey,
169  CPLString &osAccessKeyId, CPLString &osSessionToken,
170  CPLString &osRegion);
171 
172  static bool GetConfigurationFromAssumeRoleWithWebIdentity(
173  bool bForceRefresh, const std::string &osPathForOption,
174  const std::string &osRoleArnIn,
175  const std::string &osWebIdentityTokenFileIn,
176  CPLString &osSecretAccessKey, CPLString &osAccessKeyId,
177  CPLString &osSessionToken);
178 
179  static bool GetConfigurationFromEC2(bool bForceRefresh,
180  const std::string &osPathForOption,
181  CPLString &osSecretAccessKey,
182  CPLString &osAccessKeyId,
183  CPLString &osSessionToken);
184 
185  static bool GetConfigurationFromAWSConfigFiles(
186  const std::string &osPathForOption, const char *pszProfile,
187  CPLString &osSecretAccessKey, CPLString &osAccessKeyId,
188  CPLString &osSessionToken, CPLString &osRegion,
189  CPLString &osCredentials, CPLString &osRoleArn,
190  CPLString &osSourceProfile, CPLString &osExternalId,
191  CPLString &osMFASerial, CPLString &osRoleSessionName,
192  CPLString &osWebIdentityTokenFile);
193 
194  static bool GetConfiguration(const std::string &osPathForOption,
195  CSLConstList papszOptions,
196  CPLString &osSecretAccessKey,
197  CPLString &osAccessKeyId,
198  CPLString &osSessionToken, CPLString &osRegion,
199  AWSCredentialsSource &eCredentialsSource);
200 
201  void RefreshCredentials(const std::string &osPathForOption,
202  bool bForceRefresh) const;
203 
204  protected:
205  public:
206  VSIS3HandleHelper(const CPLString &osSecretAccessKey,
207  const CPLString &osAccessKeyId,
208  const CPLString &osSessionToken,
209  const CPLString &osEndpoint, const CPLString &osRegion,
210  const CPLString &osRequestPayer,
211  const CPLString &osBucket, const CPLString &osObjectKey,
212  bool bUseHTTPS, bool bUseVirtualHosting,
213  AWSCredentialsSource eCredentialsSource);
214  ~VSIS3HandleHelper();
215 
216  static VSIS3HandleHelper *BuildFromURI(const char *pszURI,
217  const char *pszFSPrefix,
218  bool bAllowNoObject,
219  CSLConstList papszOptions = nullptr);
220  static CPLString BuildURL(const CPLString &osEndpoint,
221  const CPLString &osBucket,
222  const CPLString &osObjectKey, bool bUseHTTPS,
223  bool bUseVirtualHosting);
224 
225  struct curl_slist *
226  GetCurlHeaders(const CPLString &osVerb,
227  const struct curl_slist *psExistingHeaders,
228  const void *pabyDataContent = nullptr,
229  size_t nBytesContent = 0) const override;
230 
231  bool AllowAutomaticRedirection() override
232  {
233  return false;
234  }
235  bool CanRestartOnError(const char *, const char *pszHeaders,
236  bool bSetError) override;
237 
238  const CPLString &GetURL() const override
239  {
240  return m_osURL;
241  }
242  const CPLString &GetBucket() const
243  {
244  return m_osBucket;
245  }
246  const CPLString &GetObjectKey() const
247  {
248  return m_osObjectKey;
249  }
250  const CPLString &GetEndpoint() const
251  {
252  return m_osEndpoint;
253  }
254  const CPLString &GetRegion() const
255  {
256  return m_osRegion;
257  }
258  const CPLString &GetRequestPayer() const
259  {
260  return m_osRequestPayer;
261  }
262  bool GetVirtualHosting() const
263  {
264  return m_bUseVirtualHosting;
265  }
266  void SetEndpoint(const CPLString &osStr);
267  void SetRegion(const CPLString &osStr);
268  void SetRequestPayer(const CPLString &osStr);
269  void SetVirtualHosting(bool b);
270 
271  CPLString GetCopySourceHeader() const override
272  {
273  return "x-amz-copy-source";
274  }
275  const char *GetMetadataDirectiveREPLACE() const override
276  {
277  return "x-amz-metadata-directive: REPLACE";
278  }
279 
280  CPLString GetSignedURL(CSLConstList papszOptions);
281 
282  static void CleanMutex();
283  static void ClearCache();
284 };
285 
286 class VSIS3UpdateParams
287 {
288  private:
289  CPLString m_osRegion{};
290  CPLString m_osEndpoint{};
291  CPLString m_osRequestPayer{};
292  bool m_bUseVirtualHosting = false;
293 
294  explicit VSIS3UpdateParams(const VSIS3HandleHelper *poHelper)
295  : m_osRegion(poHelper->GetRegion()),
296  m_osEndpoint(poHelper->GetEndpoint()),
297  m_osRequestPayer(poHelper->GetRequestPayer()),
298  m_bUseVirtualHosting(poHelper->GetVirtualHosting())
299  {
300  }
301 
302  void UpdateHandlerHelper(VSIS3HandleHelper *poHelper)
303  {
304  poHelper->SetRegion(m_osRegion);
305  poHelper->SetEndpoint(m_osEndpoint);
306  poHelper->SetRequestPayer(m_osRequestPayer);
307  poHelper->SetVirtualHosting(m_bUseVirtualHosting);
308  }
309 
310  static std::mutex gsMutex;
311  static std::map<CPLString, VSIS3UpdateParams> goMapBucketsToS3Params;
312 
313  public:
314  VSIS3UpdateParams() = default;
315 
316  static void UpdateMapFromHandle(VSIS3HandleHelper *poS3HandleHelper);
317  static void UpdateHandleFromMap(VSIS3HandleHelper *poS3HandleHelper);
318  static void ClearCache();
319 };
320 
321 #endif /* HAVE_CURL */
322 
323 #endif /* #ifndef DOXYGEN_SKIP */
324 
325 #endif /* CPL_AWS_INCLUDED_H */
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:311
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1178
cpl_string.h
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:226
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1042